snix_glue/
snix_store_io.rs

1//! This module provides an implementation of EvalIO talking to snix-store.
2use nix_compat::store_path::StorePathRef;
3use snix_build::buildservice::BuildService;
4use snix_build_glue::build_state::BuildState;
5use snix_eval::{EvalIO, FileType, StdIO};
6use snix_store::nar::NarCalculationService;
7use std::{
8    env,
9    ffi::{OsStr, OsString},
10    io,
11    sync::Arc,
12};
13use tokio_util::io::SyncIoBridge;
14use tracing::{Level, error, instrument};
15use url::Url;
16
17use snix_castore::{Node, blob_engine::BlobEngine, directoryservice::DirectoryService};
18use snix_store::pathinfoservice::{PathInfo, PathInfoService};
19
20/// Implements [EvalIO], asking given [PathInfoService], [DirectoryService]
21/// and [BlobEngine].
22///
23/// In case the given path does not exist in these stores, we ask StdIO.
24/// This is to both cover cases of syntactically valid store paths, that exist
25/// on the filesystem (still managed by Nix), as well as being able to read
26/// files outside store paths.
27///
28/// This structure is also directly used by the derivation builtins
29/// and tightly coupled to it.
30///
31/// In the future, we may revisit that coupling and figure out how to generalize this interface and
32/// hide this implementation detail of the glue itself so that glue can be used with more than one
33/// implementation of "Snix Store IO" which does not necessarily bring the concept of blob service,
34/// directory service or path info service.
35pub struct SnixStoreIO {
36    pub build_state: BuildState,
37
38    std_io: StdIO,
39    pub(crate) tokio_handle: tokio::runtime::Handle,
40}
41
42impl SnixStoreIO {
43    pub fn new(
44        blob_engine: Arc<dyn BlobEngine>,
45        directory_service: Arc<dyn DirectoryService>,
46        path_info_service: Arc<dyn PathInfoService>,
47        nar_calculation_service: Arc<dyn NarCalculationService>,
48        build_service: Arc<dyn BuildService>,
49        tokio_handle: tokio::runtime::Handle,
50        hashed_mirrors: Vec<Url>,
51    ) -> Self {
52        Self {
53            build_state: BuildState::new(
54                blob_engine,
55                directory_service,
56                path_info_service,
57                nar_calculation_service,
58                build_service,
59                hashed_mirrors,
60            ),
61            std_io: StdIO {},
62            tokio_handle,
63        }
64    }
65
66    /// for a given [StorePath] and additional [snix_castore::Path] inside the store path,
67    /// look up the [PathInfo], and if it exists, and then uses
68    /// [descend_to] to return the [Node] specified by `sub_path`.
69    ///
70    /// In case there is no PathInfo yet, we check "self.known_paths" (learnt by
71    /// the evaluator)
72    ///  - If there's a fetch, we do that and return the resulting PathInfo.
73    ///  - If there's a Derivation, we build it and return the resulting
74    ///    PathInfo.
75    ///    To build it, we need to do a function call to ourselves with all
76    ///    inputs of that Derivation, which will trigger fetches / builds of
77    ///    inputs, recursively.
78    ///
79    /// This should be replaced with a proper scheduler knowing about a partial
80    /// subgraph at some point, because this design doesn't allow concurrent
81    /// builds yet.
82    ///
83    /// [StorePath]: nix_compat::store_path::StorePath
84    /// [descend_to]: snix_castore::directoryservice::traversal::descend_to
85    async fn store_path_to_path_info(
86        &self,
87        store_path: &StorePathRef<'_>,
88        sub_path: &snix_castore::Path,
89    ) -> io::Result<Option<PathInfo>> {
90        self.build_state
91            .store_path_to_path_info(store_path, sub_path)
92            .await
93    }
94}
95
96/// Helper function peeking at a [snix_castore::Node] and returning its [FileType]
97fn node_get_type(node: &Node) -> FileType {
98    match node {
99        Node::Directory { .. } => FileType::Directory,
100        Node::File { .. } => FileType::Regular,
101        Node::Symlink { .. } => FileType::Symlink,
102    }
103}
104
105// Helper function converting a [std::path::Path] to a [StorePath] and [snix_castore::Path].
106#[cfg(unix)]
107fn parse_store_and_sub_path<'a>(
108    path: &'a std::path::Path,
109) -> io::Result<(StorePathRef<'a>, &'a snix_castore::Path)> {
110    let (store_path, rest) =
111        StorePathRef::from_absolute_path_full(path).map_err(std::io::Error::other)?;
112
113    use std::os::unix::ffi::OsStrExt;
114    let sub_path = snix_castore::Path::from_bytes(rest.as_os_str().as_bytes())
115        .ok_or_else(|| std::io::Error::other("sub_path is no valid path"))?;
116
117    Ok((store_path, sub_path))
118}
119
120impl EvalIO for SnixStoreIO {
121    #[instrument(skip(self), ret(level = Level::TRACE), err)]
122    fn path_exists(&self, path: &std::path::Path) -> io::Result<bool> {
123        if let Ok((store_path, sub_path)) = parse_store_and_sub_path(path) {
124            if self
125                .tokio_handle
126                .block_on(self.store_path_to_path_info(&store_path, sub_path))?
127                .is_some()
128            {
129                Ok(true)
130            } else {
131                // As snix-store doesn't manage /nix/store on the filesystem,
132                // we still need to also ask self.std_io here.
133                self.std_io.path_exists(path)
134            }
135        } else {
136            // The store path is no store path, so do regular StdIO.
137            self.std_io.path_exists(path)
138        }
139    }
140
141    #[instrument(skip(self), err)]
142    fn open(&self, path: &std::path::Path) -> io::Result<Box<dyn io::Read>> {
143        if let Ok((store_path, sub_path)) = parse_store_and_sub_path(path) {
144            self.tokio_handle.block_on(async {
145                if let Some(path_info) = self.store_path_to_path_info(&store_path, sub_path).await?
146                {
147                    // depending on the node type, treat open differently
148                    match path_info.node {
149                        Node::Directory { .. } => {
150                            // This would normally be a io::ErrorKind::IsADirectory (still unstable)
151                            Err(io::Error::new(
152                                io::ErrorKind::Unsupported,
153                                format!("tried to open directory at {path:?}"),
154                            ))
155                        }
156                        Node::File { digest, size, .. } => {
157                            if let Some(blob_reader) = self
158                                .build_state
159                                .blob_engine
160                                .open_read(&digest, Some(size))
161                                .await
162                                .map_err(std::io::Error::other)?
163                            {
164                                Ok(Box::new(SyncIoBridge::new(blob_reader)) as Box<dyn io::Read>)
165                            } else {
166                                error!(
167                                    blob.digest = %digest,
168                                    "blob not found",
169                                );
170                                Err(io::Error::new(
171                                    io::ErrorKind::NotFound,
172                                    format!("blob {} not found", digest),
173                                ))
174                            }
175                        }
176                        Node::Symlink { .. } => Err(io::Error::new(
177                            io::ErrorKind::Unsupported,
178                            "open for symlinks is unsupported",
179                        ))?,
180                    }
181                } else {
182                    // As snix-store doesn't manage /nix/store on the filesystem,
183                    // we still need to also ask self.std_io here.
184                    self.std_io.open(path)
185                }
186            })
187        } else {
188            // The store path is no store path, so do regular StdIO.
189            self.std_io.open(path)
190        }
191    }
192
193    #[instrument(skip(self), ret(level = Level::TRACE), err)]
194    fn file_type(&self, path: &std::path::Path) -> io::Result<FileType> {
195        if let Ok((store_path, sub_path)) = parse_store_and_sub_path(path) {
196            if let Some(path_info) = self
197                .tokio_handle
198                .block_on(async { self.store_path_to_path_info(&store_path, sub_path).await })?
199            {
200                Ok(node_get_type(&path_info.node))
201            } else {
202                self.std_io.file_type(path)
203            }
204        } else {
205            self.std_io.file_type(path)
206        }
207    }
208
209    #[instrument(skip(self), ret(level = Level::TRACE), err)]
210    fn read_dir(&self, path: &std::path::Path) -> io::Result<Vec<(bytes::Bytes, FileType)>> {
211        if let Ok((store_path, sub_path)) = parse_store_and_sub_path(path) {
212            self.tokio_handle.block_on(async {
213                if let Some(path_info) = self.store_path_to_path_info(&store_path, sub_path).await?
214                {
215                    match path_info.node {
216                        Node::Directory { digest, .. } => {
217                            // fetch the Directory itself.
218                            let directory = self
219                                .build_state
220                                .directory_service
221                                .as_ref()
222                                .get(&digest)
223                                .await
224                                .map_err(std::io::Error::other)?
225                                .ok_or_else(|| {
226                                    // If we didn't get the directory node that's linked, that's a store inconsistency!
227                                    error!(
228                                        directory.digest = %digest,
229                                        path = ?path,
230                                        "directory not found",
231                                    );
232                                    io::Error::new(
233                                        io::ErrorKind::NotFound,
234                                        format!("directory {digest} does not exist"),
235                                    )
236                                })?;
237
238                            // construct children from nodes
239                            Ok(directory
240                                .into_nodes()
241                                .map(|(name, node)| (name.into(), node_get_type(&node)))
242                                .collect())
243                        }
244                        Node::File { .. } => {
245                            // This would normally be a io::ErrorKind::NotADirectory (still unstable)
246                            Err(io::Error::new(
247                                io::ErrorKind::Unsupported,
248                                "tried to readdir path {:?}, which is a file",
249                            ))?
250                        }
251                        Node::Symlink { .. } => Err(io::Error::new(
252                            io::ErrorKind::Unsupported,
253                            "read_dir for symlinks is unsupported",
254                        ))?,
255                    }
256                } else {
257                    self.std_io.read_dir(path)
258                }
259            })
260        } else {
261            self.std_io.read_dir(path)
262        }
263    }
264
265    #[instrument(skip(self), ret(level = Level::TRACE), err)]
266    fn import_path(&self, path: &std::path::Path) -> io::Result<std::path::PathBuf> {
267        let file_name = path.file_name().ok_or_else(|| {
268            io::Error::new(
269                io::ErrorKind::InvalidFilename,
270                "path without basename encountered",
271            )
272        })?;
273        let name = nix_compat::store_path::StorePathName::from_os_str(file_name)
274            .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
275        let path_info = self.tokio_handle.block_on({
276            snix_store::import::import_path_as_nar_ca(
277                path,
278                name,
279                &self.build_state.blob_engine,
280                &self.build_state.directory_service,
281                &self.build_state.path_info_service,
282                &self.build_state.nar_calculation_service,
283            )
284        })?;
285
286        // From the returned PathInfo, extract the store path and return it.
287        Ok(path_info.store_path.to_absolute_path().into())
288    }
289
290    #[instrument(skip(self), ret(level = Level::TRACE))]
291    fn store_dir(&self) -> Option<String> {
292        Some("/nix/store".to_string())
293    }
294
295    fn get_env(&self, key: &OsStr) -> Option<OsString> {
296        env::var_os(key)
297    }
298}
299
300#[cfg(test)]
301mod tests {
302    use std::{path::Path, rc::Rc, sync::Arc};
303
304    use bstr::ByteSlice;
305    use clap::Parser;
306    use snix_build::buildservice::DummyBuildService;
307
308    use snix_eval::{EvalIO, EvaluationResult};
309    use snix_store::utils::{ServiceUrlsMemory, construct_services};
310    use tempfile::TempDir;
311
312    use super::SnixStoreIO;
313    use crate::builtins::{add_derivation_builtins, add_fetcher_builtins, add_import_builtins};
314
315    /// evaluates a given nix expression and returns the result.
316    /// Takes care of setting up the evaluator so it knows about the
317    // `derivation` builtin.
318    fn eval(str: &str) -> EvaluationResult {
319        let tokio_runtime = tokio::runtime::Runtime::new().unwrap();
320        let (
321            _blob_store,
322            _chunk_store,
323            blob_engine,
324            directory_service,
325            path_info_service,
326            nar_calculation_service,
327        ) = tokio_runtime
328            .block_on(async {
329                construct_services(ServiceUrlsMemory::parse_from(std::iter::empty::<&str>())).await
330            })
331            .unwrap();
332
333        let io = Rc::new(SnixStoreIO::new(
334            blob_engine,
335            directory_service,
336            path_info_service,
337            nar_calculation_service,
338            Arc::<DummyBuildService>::default(),
339            tokio_runtime.handle().clone(),
340            Vec::new(),
341        ));
342
343        let mut eval_builder =
344            snix_eval::Evaluation::builder(io.clone() as Rc<dyn EvalIO>).enable_import();
345        eval_builder = add_derivation_builtins(eval_builder, Rc::clone(&io));
346        eval_builder = add_fetcher_builtins(eval_builder, Rc::clone(&io));
347        eval_builder = add_import_builtins(eval_builder, io);
348        let eval = eval_builder.build();
349
350        // run the evaluation itself.
351        eval.evaluate(str, None)
352    }
353
354    /// Helper function that takes a &Path, and invokes a snix evaluator coercing that path to a string
355    /// (via "${/this/path}"). The path can be both absolute or not.
356    /// It returns Option<String>, depending on whether the evaluation succeeded or not.
357    fn import_path_and_compare<P: AsRef<Path>>(p: P) -> Option<String> {
358        // Try to import the path using "${/tmp/path/to/test}".
359        // The format string looks funny, the {} passed to Nix needs to be
360        // escaped.
361        let code = format!(r#""${{{}}}""#, p.as_ref().display());
362        let result = eval(&code);
363
364        if !result.errors.is_empty() {
365            return None;
366        }
367
368        let value = result.value.expect("must be some");
369        match value {
370            snix_eval::Value::String(s) => Some(s.to_str_lossy().into_owned()),
371            _ => panic!("unexpected value type: {value:?}"),
372        }
373    }
374
375    /// Import a directory with a zero-sized ".keep" regular file.
376    /// Ensure it matches the (pre-recorded) store path that Nix would produce.
377    #[test]
378    fn import_directory() {
379        let tmpdir = TempDir::new().unwrap();
380
381        // create a directory named "test"
382        let src_path = tmpdir.path().join("test");
383        std::fs::create_dir(&src_path).unwrap();
384
385        // write a regular file `.keep`.
386        std::fs::write(src_path.join(".keep"), vec![]).unwrap();
387
388        // importing the path with .../test at the end.
389        assert_eq!(
390            Some("/nix/store/gq3xcv4xrj4yr64dflyr38acbibv3rm9-test".to_string()),
391            import_path_and_compare(&src_path)
392        );
393
394        // importing the path with .../test/. at the end.
395        assert_eq!(
396            Some("/nix/store/gq3xcv4xrj4yr64dflyr38acbibv3rm9-test".to_string()),
397            import_path_and_compare(src_path.join("."))
398        );
399    }
400
401    /// Import a file into the store. Nix uses the "recursive"/NAR-based hashing
402    /// scheme for these.
403    #[test]
404    fn import_file() {
405        let tmpdir = TempDir::new().unwrap();
406
407        // write a regular file `empty`.
408        std::fs::write(tmpdir.path().join("empty"), vec![]).unwrap();
409
410        assert_eq!(
411            Some("/nix/store/lx5i78a4izwk2qj1nq8rdc07y8zrwy90-empty".to_string()),
412            import_path_and_compare(tmpdir.path().join("empty"))
413        );
414
415        // write a regular file `hello.txt`.
416        std::fs::write(tmpdir.path().join("hello.txt"), b"Hello World!").unwrap();
417
418        assert_eq!(
419            Some("/nix/store/925f1jb1ajrypjbyq7rylwryqwizvhp0-hello.txt".to_string()),
420            import_path_and_compare(tmpdir.path().join("hello.txt"))
421        );
422    }
423
424    /// Invoke toString on a nonexisting file, and access the .file attribute.
425    /// This should not cause an error, because it shouldn't trigger an import,
426    /// and leave the path as-is.
427    #[test]
428    fn nonexisting_path_without_import() {
429        let result = eval("toString ({ line = 42; col = 42; file = /deep/thought; }.file)");
430
431        assert!(result.errors.is_empty(), "expect evaluation to succeed");
432        let value = result.value.expect("must be some");
433
434        match value {
435            snix_eval::Value::String(s) => {
436                assert_eq!(*s, "/deep/thought");
437            }
438            _ => panic!("unexpected value type: {value:?}"),
439        }
440    }
441}
snix_glue/snix_store_io.rs

snix_glue/
snix_store_io.rs
