snix_glue/builtins/
import.rs

1//! Implements builtins used to import paths in the store.
2
3use crate::snix_store_io::SnixStoreIO;
4use snix_castore::Node;
5use snix_castore::import::ingest_entries;
6use snix_eval::{
7    ErrorKind, EvalIO, Value,
8    builtin_macros::builtins,
9    generators::{self, GenCo},
10};
11use std::path::Path;
12
13use std::rc::Rc;
14
15async fn filtered_ingest(
16    state: Rc<SnixStoreIO>,
17    co: GenCo,
18    path: &Path,
19    filter: Option<&Value>,
20) -> Result<Node, ErrorKind> {
21    let mut entries: Vec<walkdir::DirEntry> = vec![];
22    let mut it = walkdir::WalkDir::new(path)
23        .follow_links(false)
24        .follow_root_links(false)
25        .contents_first(false)
26        .into_iter();
27
28    // Skip root node.
29    entries.push(
30        it.next()
31            .ok_or_else(|| ErrorKind::IO {
32                path: Some(path.to_path_buf()),
33                error: std::io::Error::new(std::io::ErrorKind::NotFound, "No root node emitted")
34                    .into(),
35            })?
36            .map_err(|err| ErrorKind::IO {
37                path: Some(path.to_path_buf()),
38                error: std::io::Error::from(err).into(),
39            })?,
40    );
41
42    while let Some(entry) = it.next() {
43        // Entry could be a NotFound, if the root path specified does not exist.
44        let entry = entry.map_err(|err| ErrorKind::IO {
45            path: err.path().map(|p| p.to_path_buf()),
46            error: std::io::Error::from(err).into(),
47        })?;
48
49        // As per Nix documentation `:doc builtins.filterSource`.
50        let file_type = if entry.file_type().is_dir() {
51            "directory"
52        } else if entry.file_type().is_file() {
53            "regular"
54        } else if entry.file_type().is_symlink() {
55            "symlink"
56        } else {
57            "unknown"
58        };
59
60        let should_keep: bool = if let Some(filter) = filter {
61            generators::request_force(
62                &co,
63                generators::request_call_with(
64                    &co,
65                    filter.clone(),
66                    [
67                        Value::String(entry.path().as_os_str().as_encoded_bytes().into()),
68                        Value::String(file_type.into()),
69                    ],
70                )
71                .await,
72            )
73            .await
74            .as_bool()?
75        } else {
76            true
77        };
78
79        if !should_keep {
80            if file_type == "directory" {
81                it.skip_current_dir();
82            }
83            continue;
84        }
85
86        entries.push(entry);
87    }
88
89    let dir_entries = entries.into_iter().rev().map(Ok);
90
91    state.tokio_handle.block_on(async {
92        let entries = snix_castore::import::fs::dir_entries_to_ingestion_stream::<'_, _, _, &[u8]>(
93            &state.build_state.blob_service,
94            dir_entries,
95            path,
96            None, // TODO re-scan
97        );
98        ingest_entries(&state.build_state.directory_service, entries)
99            .await
100            .map_err(|e| ErrorKind::IO {
101                path: Some(path.to_path_buf()),
102                error: Rc::new(std::io::Error::other(e)),
103            })
104    })
105}
106
107#[builtins(state = "Rc<SnixStoreIO>")]
108mod import_builtins {
109    use super::*;
110
111    use crate::builtins::ImportError;
112    use crate::snix_store_io::SnixStoreIO;
113    use bstr::ByteSlice;
114    use nix_compat::nixhash::{CAHash, HashAlgo, NixHash};
115    use nix_compat::store_path::{
116        StorePath, StorePathRef, build_ca_path, build_text_path_from_content_digest,
117    };
118    use sha2::Digest;
119    use snix_castore::blobservice::BlobService;
120    use snix_eval::builtins::coerce_value_to_path;
121    use snix_eval::generators::Gen;
122    use snix_eval::{AddContext, FileType, NixContext, NixContextElement, NixString};
123    use snix_eval::{ErrorKind, Value, generators::GenCo};
124    use snix_store::path_info::PathInfo;
125    use std::rc::Rc;
126    use std::sync::Arc;
127    use tokio::io::AsyncWriteExt;
128
129    /// Helper function dealing with uploading something from a std::io::Read to
130    /// the passed [BlobService], returning the B3Digest and size.
131    /// This function is sync (and uses the tokio handle to block).
132    /// A sync closure getting a copy of all bytes read can be passed in,
133    /// allowing to do other hashing where needed.
134    fn copy_to_blobservice<F>(
135        tokio_handle: tokio::runtime::Handle,
136        blob_service: impl BlobService,
137        mut r: impl std::io::Read,
138        mut inspect_f: F,
139    ) -> std::io::Result<(snix_castore::B3Digest, u64)>
140    where
141        F: FnMut(&[u8]),
142    {
143        let mut blob_size = 0;
144
145        let mut blob_writer = tokio_handle.block_on(async { blob_service.open_write().await });
146
147        // read piece by piece and write to blob_writer.
148        // This is a bit manual due to EvalIO being sync, while the blob writer being async.
149        {
150            let mut buf = [0u8; 4096];
151
152            loop {
153                // read bytes into buffer, break out if EOF
154                let len = r.read(&mut buf)?;
155                if len == 0 {
156                    break;
157                }
158                blob_size += len as u64;
159
160                let data = &buf[0..len];
161
162                // write to blobwriter
163                tokio_handle.block_on(async { blob_writer.write_all(data).await })?;
164
165                // Call inspect_f
166                inspect_f(data);
167            }
168
169            let blob_digest = tokio_handle.block_on(async { blob_writer.close().await })?;
170
171            Ok((blob_digest, blob_size))
172        }
173    }
174
175    // This is a helper used by both builtins.path and builtins.filterSource.
176    async fn import_helper(
177        state: Rc<SnixStoreIO>,
178        co: GenCo,
179        path: std::path::PathBuf,
180        name: Option<&Value>,
181        filter: Option<&Value>,
182        recursive_ingestion: bool,
183        expected_sha256: Option<[u8; 32]>,
184    ) -> Result<Value, ErrorKind> {
185        // Determine the name, either chosen by the user or derived from the path.
186        let name: String = match name {
187            Some(name) => {
188                let nix_str = generators::request_force(&co, name.clone())
189                    .await
190                    .to_str()?;
191
192                nix_compat::store_path::validate_name(&nix_str)
193                    .map_err(|err| {
194                        ErrorKind::SnixError(Arc::new(
195                            nix_compat::store_path::ParseStorePathError::from(err),
196                        ))
197                    })?
198                    .to_owned()
199            }
200            None => nix_compat::store_path::validate_name_as_os_str(path.file_name().ok_or_else(
201                || {
202                    std::io::Error::new(
203                        std::io::ErrorKind::InvalidFilename,
204                        "path without basename encountered",
205                    )
206                },
207            )?)
208            .map_err(|err| ErrorKind::SnixError(Arc::new(err)))?
209            .to_owned(),
210        };
211        // As a first step, we ingest the contents, and get back a root node,
212        // and optionally the sha256 a flat file.
213        let (root_node, ca) = match std::fs::metadata(&path)?.file_type().into() {
214            // Check if the path points to a regular file.
215            // If it does, the filter function is never executed, and we copy to the blobservice directly.
216            // If recursive is false, we need to calculate the sha256 digest of the raw contents,
217            // as that affects the output path calculation.
218            FileType::Regular => {
219                let mut file = state.open(&path)?;
220                let mut h = (!recursive_ingestion).then(sha2::Sha256::new);
221
222                let (blob_digest, blob_size) = copy_to_blobservice(
223                    state.tokio_handle.clone(),
224                    &state.build_state.blob_service,
225                    &mut file,
226                    |data| {
227                        // update blob_sha256 if needed.
228                        if let Some(h) = h.as_mut() {
229                            h.update(data)
230                        }
231                    },
232                )?;
233
234                (
235                    Node::File {
236                        digest: blob_digest,
237                        size: blob_size,
238                        executable: false,
239                    },
240                    h.map(|h| {
241                        // If non-recursive ingestion was requested, we return that one.
242                        let actual_sha256 = h.finalize().into();
243
244                        // If an expected hash was provided upfront, compare and bail out.
245                        if let Some(expected_sha256) = expected_sha256
246                            && actual_sha256 != expected_sha256
247                        {
248                            return Err(ImportError::HashMismatch(
249                                path.clone(),
250                                NixHash::Sha256(expected_sha256),
251                                NixHash::Sha256(actual_sha256),
252                            ));
253                        }
254                        Ok(CAHash::Flat(NixHash::Sha256(actual_sha256)))
255                    })
256                    .transpose()?,
257                )
258            }
259
260            FileType::Directory if !recursive_ingestion => {
261                return Err(ImportError::FlatImportOfNonFile(path))?;
262            }
263
264            // do the filtered ingest
265            FileType::Directory => (
266                filtered_ingest(state.clone(), co, path.as_ref(), filter).await?,
267                None,
268            ),
269            FileType::Symlink => {
270                // FUTUREWORK: Nix follows a symlink if it's at the root,
271                // except if it's not resolve-able (NixOS/nix#7761).
272                return Err(snix_eval::ErrorKind::IO {
273                    path: Some(path),
274                    error: Rc::new(std::io::Error::new(
275                        std::io::ErrorKind::Unsupported,
276                        "builtins.path pointing to a symlink is ill-defined.",
277                    )),
278                });
279            }
280            FileType::Unknown => {
281                return Err(snix_eval::ErrorKind::IO {
282                    path: Some(path),
283                    error: Rc::new(std::io::Error::new(
284                        std::io::ErrorKind::Unsupported,
285                        "unsupported file type",
286                    )),
287                });
288            }
289        };
290
291        // Calculate the NAR sha256.
292        let (nar_size, nar_sha256) = state
293            .tokio_handle
294            .block_on(async {
295                state
296                    .build_state
297                    .nar_calculation_service
298                    .as_ref()
299                    .calculate_nar(&root_node)
300                    .await
301            })
302            .map_err(|e| snix_eval::ErrorKind::SnixError(Arc::from(e)))?;
303
304        // Calculate the CA hash for the recursive cases, this is only already
305        // `Some(_)` for flat ingestion.
306        let ca = match ca {
307            None => {
308                // If an upfront-expected NAR hash was specified, compare.
309                if let Some(expected_nar_sha256) = expected_sha256
310                    && expected_nar_sha256 != nar_sha256
311                {
312                    return Err(ImportError::HashMismatch(
313                        path,
314                        NixHash::Sha256(expected_nar_sha256),
315                        NixHash::Sha256(nar_sha256),
316                    )
317                    .into());
318                }
319                CAHash::Nar(NixHash::Sha256(nar_sha256))
320            }
321            Some(ca) => ca,
322        };
323
324        let store_path = build_ca_path(&name, recursive_ingestion, &ca.hash(), [], false)
325            .map_err(|e| snix_eval::ErrorKind::SnixError(Arc::from(e)))?
326            .to_owned();
327
328        let path_info = state
329            .tokio_handle
330            .block_on(async {
331                state
332                    .build_state
333                    .path_info_service
334                    .as_ref()
335                    .put(PathInfo {
336                        store_path,
337                        node: root_node,
338                        // There's no reference scanning on path contents ingested like this.
339                        references: vec![],
340                        nar_size,
341                        nar_sha256,
342                        signatures: vec![],
343                        deriver: None,
344                        ca: Some(ca),
345                    })
346                    .await
347            })
348            .map_err(|e| snix_eval::ErrorKind::IO {
349                path: Some(path),
350                error: Rc::new(std::io::Error::other(e)),
351            })?;
352
353        // We need to attach context to the final output path.
354        let outpath = path_info.store_path.to_absolute_path();
355
356        Ok(
357            NixString::new_context_from(NixContextElement::Plain(outpath.clone()).into(), outpath)
358                .into(),
359        )
360    }
361
362    #[builtin("path")]
363    async fn builtin_path(
364        state: Rc<SnixStoreIO>,
365        co: GenCo,
366        args: Value,
367    ) -> Result<Value, ErrorKind> {
368        let args = args.to_attrs()?;
369
370        let path = match coerce_value_to_path(
371            &co,
372            generators::request_force(&co, args.select_required("path")?.clone()).await,
373        )
374        .await?
375        {
376            Ok(path) => path,
377            Err(cek) => return Ok(cek.into()),
378        };
379
380        let filter = args.select("filter");
381
382        // Construct a sha256 hasher, which is needed for flat ingestion.
383        let recursive_ingestion = args
384            .select("recursive")
385            .map(|r| r.as_bool())
386            .transpose()?
387            .unwrap_or(true); // Yes, yes, Nix, by default, puts `recursive = true;`.
388
389        let expected_sha256 = args
390            .select("sha256")
391            .map(|h| {
392                h.to_str().and_then(|expected| {
393                    match NixHash::from_str(expected.to_str()?, Some(HashAlgo::Sha256)) {
394                        Ok(NixHash::Sha256(digest)) => Ok(digest),
395                        Ok(_) => unreachable!(),
396                        Err(e) => Err(ErrorKind::InvalidHash(e.to_string())),
397                    }
398                })
399            })
400            .transpose()?;
401
402        import_helper(
403            state,
404            co,
405            path,
406            args.select("name"),
407            filter,
408            recursive_ingestion,
409            expected_sha256,
410        )
411        .await
412    }
413
414    #[builtin("filterSource")]
415    async fn builtin_filter_source(
416        state: Rc<SnixStoreIO>,
417        co: GenCo,
418        #[lazy] filter: Value,
419        path: Value,
420    ) -> Result<Value, ErrorKind> {
421        let path =
422            match coerce_value_to_path(&co, generators::request_force(&co, path).await).await? {
423                Ok(path) => path,
424                Err(cek) => return Ok(cek.into()),
425            };
426
427        import_helper(state, co, path, None, Some(&filter), true, None).await
428    }
429
430    #[builtin("storePath")]
431    async fn builtin_store_path(
432        state: Rc<SnixStoreIO>,
433        co: GenCo,
434        path: Value,
435    ) -> Result<Value, ErrorKind> {
436        let p = match &path {
437            Value::String(s) => Path::new(s.as_bytes().to_os_str()?),
438            Value::Path(p) => p.as_path(),
439            _ => {
440                return Err(ErrorKind::TypeError {
441                    expected: "string or path",
442                    actual: path.type_of(),
443                });
444            }
445        };
446
447        // For this builtin, the path needs to start with an absolute store path.
448        let (store_path, _sub_path) = StorePathRef::from_absolute_path_full(p)
449            .map_err(|_e| ImportError::PathNotAbsoluteOrInvalid(p.to_path_buf()))?;
450
451        if state.path_exists(p)? {
452            Ok(Value::String(NixString::new_context_from(
453                [NixContextElement::Plain(store_path.to_absolute_path())].into(),
454                p.as_os_str().as_encoded_bytes(),
455            )))
456        } else {
457            Err(ErrorKind::IO {
458                path: Some(p.to_path_buf()),
459                error: Rc::new(std::io::ErrorKind::NotFound.into()),
460            })
461        }
462    }
463
464    #[builtin("toFile")]
465    async fn builtin_to_file(
466        state: Rc<SnixStoreIO>,
467        co: GenCo,
468        name: Value,
469        content: Value,
470    ) -> Result<Value, ErrorKind> {
471        if name.is_catchable() {
472            return Ok(name);
473        }
474
475        if content.is_catchable() {
476            return Ok(content);
477        }
478
479        let name = name
480            .to_str()
481            .context("evaluating the `name` parameter of builtins.toFile")?;
482        let content = content
483            .to_contextful_str()
484            .context("evaluating the `content` parameter of builtins.toFile")?;
485
486        if content.iter_ctx_derivation().count() > 0
487            || content.iter_ctx_single_outputs().count() > 0
488        {
489            return Err(ErrorKind::UnexpectedContext);
490        }
491
492        // upload contents to the blobservice and create a root node
493        let mut h = sha2::Sha256::new();
494        let (blob_digest, blob_size) = copy_to_blobservice(
495            state.tokio_handle.clone(),
496            &state.build_state.blob_service,
497            std::io::Cursor::new(&content),
498            |data| h.update(data),
499        )?;
500
501        let root_node = Node::File {
502            digest: blob_digest,
503            size: blob_size,
504            executable: false,
505        };
506
507        // calculate the nar hash
508        let (nar_size, nar_sha256) = state
509            .build_state
510            .nar_calculation_service
511            .calculate_nar(&root_node)
512            .await
513            .map_err(|e| ErrorKind::SnixError(Arc::from(e)))?;
514
515        let content_digest: [u8; 32] = h.finalize().into();
516        let references = content.iter_ctx_plain().map(|sp| {
517            StorePathRef::from_absolute_path(sp.as_bytes())
518                .expect("Snix bug: must parse as store path")
519        });
520
521        // persist via pathinfo service.
522        let store_path = state
523            .tokio_handle
524            .block_on(
525                state.build_state.path_info_service.put(PathInfo {
526                    store_path: build_text_path_from_content_digest(
527                        name.to_str()?,
528                        content_digest,
529                        references,
530                    )
531                    .map_err(|_e| {
532                        nix_compat::derivation::DerivationError::InvalidOutputs(
533                            nix_compat::derivation::outputs::OutputsError::InvalidOutputName(
534                                name.to_str_lossy().into_owned(),
535                            ),
536                        )
537                    })
538                    .map_err(crate::builtins::DerivationError::InvalidDerivation)?
539                    .to_owned(),
540                    node: root_node,
541                    // assemble references from plain context.
542                    references: content
543                        .iter_ctx_plain()
544                        .map(|elem| StorePath::from_absolute_path(elem.as_bytes()))
545                        .collect::<Result<_, _>>()
546                        .map_err(|e| ErrorKind::SnixError(Arc::from(e)))?,
547                    nar_size,
548                    nar_sha256,
549                    signatures: vec![],
550                    deriver: None,
551                    ca: Some(CAHash::Text(content_digest)),
552                }),
553            )
554            .map_err(|e| ErrorKind::SnixError(Arc::from(e)))
555            .map(|path_info| path_info.store_path)?;
556
557        let abs_path = store_path.to_absolute_path();
558        let context: NixContext = NixContextElement::Plain(abs_path.clone()).into();
559
560        Ok(Value::from(NixString::new_context_from(context, abs_path)))
561    }
562}
563
564pub use import_builtins::builtins as import_builtins;
snix_glue/builtins/import.rs

snix_glue/builtins/
import.rs
