snix_glue/builtins/
import.rs

1//! Implements builtins used to import paths in the store.
2
3use crate::snix_store_io::SnixStoreIO;
4use snix_castore::Node;
5use snix_castore::import::ingest_entries;
6use snix_eval::{
7    ErrorKind, EvalIO, Value,
8    builtin_macros::builtins,
9    generators::{self, GenCo},
10};
11use std::path::Path;
12
13use std::rc::Rc;
14
15async fn filtered_ingest(
16    state: Rc<SnixStoreIO>,
17    co: GenCo,
18    path: &Path,
19    filter: Option<&Value>,
20) -> Result<Node, ErrorKind> {
21    let mut entries: Vec<walkdir::DirEntry> = vec![];
22    let mut it = walkdir::WalkDir::new(path)
23        .follow_links(false)
24        .follow_root_links(false)
25        .contents_first(false)
26        .into_iter();
27
28    // Skip root node.
29    entries.push(
30        it.next()
31            .ok_or_else(|| ErrorKind::IO {
32                path: Some(path.to_path_buf()),
33                error: std::io::Error::new(std::io::ErrorKind::NotFound, "No root node emitted")
34                    .into(),
35            })?
36            .map_err(|err| ErrorKind::IO {
37                path: Some(path.to_path_buf()),
38                error: std::io::Error::from(err).into(),
39            })?,
40    );
41
42    while let Some(entry) = it.next() {
43        // Entry could be a NotFound, if the root path specified does not exist.
44        let entry = entry.map_err(|err| ErrorKind::IO {
45            path: err.path().map(|p| p.to_path_buf()),
46            error: std::io::Error::from(err).into(),
47        })?;
48
49        // As per Nix documentation `:doc builtins.filterSource`.
50        let file_type = if entry.file_type().is_dir() {
51            "directory"
52        } else if entry.file_type().is_file() {
53            "regular"
54        } else if entry.file_type().is_symlink() {
55            "symlink"
56        } else {
57            "unknown"
58        };
59
60        let should_keep: bool = if let Some(filter) = filter {
61            generators::request_force(
62                &co,
63                generators::request_call_with(
64                    &co,
65                    filter.clone(),
66                    [
67                        Value::String(entry.path().as_os_str().as_encoded_bytes().into()),
68                        Value::String(file_type.into()),
69                    ],
70                )
71                .await,
72            )
73            .await
74            .as_bool()?
75        } else {
76            true
77        };
78
79        if !should_keep {
80            if file_type == "directory" {
81                it.skip_current_dir();
82            }
83            continue;
84        }
85
86        entries.push(entry);
87    }
88
89    let dir_entries = entries.into_iter().rev().map(Ok);
90
91    state.tokio_handle.block_on(async {
92        let entries = snix_castore::import::fs::dir_entries_to_ingestion_stream::<'_, _, _, &[u8]>(
93            &state.blob_service,
94            dir_entries,
95            path,
96            None, // TODO re-scan
97        );
98        ingest_entries(&state.directory_service, entries)
99            .await
100            .map_err(|e| ErrorKind::IO {
101                path: Some(path.to_path_buf()),
102                error: Rc::new(std::io::Error::new(std::io::ErrorKind::Other, e)),
103            })
104    })
105}
106
107#[builtins(state = "Rc<SnixStoreIO>")]
108mod import_builtins {
109    use super::*;
110
111    use crate::builtins::ImportError;
112    use crate::snix_store_io::SnixStoreIO;
113    use bstr::ByteSlice;
114    use nix_compat::nixhash::{CAHash, NixHash};
115    use nix_compat::store_path::{StorePath, StorePathRef, build_ca_path};
116    use sha2::Digest;
117    use snix_castore::blobservice::BlobService;
118    use snix_eval::builtins::coerce_value_to_path;
119    use snix_eval::generators::Gen;
120    use snix_eval::{AddContext, FileType, NixContext, NixContextElement, NixString};
121    use snix_eval::{ErrorKind, Value, generators::GenCo};
122    use snix_store::path_info::PathInfo;
123    use std::rc::Rc;
124    use tokio::io::AsyncWriteExt;
125
126    /// Helper function dealing with uploading something from a std::io::Read to
127    /// the passed [BlobService], returning the B3Digest and size.
128    /// This function is sync (and uses the tokio handle to block).
129    /// A sync closure getting a copy of all bytes read can be passed in,
130    /// allowing to do other hashing where needed.
131    fn copy_to_blobservice<F>(
132        tokio_handle: tokio::runtime::Handle,
133        blob_service: impl BlobService,
134        mut r: impl std::io::Read,
135        mut inspect_f: F,
136    ) -> std::io::Result<(snix_castore::B3Digest, u64)>
137    where
138        F: FnMut(&[u8]),
139    {
140        let mut blob_size = 0;
141
142        let mut blob_writer = tokio_handle.block_on(async { blob_service.open_write().await });
143
144        // read piece by piece and write to blob_writer.
145        // This is a bit manual due to EvalIO being sync, while the blob writer being async.
146        {
147            let mut buf = [0u8; 4096];
148
149            loop {
150                // read bytes into buffer, break out if EOF
151                let len = r.read(&mut buf)?;
152                if len == 0 {
153                    break;
154                }
155                blob_size += len as u64;
156
157                let data = &buf[0..len];
158
159                // write to blobwriter
160                tokio_handle.block_on(async { blob_writer.write_all(data).await })?;
161
162                // Call inspect_f
163                inspect_f(data);
164            }
165
166            let blob_digest = tokio_handle.block_on(async { blob_writer.close().await })?;
167
168            Ok((blob_digest, blob_size))
169        }
170    }
171
172    // This is a helper used by both builtins.path and builtins.filterSource.
173    async fn import_helper(
174        state: Rc<SnixStoreIO>,
175        co: GenCo,
176        path: std::path::PathBuf,
177        name: Option<&Value>,
178        filter: Option<&Value>,
179        recursive_ingestion: bool,
180        expected_sha256: Option<[u8; 32]>,
181    ) -> Result<Value, ErrorKind> {
182        let name: String = match name {
183            Some(name) => generators::request_force(&co, name.clone())
184                .await
185                .to_str()?
186                .as_bstr()
187                .to_string(),
188            None => snix_store::import::path_to_name(&path)
189                .expect("Failed to derive the default name out of the path")
190                .to_string(),
191        };
192        // As a first step, we ingest the contents, and get back a root node,
193        // and optionally the sha256 a flat file.
194        let (root_node, ca) = match std::fs::metadata(&path)?.file_type().into() {
195            // Check if the path points to a regular file.
196            // If it does, the filter function is never executed, and we copy to the blobservice directly.
197            // If recursive is false, we need to calculate the sha256 digest of the raw contents,
198            // as that affects the output path calculation.
199            FileType::Regular => {
200                let mut file = state.open(&path)?;
201                let mut h = (!recursive_ingestion).then(sha2::Sha256::new);
202
203                let (blob_digest, blob_size) = copy_to_blobservice(
204                    state.tokio_handle.clone(),
205                    &state.blob_service,
206                    &mut file,
207                    |data| {
208                        // update blob_sha256 if needed.
209                        if let Some(h) = h.as_mut() {
210                            h.update(data)
211                        }
212                    },
213                )?;
214
215                (
216                    Node::File {
217                        digest: blob_digest,
218                        size: blob_size,
219                        executable: false,
220                    },
221                    h.map(|h| {
222                        // If non-recursive ingestion was requested, we return that one.
223                        let actual_sha256 = h.finalize().into();
224
225                        // If an expected hash was provided upfront, compare and bail out.
226                        if let Some(expected_sha256) = expected_sha256 {
227                            if actual_sha256 != expected_sha256 {
228                                return Err(ImportError::HashMismatch(
229                                    path.clone(),
230                                    NixHash::Sha256(expected_sha256),
231                                    NixHash::Sha256(actual_sha256),
232                                ));
233                            }
234                        }
235                        Ok(CAHash::Flat(NixHash::Sha256(actual_sha256)))
236                    })
237                    .transpose()?,
238                )
239            }
240
241            FileType::Directory if !recursive_ingestion => {
242                return Err(ImportError::FlatImportOfNonFile(path))?;
243            }
244
245            // do the filtered ingest
246            FileType::Directory => (
247                filtered_ingest(state.clone(), co, path.as_ref(), filter).await?,
248                None,
249            ),
250            FileType::Symlink => {
251                // FUTUREWORK: Nix follows a symlink if it's at the root,
252                // except if it's not resolve-able (NixOS/nix#7761).i
253                return Err(snix_eval::ErrorKind::IO {
254                    path: Some(path),
255                    error: Rc::new(std::io::Error::new(
256                        std::io::ErrorKind::Unsupported,
257                        "builtins.path pointing to a symlink is ill-defined.",
258                    )),
259                });
260            }
261            FileType::Unknown => {
262                return Err(snix_eval::ErrorKind::IO {
263                    path: Some(path),
264                    error: Rc::new(std::io::Error::new(
265                        std::io::ErrorKind::Unsupported,
266                        "unsupported file type",
267                    )),
268                });
269            }
270        };
271
272        // Calculate the NAR sha256.
273        let (nar_size, nar_sha256) = state
274            .tokio_handle
275            .block_on(async {
276                state
277                    .nar_calculation_service
278                    .as_ref()
279                    .calculate_nar(&root_node)
280                    .await
281            })
282            .map_err(|e| snix_eval::ErrorKind::SnixError(Rc::new(e)))?;
283
284        // Calculate the CA hash for the recursive cases, this is only already
285        // `Some(_)` for flat ingestion.
286        let ca = match ca {
287            None => {
288                // If an upfront-expected NAR hash was specified, compare.
289                if let Some(expected_nar_sha256) = expected_sha256 {
290                    if expected_nar_sha256 != nar_sha256 {
291                        return Err(ImportError::HashMismatch(
292                            path,
293                            NixHash::Sha256(expected_nar_sha256),
294                            NixHash::Sha256(nar_sha256),
295                        )
296                        .into());
297                    }
298                }
299                CAHash::Nar(NixHash::Sha256(nar_sha256))
300            }
301            Some(ca) => ca,
302        };
303
304        let store_path = build_ca_path(&name, &ca, Vec::<&str>::new(), false)
305            .map_err(|e| snix_eval::ErrorKind::SnixError(Rc::new(e)))?;
306
307        let path_info = state
308            .tokio_handle
309            .block_on(async {
310                state
311                    .path_info_service
312                    .as_ref()
313                    .put(PathInfo {
314                        store_path,
315                        node: root_node,
316                        // There's no reference scanning on path contents ingested like this.
317                        references: vec![],
318                        nar_size,
319                        nar_sha256,
320                        signatures: vec![],
321                        deriver: None,
322                        ca: Some(ca),
323                    })
324                    .await
325            })
326            .map_err(|e| snix_eval::ErrorKind::IO {
327                path: Some(path),
328                error: Rc::new(e.into()),
329            })?;
330
331        // We need to attach context to the final output path.
332        let outpath = path_info.store_path.to_absolute_path();
333
334        Ok(
335            NixString::new_context_from(NixContextElement::Plain(outpath.clone()).into(), outpath)
336                .into(),
337        )
338    }
339
340    #[builtin("path")]
341    async fn builtin_path(
342        state: Rc<SnixStoreIO>,
343        co: GenCo,
344        args: Value,
345    ) -> Result<Value, ErrorKind> {
346        let args = args.to_attrs()?;
347
348        let path = match coerce_value_to_path(
349            &co,
350            generators::request_force(&co, args.select_required("path")?.clone()).await,
351        )
352        .await?
353        {
354            Ok(path) => path,
355            Err(cek) => return Ok(cek.into()),
356        };
357
358        let filter = args.select("filter");
359
360        // Construct a sha256 hasher, which is needed for flat ingestion.
361        let recursive_ingestion = args
362            .select("recursive")
363            .map(|r| r.as_bool())
364            .transpose()?
365            .unwrap_or(true); // Yes, yes, Nix, by default, puts `recursive = true;`.
366
367        let expected_sha256 = args
368            .select("sha256")
369            .map(|h| {
370                h.to_str().and_then(|expected| {
371                    match nix_compat::nixhash::from_str(expected.to_str()?, Some("sha256")) {
372                        Ok(NixHash::Sha256(digest)) => Ok(digest),
373                        Ok(_) => unreachable!(),
374                        Err(e) => Err(ErrorKind::InvalidHash(e.to_string())),
375                    }
376                })
377            })
378            .transpose()?;
379
380        import_helper(
381            state,
382            co,
383            path,
384            args.select("name"),
385            filter,
386            recursive_ingestion,
387            expected_sha256,
388        )
389        .await
390    }
391
392    #[builtin("filterSource")]
393    async fn builtin_filter_source(
394        state: Rc<SnixStoreIO>,
395        co: GenCo,
396        #[lazy] filter: Value,
397        path: Value,
398    ) -> Result<Value, ErrorKind> {
399        let path =
400            match coerce_value_to_path(&co, generators::request_force(&co, path).await).await? {
401                Ok(path) => path,
402                Err(cek) => return Ok(cek.into()),
403            };
404
405        import_helper(state, co, path, None, Some(&filter), true, None).await
406    }
407
408    #[builtin("storePath")]
409    async fn builtin_store_path(
410        state: Rc<SnixStoreIO>,
411        co: GenCo,
412        path: Value,
413    ) -> Result<Value, ErrorKind> {
414        let p = match &path {
415            Value::String(s) => Path::new(s.as_bytes().to_os_str()?),
416            Value::Path(p) => p.as_path(),
417            _ => {
418                return Err(ErrorKind::TypeError {
419                    expected: "string or path",
420                    actual: path.type_of(),
421                });
422            }
423        };
424
425        // For this builtin, the path needs to start with an absolute store path.
426        let (store_path, _sub_path) = StorePathRef::from_absolute_path_full(p)
427            .map_err(|_e| ImportError::PathNotAbsoluteOrInvalid(p.to_path_buf()))?;
428
429        if state.path_exists(p)? {
430            Ok(Value::String(NixString::new_context_from(
431                [NixContextElement::Plain(store_path.to_absolute_path())].into(),
432                p.as_os_str().as_encoded_bytes(),
433            )))
434        } else {
435            Err(ErrorKind::IO {
436                path: Some(p.to_path_buf()),
437                error: Rc::new(std::io::ErrorKind::NotFound.into()),
438            })
439        }
440    }
441
442    #[builtin("toFile")]
443    async fn builtin_to_file(
444        state: Rc<SnixStoreIO>,
445        co: GenCo,
446        name: Value,
447        content: Value,
448    ) -> Result<Value, ErrorKind> {
449        if name.is_catchable() {
450            return Ok(name);
451        }
452
453        if content.is_catchable() {
454            return Ok(content);
455        }
456
457        let name = name
458            .to_str()
459            .context("evaluating the `name` parameter of builtins.toFile")?;
460        let content = content
461            .to_contextful_str()
462            .context("evaluating the `content` parameter of builtins.toFile")?;
463
464        if content.iter_ctx_derivation().count() > 0
465            || content.iter_ctx_single_outputs().count() > 0
466        {
467            return Err(ErrorKind::UnexpectedContext);
468        }
469
470        // upload contents to the blobservice and create a root node
471        let mut h = sha2::Sha256::new();
472        let (blob_digest, blob_size) = copy_to_blobservice(
473            state.tokio_handle.clone(),
474            &state.blob_service,
475            std::io::Cursor::new(&content),
476            |data| h.update(data),
477        )?;
478
479        let root_node = Node::File {
480            digest: blob_digest,
481            size: blob_size,
482            executable: false,
483        };
484
485        // calculate the nar hash
486        let (nar_size, nar_sha256) = state
487            .nar_calculation_service
488            .calculate_nar(&root_node)
489            .await
490            .map_err(|e| ErrorKind::SnixError(Rc::new(e)))?;
491
492        let ca_hash = CAHash::Text(h.finalize().into());
493
494        // persist via pathinfo service.
495        let store_path = state
496            .tokio_handle
497            .block_on(
498                state.path_info_service.put(PathInfo {
499                    store_path: build_ca_path(
500                        name.to_str()?,
501                        &ca_hash,
502                        content.iter_ctx_plain(),
503                        false,
504                    )
505                    .map_err(|_e| {
506                        nix_compat::derivation::DerivationError::InvalidOutputName(
507                            name.to_str_lossy().into_owned(),
508                        )
509                    })
510                    .map_err(crate::builtins::DerivationError::InvalidDerivation)?,
511                    node: root_node,
512                    // assemble references from plain context.
513                    references: content
514                        .iter_ctx_plain()
515                        .map(|elem| StorePath::from_absolute_path(elem.as_bytes()))
516                        .collect::<Result<_, _>>()
517                        .map_err(|e| ErrorKind::SnixError(Rc::new(e)))?,
518                    nar_size,
519                    nar_sha256,
520                    signatures: vec![],
521                    deriver: None,
522                    ca: Some(ca_hash),
523                }),
524            )
525            .map_err(|e| ErrorKind::SnixError(Rc::new(e)))
526            .map(|path_info| path_info.store_path)?;
527
528        let abs_path = store_path.to_absolute_path();
529        let context: NixContext = NixContextElement::Plain(abs_path.clone()).into();
530
531        Ok(Value::from(NixString::new_context_from(context, abs_path)))
532    }
533}
534
535pub use import_builtins::builtins as import_builtins;
snix_glue/builtins/import.rs

snix_glue/builtins/
import.rs
