snix_glue/builtins/
import.rs

1//! Implements builtins used to import paths in the store.
2
3use crate::snix_store_io::SnixStoreIO;
4use snix_castore::Node;
5use snix_castore::import::ingest_entries;
6use snix_eval::{
7    ErrorKind, EvalIO, Value,
8    builtin_macros::builtins,
9    generators::{self, GenCo},
10};
11use std::path::Path;
12
13use std::rc::Rc;
14
15async fn filtered_ingest(
16    state: Rc<SnixStoreIO>,
17    co: GenCo,
18    path: &Path,
19    filter: Option<&Value>,
20) -> Result<Node, ErrorKind> {
21    let mut entries: Vec<walkdir::DirEntry> = vec![];
22    let mut it = walkdir::WalkDir::new(path)
23        .follow_links(false)
24        .follow_root_links(false)
25        .contents_first(false)
26        .into_iter();
27
28    // Skip root node.
29    entries.push(
30        it.next()
31            .ok_or_else(|| ErrorKind::IO {
32                path: Some(path.to_path_buf()),
33                error: std::io::Error::new(std::io::ErrorKind::NotFound, "No root node emitted")
34                    .into(),
35            })?
36            .map_err(|err| ErrorKind::IO {
37                path: Some(path.to_path_buf()),
38                error: std::io::Error::from(err).into(),
39            })?,
40    );
41
42    while let Some(entry) = it.next() {
43        // Entry could be a NotFound, if the root path specified does not exist.
44        let entry = entry.map_err(|err| ErrorKind::IO {
45            path: err.path().map(|p| p.to_path_buf()),
46            error: std::io::Error::from(err).into(),
47        })?;
48
49        // As per Nix documentation `:doc builtins.filterSource`.
50        let file_type = if entry.file_type().is_dir() {
51            "directory"
52        } else if entry.file_type().is_file() {
53            "regular"
54        } else if entry.file_type().is_symlink() {
55            "symlink"
56        } else {
57            "unknown"
58        };
59
60        let should_keep: bool = if let Some(filter) = filter {
61            generators::request_force(
62                &co,
63                generators::request_call_with(
64                    &co,
65                    filter.clone(),
66                    [
67                        Value::String(entry.path().as_os_str().as_encoded_bytes().into()),
68                        Value::String(file_type.into()),
69                    ],
70                )
71                .await,
72            )
73            .await
74            .as_bool()?
75        } else {
76            true
77        };
78
79        if !should_keep {
80            if file_type == "directory" {
81                it.skip_current_dir();
82            }
83            continue;
84        }
85
86        entries.push(entry);
87    }
88
89    let dir_entries = entries.into_iter().rev().map(Ok);
90
91    state.tokio_handle.block_on(async {
92        let entries = snix_castore::import::fs::dir_entries_to_ingestion_stream::<'_, _, _, &[u8]>(
93            &state.blob_service,
94            dir_entries,
95            path,
96            None, // TODO re-scan
97        );
98        ingest_entries(&state.directory_service, entries)
99            .await
100            .map_err(|e| ErrorKind::IO {
101                path: Some(path.to_path_buf()),
102                error: Rc::new(std::io::Error::other(e)),
103            })
104    })
105}
106
107#[builtins(state = "Rc<SnixStoreIO>")]
108mod import_builtins {
109    use super::*;
110
111    use crate::builtins::ImportError;
112    use crate::snix_store_io::SnixStoreIO;
113    use bstr::ByteSlice;
114    use nix_compat::nixhash::{CAHash, HashAlgo, NixHash};
115    use nix_compat::store_path::{StorePath, StorePathRef, build_ca_path};
116    use sha2::Digest;
117    use snix_castore::blobservice::BlobService;
118    use snix_eval::builtins::coerce_value_to_path;
119    use snix_eval::generators::Gen;
120    use snix_eval::{AddContext, FileType, NixContext, NixContextElement, NixString};
121    use snix_eval::{ErrorKind, Value, generators::GenCo};
122    use snix_store::path_info::PathInfo;
123    use std::rc::Rc;
124    use std::sync::Arc;
125    use tokio::io::AsyncWriteExt;
126
127    /// Helper function dealing with uploading something from a std::io::Read to
128    /// the passed [BlobService], returning the B3Digest and size.
129    /// This function is sync (and uses the tokio handle to block).
130    /// A sync closure getting a copy of all bytes read can be passed in,
131    /// allowing to do other hashing where needed.
132    fn copy_to_blobservice<F>(
133        tokio_handle: tokio::runtime::Handle,
134        blob_service: impl BlobService,
135        mut r: impl std::io::Read,
136        mut inspect_f: F,
137    ) -> std::io::Result<(snix_castore::B3Digest, u64)>
138    where
139        F: FnMut(&[u8]),
140    {
141        let mut blob_size = 0;
142
143        let mut blob_writer = tokio_handle.block_on(async { blob_service.open_write().await });
144
145        // read piece by piece and write to blob_writer.
146        // This is a bit manual due to EvalIO being sync, while the blob writer being async.
147        {
148            let mut buf = [0u8; 4096];
149
150            loop {
151                // read bytes into buffer, break out if EOF
152                let len = r.read(&mut buf)?;
153                if len == 0 {
154                    break;
155                }
156                blob_size += len as u64;
157
158                let data = &buf[0..len];
159
160                // write to blobwriter
161                tokio_handle.block_on(async { blob_writer.write_all(data).await })?;
162
163                // Call inspect_f
164                inspect_f(data);
165            }
166
167            let blob_digest = tokio_handle.block_on(async { blob_writer.close().await })?;
168
169            Ok((blob_digest, blob_size))
170        }
171    }
172
173    // This is a helper used by both builtins.path and builtins.filterSource.
174    async fn import_helper(
175        state: Rc<SnixStoreIO>,
176        co: GenCo,
177        path: std::path::PathBuf,
178        name: Option<&Value>,
179        filter: Option<&Value>,
180        recursive_ingestion: bool,
181        expected_sha256: Option<[u8; 32]>,
182    ) -> Result<Value, ErrorKind> {
183        let name: String = match name {
184            Some(name) => generators::request_force(&co, name.clone())
185                .await
186                .to_str()?
187                .as_bstr()
188                .to_string(),
189            None => snix_store::import::path_to_name(&path)
190                .expect("Failed to derive the default name out of the path")
191                .to_string(),
192        };
193        // As a first step, we ingest the contents, and get back a root node,
194        // and optionally the sha256 a flat file.
195        let (root_node, ca) = match std::fs::metadata(&path)?.file_type().into() {
196            // Check if the path points to a regular file.
197            // If it does, the filter function is never executed, and we copy to the blobservice directly.
198            // If recursive is false, we need to calculate the sha256 digest of the raw contents,
199            // as that affects the output path calculation.
200            FileType::Regular => {
201                let mut file = state.open(&path)?;
202                let mut h = (!recursive_ingestion).then(sha2::Sha256::new);
203
204                let (blob_digest, blob_size) = copy_to_blobservice(
205                    state.tokio_handle.clone(),
206                    &state.blob_service,
207                    &mut file,
208                    |data| {
209                        // update blob_sha256 if needed.
210                        if let Some(h) = h.as_mut() {
211                            h.update(data)
212                        }
213                    },
214                )?;
215
216                (
217                    Node::File {
218                        digest: blob_digest,
219                        size: blob_size,
220                        executable: false,
221                    },
222                    h.map(|h| {
223                        // If non-recursive ingestion was requested, we return that one.
224                        let actual_sha256 = h.finalize().into();
225
226                        // If an expected hash was provided upfront, compare and bail out.
227                        if let Some(expected_sha256) = expected_sha256
228                            && actual_sha256 != expected_sha256
229                        {
230                            return Err(ImportError::HashMismatch(
231                                path.clone(),
232                                NixHash::Sha256(expected_sha256),
233                                NixHash::Sha256(actual_sha256),
234                            ));
235                        }
236                        Ok(CAHash::Flat(NixHash::Sha256(actual_sha256)))
237                    })
238                    .transpose()?,
239                )
240            }
241
242            FileType::Directory if !recursive_ingestion => {
243                return Err(ImportError::FlatImportOfNonFile(path))?;
244            }
245
246            // do the filtered ingest
247            FileType::Directory => (
248                filtered_ingest(state.clone(), co, path.as_ref(), filter).await?,
249                None,
250            ),
251            FileType::Symlink => {
252                // FUTUREWORK: Nix follows a symlink if it's at the root,
253                // except if it's not resolve-able (NixOS/nix#7761).i
254                return Err(snix_eval::ErrorKind::IO {
255                    path: Some(path),
256                    error: Rc::new(std::io::Error::new(
257                        std::io::ErrorKind::Unsupported,
258                        "builtins.path pointing to a symlink is ill-defined.",
259                    )),
260                });
261            }
262            FileType::Unknown => {
263                return Err(snix_eval::ErrorKind::IO {
264                    path: Some(path),
265                    error: Rc::new(std::io::Error::new(
266                        std::io::ErrorKind::Unsupported,
267                        "unsupported file type",
268                    )),
269                });
270            }
271        };
272
273        // Calculate the NAR sha256.
274        let (nar_size, nar_sha256) = state
275            .tokio_handle
276            .block_on(async {
277                state
278                    .nar_calculation_service
279                    .as_ref()
280                    .calculate_nar(&root_node)
281                    .await
282            })
283            .map_err(|e| snix_eval::ErrorKind::SnixError(Arc::from(e)))?;
284
285        // Calculate the CA hash for the recursive cases, this is only already
286        // `Some(_)` for flat ingestion.
287        let ca = match ca {
288            None => {
289                // If an upfront-expected NAR hash was specified, compare.
290                if let Some(expected_nar_sha256) = expected_sha256
291                    && expected_nar_sha256 != nar_sha256
292                {
293                    return Err(ImportError::HashMismatch(
294                        path,
295                        NixHash::Sha256(expected_nar_sha256),
296                        NixHash::Sha256(nar_sha256),
297                    )
298                    .into());
299                }
300                CAHash::Nar(NixHash::Sha256(nar_sha256))
301            }
302            Some(ca) => ca,
303        };
304
305        let store_path = build_ca_path(&name, &ca, Vec::<&str>::new(), false)
306            .map_err(|e| snix_eval::ErrorKind::SnixError(Arc::from(e)))?;
307
308        let path_info = state
309            .tokio_handle
310            .block_on(async {
311                state
312                    .path_info_service
313                    .as_ref()
314                    .put(PathInfo {
315                        store_path,
316                        node: root_node,
317                        // There's no reference scanning on path contents ingested like this.
318                        references: vec![],
319                        nar_size,
320                        nar_sha256,
321                        signatures: vec![],
322                        deriver: None,
323                        ca: Some(ca),
324                    })
325                    .await
326            })
327            .map_err(|e| snix_eval::ErrorKind::IO {
328                path: Some(path),
329                error: Rc::new(std::io::Error::other(e)),
330            })?;
331
332        // We need to attach context to the final output path.
333        let outpath = path_info.store_path.to_absolute_path();
334
335        Ok(
336            NixString::new_context_from(NixContextElement::Plain(outpath.clone()).into(), outpath)
337                .into(),
338        )
339    }
340
341    #[builtin("path")]
342    async fn builtin_path(
343        state: Rc<SnixStoreIO>,
344        co: GenCo,
345        args: Value,
346    ) -> Result<Value, ErrorKind> {
347        let args = args.to_attrs()?;
348
349        let path = match coerce_value_to_path(
350            &co,
351            generators::request_force(&co, args.select_required("path")?.clone()).await,
352        )
353        .await?
354        {
355            Ok(path) => path,
356            Err(cek) => return Ok(cek.into()),
357        };
358
359        let filter = args.select("filter");
360
361        // Construct a sha256 hasher, which is needed for flat ingestion.
362        let recursive_ingestion = args
363            .select("recursive")
364            .map(|r| r.as_bool())
365            .transpose()?
366            .unwrap_or(true); // Yes, yes, Nix, by default, puts `recursive = true;`.
367
368        let expected_sha256 = args
369            .select("sha256")
370            .map(|h| {
371                h.to_str().and_then(|expected| {
372                    match NixHash::from_str(expected.to_str()?, Some(HashAlgo::Sha256)) {
373                        Ok(NixHash::Sha256(digest)) => Ok(digest),
374                        Ok(_) => unreachable!(),
375                        Err(e) => Err(ErrorKind::InvalidHash(e.to_string())),
376                    }
377                })
378            })
379            .transpose()?;
380
381        import_helper(
382            state,
383            co,
384            path,
385            args.select("name"),
386            filter,
387            recursive_ingestion,
388            expected_sha256,
389        )
390        .await
391    }
392
393    #[builtin("filterSource")]
394    async fn builtin_filter_source(
395        state: Rc<SnixStoreIO>,
396        co: GenCo,
397        #[lazy] filter: Value,
398        path: Value,
399    ) -> Result<Value, ErrorKind> {
400        let path =
401            match coerce_value_to_path(&co, generators::request_force(&co, path).await).await? {
402                Ok(path) => path,
403                Err(cek) => return Ok(cek.into()),
404            };
405
406        import_helper(state, co, path, None, Some(&filter), true, None).await
407    }
408
409    #[builtin("storePath")]
410    async fn builtin_store_path(
411        state: Rc<SnixStoreIO>,
412        co: GenCo,
413        path: Value,
414    ) -> Result<Value, ErrorKind> {
415        let p = match &path {
416            Value::String(s) => Path::new(s.as_bytes().to_os_str()?),
417            Value::Path(p) => p.as_path(),
418            _ => {
419                return Err(ErrorKind::TypeError {
420                    expected: "string or path",
421                    actual: path.type_of(),
422                });
423            }
424        };
425
426        // For this builtin, the path needs to start with an absolute store path.
427        let (store_path, _sub_path) = StorePathRef::from_absolute_path_full(p)
428            .map_err(|_e| ImportError::PathNotAbsoluteOrInvalid(p.to_path_buf()))?;
429
430        if state.path_exists(p)? {
431            Ok(Value::String(NixString::new_context_from(
432                [NixContextElement::Plain(store_path.to_absolute_path())].into(),
433                p.as_os_str().as_encoded_bytes(),
434            )))
435        } else {
436            Err(ErrorKind::IO {
437                path: Some(p.to_path_buf()),
438                error: Rc::new(std::io::ErrorKind::NotFound.into()),
439            })
440        }
441    }
442
443    #[builtin("toFile")]
444    async fn builtin_to_file(
445        state: Rc<SnixStoreIO>,
446        co: GenCo,
447        name: Value,
448        content: Value,
449    ) -> Result<Value, ErrorKind> {
450        if name.is_catchable() {
451            return Ok(name);
452        }
453
454        if content.is_catchable() {
455            return Ok(content);
456        }
457
458        let name = name
459            .to_str()
460            .context("evaluating the `name` parameter of builtins.toFile")?;
461        let content = content
462            .to_contextful_str()
463            .context("evaluating the `content` parameter of builtins.toFile")?;
464
465        if content.iter_ctx_derivation().count() > 0
466            || content.iter_ctx_single_outputs().count() > 0
467        {
468            return Err(ErrorKind::UnexpectedContext);
469        }
470
471        // upload contents to the blobservice and create a root node
472        let mut h = sha2::Sha256::new();
473        let (blob_digest, blob_size) = copy_to_blobservice(
474            state.tokio_handle.clone(),
475            &state.blob_service,
476            std::io::Cursor::new(&content),
477            |data| h.update(data),
478        )?;
479
480        let root_node = Node::File {
481            digest: blob_digest,
482            size: blob_size,
483            executable: false,
484        };
485
486        // calculate the nar hash
487        let (nar_size, nar_sha256) = state
488            .nar_calculation_service
489            .calculate_nar(&root_node)
490            .await
491            .map_err(|e| ErrorKind::SnixError(Arc::from(e)))?;
492
493        let ca_hash = CAHash::Text(h.finalize().into());
494
495        // persist via pathinfo service.
496        let store_path = state
497            .tokio_handle
498            .block_on(
499                state.path_info_service.put(PathInfo {
500                    store_path: build_ca_path(
501                        name.to_str()?,
502                        &ca_hash,
503                        content.iter_ctx_plain(),
504                        false,
505                    )
506                    .map_err(|_e| {
507                        nix_compat::derivation::DerivationError::InvalidOutputName(
508                            name.to_str_lossy().into_owned(),
509                        )
510                    })
511                    .map_err(crate::builtins::DerivationError::InvalidDerivation)?,
512                    node: root_node,
513                    // assemble references from plain context.
514                    references: content
515                        .iter_ctx_plain()
516                        .map(|elem| StorePath::from_absolute_path(elem.as_bytes()))
517                        .collect::<Result<_, _>>()
518                        .map_err(|e| ErrorKind::SnixError(Arc::from(e)))?,
519                    nar_size,
520                    nar_sha256,
521                    signatures: vec![],
522                    deriver: None,
523                    ca: Some(ca_hash),
524                }),
525            )
526            .map_err(|e| ErrorKind::SnixError(Arc::from(e)))
527            .map(|path_info| path_info.store_path)?;
528
529        let abs_path = store_path.to_absolute_path();
530        let context: NixContext = NixContextElement::Plain(abs_path.clone()).into();
531
532        Ok(Value::from(NixString::new_context_from(context, abs_path)))
533    }
534}
535
536pub use import_builtins::builtins as import_builtins;
snix_glue/builtins/import.rs

snix_glue/builtins/
import.rs
