snix_glue/builtins/
import.rs

1//! Implements builtins used to import paths in the store.
2
3use crate::snix_store_io::SnixStoreIO;
4use snix_castore::Node;
5use snix_castore::import::ingest_entries;
6use snix_eval::{
7    ErrorKind, EvalIO, Value,
8    builtin_macros::builtins,
9    generators::{self, GenCo},
10};
11use std::path::Path;
12
13use std::rc::Rc;
14
15async fn filtered_ingest(
16    state: Rc<SnixStoreIO>,
17    co: GenCo,
18    path: &Path,
19    filter: Option<&Value>,
20) -> Result<Node, ErrorKind> {
21    let mut entries: Vec<walkdir::DirEntry> = vec![];
22    let mut it = walkdir::WalkDir::new(path)
23        .follow_links(false)
24        .follow_root_links(false)
25        .contents_first(false)
26        .into_iter();
27
28    // Skip root node.
29    entries.push(
30        it.next()
31            .ok_or_else(|| ErrorKind::IO {
32                path: Some(path.to_path_buf()),
33                error: std::io::Error::new(std::io::ErrorKind::NotFound, "No root node emitted")
34                    .into(),
35            })?
36            .map_err(|err| ErrorKind::IO {
37                path: Some(path.to_path_buf()),
38                error: std::io::Error::from(err).into(),
39            })?,
40    );
41
42    while let Some(entry) = it.next() {
43        // Entry could be a NotFound, if the root path specified does not exist.
44        let entry = entry.map_err(|err| ErrorKind::IO {
45            path: err.path().map(|p| p.to_path_buf()),
46            error: std::io::Error::from(err).into(),
47        })?;
48
49        // As per Nix documentation `:doc builtins.filterSource`.
50        let file_type = if entry.file_type().is_dir() {
51            "directory"
52        } else if entry.file_type().is_file() {
53            "regular"
54        } else if entry.file_type().is_symlink() {
55            "symlink"
56        } else {
57            "unknown"
58        };
59
60        let should_keep: bool = if let Some(filter) = filter {
61            generators::request_force(
62                &co,
63                generators::request_call_with(
64                    &co,
65                    filter.clone(),
66                    [
67                        Value::String(entry.path().as_os_str().as_encoded_bytes().into()),
68                        Value::String(file_type.into()),
69                    ],
70                )
71                .await,
72            )
73            .await
74            .as_bool()?
75        } else {
76            true
77        };
78
79        if !should_keep {
80            if file_type == "directory" {
81                it.skip_current_dir();
82            }
83            continue;
84        }
85
86        entries.push(entry);
87    }
88
89    let dir_entries = entries.into_iter().rev().map(Ok);
90
91    state.tokio_handle.block_on(async {
92        let entries = snix_castore::import::fs::dir_entries_to_ingestion_stream::<'_, _, _, &[u8]>(
93            &state.blob_service,
94            dir_entries,
95            path,
96            None, // TODO re-scan
97        );
98        ingest_entries(&state.directory_service, entries)
99            .await
100            .map_err(|e| ErrorKind::IO {
101                path: Some(path.to_path_buf()),
102                error: Rc::new(std::io::Error::other(e)),
103            })
104    })
105}
106
107#[builtins(state = "Rc<SnixStoreIO>")]
108mod import_builtins {
109    use super::*;
110
111    use crate::builtins::ImportError;
112    use crate::snix_store_io::SnixStoreIO;
113    use bstr::ByteSlice;
114    use nix_compat::nixhash::{CAHash, HashAlgo, NixHash};
115    use nix_compat::store_path::{StorePath, StorePathRef, build_ca_path};
116    use sha2::Digest;
117    use snix_castore::blobservice::BlobService;
118    use snix_eval::builtins::coerce_value_to_path;
119    use snix_eval::generators::Gen;
120    use snix_eval::{AddContext, FileType, NixContext, NixContextElement, NixString};
121    use snix_eval::{ErrorKind, Value, generators::GenCo};
122    use snix_store::path_info::PathInfo;
123    use std::rc::Rc;
124    use std::sync::Arc;
125    use tokio::io::AsyncWriteExt;
126
127    /// Helper function dealing with uploading something from a std::io::Read to
128    /// the passed [BlobService], returning the B3Digest and size.
129    /// This function is sync (and uses the tokio handle to block).
130    /// A sync closure getting a copy of all bytes read can be passed in,
131    /// allowing to do other hashing where needed.
132    fn copy_to_blobservice<F>(
133        tokio_handle: tokio::runtime::Handle,
134        blob_service: impl BlobService,
135        mut r: impl std::io::Read,
136        mut inspect_f: F,
137    ) -> std::io::Result<(snix_castore::B3Digest, u64)>
138    where
139        F: FnMut(&[u8]),
140    {
141        let mut blob_size = 0;
142
143        let mut blob_writer = tokio_handle.block_on(async { blob_service.open_write().await });
144
145        // read piece by piece and write to blob_writer.
146        // This is a bit manual due to EvalIO being sync, while the blob writer being async.
147        {
148            let mut buf = [0u8; 4096];
149
150            loop {
151                // read bytes into buffer, break out if EOF
152                let len = r.read(&mut buf)?;
153                if len == 0 {
154                    break;
155                }
156                blob_size += len as u64;
157
158                let data = &buf[0..len];
159
160                // write to blobwriter
161                tokio_handle.block_on(async { blob_writer.write_all(data).await })?;
162
163                // Call inspect_f
164                inspect_f(data);
165            }
166
167            let blob_digest = tokio_handle.block_on(async { blob_writer.close().await })?;
168
169            Ok((blob_digest, blob_size))
170        }
171    }
172
173    // This is a helper used by both builtins.path and builtins.filterSource.
174    async fn import_helper(
175        state: Rc<SnixStoreIO>,
176        co: GenCo,
177        path: std::path::PathBuf,
178        name: Option<&Value>,
179        filter: Option<&Value>,
180        recursive_ingestion: bool,
181        expected_sha256: Option<[u8; 32]>,
182    ) -> Result<Value, ErrorKind> {
183        // Determine the name, either chosen by the user or derived from the path.
184        let name: String = match name {
185            Some(name) => {
186                let nix_str = generators::request_force(&co, name.clone())
187                    .await
188                    .to_str()?;
189
190                nix_compat::store_path::validate_name(&nix_str)
191                    .map_err(|err| ErrorKind::SnixError(Arc::new(err)))?
192                    .to_owned()
193            }
194            None => nix_compat::store_path::validate_name_as_os_str(path.file_name().ok_or_else(
195                || {
196                    std::io::Error::new(
197                        std::io::ErrorKind::InvalidFilename,
198                        "path without basename encountered",
199                    )
200                },
201            )?)
202            .map_err(|err| ErrorKind::SnixError(Arc::new(err)))?
203            .to_owned(),
204        };
205        // As a first step, we ingest the contents, and get back a root node,
206        // and optionally the sha256 a flat file.
207        let (root_node, ca) = match std::fs::metadata(&path)?.file_type().into() {
208            // Check if the path points to a regular file.
209            // If it does, the filter function is never executed, and we copy to the blobservice directly.
210            // If recursive is false, we need to calculate the sha256 digest of the raw contents,
211            // as that affects the output path calculation.
212            FileType::Regular => {
213                let mut file = state.open(&path)?;
214                let mut h = (!recursive_ingestion).then(sha2::Sha256::new);
215
216                let (blob_digest, blob_size) = copy_to_blobservice(
217                    state.tokio_handle.clone(),
218                    &state.blob_service,
219                    &mut file,
220                    |data| {
221                        // update blob_sha256 if needed.
222                        if let Some(h) = h.as_mut() {
223                            h.update(data)
224                        }
225                    },
226                )?;
227
228                (
229                    Node::File {
230                        digest: blob_digest,
231                        size: blob_size,
232                        executable: false,
233                    },
234                    h.map(|h| {
235                        // If non-recursive ingestion was requested, we return that one.
236                        let actual_sha256 = h.finalize().into();
237
238                        // If an expected hash was provided upfront, compare and bail out.
239                        if let Some(expected_sha256) = expected_sha256
240                            && actual_sha256 != expected_sha256
241                        {
242                            return Err(ImportError::HashMismatch(
243                                path.clone(),
244                                NixHash::Sha256(expected_sha256),
245                                NixHash::Sha256(actual_sha256),
246                            ));
247                        }
248                        Ok(CAHash::Flat(NixHash::Sha256(actual_sha256)))
249                    })
250                    .transpose()?,
251                )
252            }
253
254            FileType::Directory if !recursive_ingestion => {
255                return Err(ImportError::FlatImportOfNonFile(path))?;
256            }
257
258            // do the filtered ingest
259            FileType::Directory => (
260                filtered_ingest(state.clone(), co, path.as_ref(), filter).await?,
261                None,
262            ),
263            FileType::Symlink => {
264                // FUTUREWORK: Nix follows a symlink if it's at the root,
265                // except if it's not resolve-able (NixOS/nix#7761).i
266                return Err(snix_eval::ErrorKind::IO {
267                    path: Some(path),
268                    error: Rc::new(std::io::Error::new(
269                        std::io::ErrorKind::Unsupported,
270                        "builtins.path pointing to a symlink is ill-defined.",
271                    )),
272                });
273            }
274            FileType::Unknown => {
275                return Err(snix_eval::ErrorKind::IO {
276                    path: Some(path),
277                    error: Rc::new(std::io::Error::new(
278                        std::io::ErrorKind::Unsupported,
279                        "unsupported file type",
280                    )),
281                });
282            }
283        };
284
285        // Calculate the NAR sha256.
286        let (nar_size, nar_sha256) = state
287            .tokio_handle
288            .block_on(async {
289                state
290                    .nar_calculation_service
291                    .as_ref()
292                    .calculate_nar(&root_node)
293                    .await
294            })
295            .map_err(|e| snix_eval::ErrorKind::SnixError(Arc::from(e)))?;
296
297        // Calculate the CA hash for the recursive cases, this is only already
298        // `Some(_)` for flat ingestion.
299        let ca = match ca {
300            None => {
301                // If an upfront-expected NAR hash was specified, compare.
302                if let Some(expected_nar_sha256) = expected_sha256
303                    && expected_nar_sha256 != nar_sha256
304                {
305                    return Err(ImportError::HashMismatch(
306                        path,
307                        NixHash::Sha256(expected_nar_sha256),
308                        NixHash::Sha256(nar_sha256),
309                    )
310                    .into());
311                }
312                CAHash::Nar(NixHash::Sha256(nar_sha256))
313            }
314            Some(ca) => ca,
315        };
316
317        let store_path = build_ca_path(&name, &ca, Vec::<&str>::new(), false)
318            .map_err(|e| snix_eval::ErrorKind::SnixError(Arc::from(e)))?;
319
320        let path_info = state
321            .tokio_handle
322            .block_on(async {
323                state
324                    .path_info_service
325                    .as_ref()
326                    .put(PathInfo {
327                        store_path,
328                        node: root_node,
329                        // There's no reference scanning on path contents ingested like this.
330                        references: vec![],
331                        nar_size,
332                        nar_sha256,
333                        signatures: vec![],
334                        deriver: None,
335                        ca: Some(ca),
336                    })
337                    .await
338            })
339            .map_err(|e| snix_eval::ErrorKind::IO {
340                path: Some(path),
341                error: Rc::new(std::io::Error::other(e)),
342            })?;
343
344        // We need to attach context to the final output path.
345        let outpath = path_info.store_path.to_absolute_path();
346
347        Ok(
348            NixString::new_context_from(NixContextElement::Plain(outpath.clone()).into(), outpath)
349                .into(),
350        )
351    }
352
353    #[builtin("path")]
354    async fn builtin_path(
355        state: Rc<SnixStoreIO>,
356        co: GenCo,
357        args: Value,
358    ) -> Result<Value, ErrorKind> {
359        let args = args.to_attrs()?;
360
361        let path = match coerce_value_to_path(
362            &co,
363            generators::request_force(&co, args.select_required("path")?.clone()).await,
364        )
365        .await?
366        {
367            Ok(path) => path,
368            Err(cek) => return Ok(cek.into()),
369        };
370
371        let filter = args.select("filter");
372
373        // Construct a sha256 hasher, which is needed for flat ingestion.
374        let recursive_ingestion = args
375            .select("recursive")
376            .map(|r| r.as_bool())
377            .transpose()?
378            .unwrap_or(true); // Yes, yes, Nix, by default, puts `recursive = true;`.
379
380        let expected_sha256 = args
381            .select("sha256")
382            .map(|h| {
383                h.to_str().and_then(|expected| {
384                    match NixHash::from_str(expected.to_str()?, Some(HashAlgo::Sha256)) {
385                        Ok(NixHash::Sha256(digest)) => Ok(digest),
386                        Ok(_) => unreachable!(),
387                        Err(e) => Err(ErrorKind::InvalidHash(e.to_string())),
388                    }
389                })
390            })
391            .transpose()?;
392
393        import_helper(
394            state,
395            co,
396            path,
397            args.select("name"),
398            filter,
399            recursive_ingestion,
400            expected_sha256,
401        )
402        .await
403    }
404
405    #[builtin("filterSource")]
406    async fn builtin_filter_source(
407        state: Rc<SnixStoreIO>,
408        co: GenCo,
409        #[lazy] filter: Value,
410        path: Value,
411    ) -> Result<Value, ErrorKind> {
412        let path =
413            match coerce_value_to_path(&co, generators::request_force(&co, path).await).await? {
414                Ok(path) => path,
415                Err(cek) => return Ok(cek.into()),
416            };
417
418        import_helper(state, co, path, None, Some(&filter), true, None).await
419    }
420
421    #[builtin("storePath")]
422    async fn builtin_store_path(
423        state: Rc<SnixStoreIO>,
424        co: GenCo,
425        path: Value,
426    ) -> Result<Value, ErrorKind> {
427        let p = match &path {
428            Value::String(s) => Path::new(s.as_bytes().to_os_str()?),
429            Value::Path(p) => p.as_path(),
430            _ => {
431                return Err(ErrorKind::TypeError {
432                    expected: "string or path",
433                    actual: path.type_of(),
434                });
435            }
436        };
437
438        // For this builtin, the path needs to start with an absolute store path.
439        let (store_path, _sub_path) = StorePathRef::from_absolute_path_full(p)
440            .map_err(|_e| ImportError::PathNotAbsoluteOrInvalid(p.to_path_buf()))?;
441
442        if state.path_exists(p)? {
443            Ok(Value::String(NixString::new_context_from(
444                [NixContextElement::Plain(store_path.to_absolute_path())].into(),
445                p.as_os_str().as_encoded_bytes(),
446            )))
447        } else {
448            Err(ErrorKind::IO {
449                path: Some(p.to_path_buf()),
450                error: Rc::new(std::io::ErrorKind::NotFound.into()),
451            })
452        }
453    }
454
455    #[builtin("toFile")]
456    async fn builtin_to_file(
457        state: Rc<SnixStoreIO>,
458        co: GenCo,
459        name: Value,
460        content: Value,
461    ) -> Result<Value, ErrorKind> {
462        if name.is_catchable() {
463            return Ok(name);
464        }
465
466        if content.is_catchable() {
467            return Ok(content);
468        }
469
470        let name = name
471            .to_str()
472            .context("evaluating the `name` parameter of builtins.toFile")?;
473        let content = content
474            .to_contextful_str()
475            .context("evaluating the `content` parameter of builtins.toFile")?;
476
477        if content.iter_ctx_derivation().count() > 0
478            || content.iter_ctx_single_outputs().count() > 0
479        {
480            return Err(ErrorKind::UnexpectedContext);
481        }
482
483        // upload contents to the blobservice and create a root node
484        let mut h = sha2::Sha256::new();
485        let (blob_digest, blob_size) = copy_to_blobservice(
486            state.tokio_handle.clone(),
487            &state.blob_service,
488            std::io::Cursor::new(&content),
489            |data| h.update(data),
490        )?;
491
492        let root_node = Node::File {
493            digest: blob_digest,
494            size: blob_size,
495            executable: false,
496        };
497
498        // calculate the nar hash
499        let (nar_size, nar_sha256) = state
500            .nar_calculation_service
501            .calculate_nar(&root_node)
502            .await
503            .map_err(|e| ErrorKind::SnixError(Arc::from(e)))?;
504
505        let ca_hash = CAHash::Text(h.finalize().into());
506
507        // persist via pathinfo service.
508        let store_path = state
509            .tokio_handle
510            .block_on(
511                state.path_info_service.put(PathInfo {
512                    store_path: build_ca_path(
513                        name.to_str()?,
514                        &ca_hash,
515                        content.iter_ctx_plain(),
516                        false,
517                    )
518                    .map_err(|_e| {
519                        nix_compat::derivation::DerivationError::InvalidOutputName(
520                            name.to_str_lossy().into_owned(),
521                        )
522                    })
523                    .map_err(crate::builtins::DerivationError::InvalidDerivation)?,
524                    node: root_node,
525                    // assemble references from plain context.
526                    references: content
527                        .iter_ctx_plain()
528                        .map(|elem| StorePath::from_absolute_path(elem.as_bytes()))
529                        .collect::<Result<_, _>>()
530                        .map_err(|e| ErrorKind::SnixError(Arc::from(e)))?,
531                    nar_size,
532                    nar_sha256,
533                    signatures: vec![],
534                    deriver: None,
535                    ca: Some(ca_hash),
536                }),
537            )
538            .map_err(|e| ErrorKind::SnixError(Arc::from(e)))
539            .map(|path_info| path_info.store_path)?;
540
541        let abs_path = store_path.to_absolute_path();
542        let context: NixContext = NixContextElement::Plain(abs_path.clone()).into();
543
544        Ok(Value::from(NixString::new_context_from(context, abs_path)))
545    }
546}
547
548pub use import_builtins::builtins as import_builtins;
snix_glue/builtins/import.rs

snix_glue/builtins/
import.rs
