1use crate::snix_store_io::SnixStoreIO;
4use snix_castore::Node;
5use snix_castore::import::ingest_entries;
6use snix_eval::{
7 ErrorKind, EvalIO, Value,
8 builtin_macros::builtins,
9 generators::{self, GenCo},
10};
11use std::path::Path;
12
13use std::rc::Rc;
14
15async fn filtered_ingest(
16 state: Rc<SnixStoreIO>,
17 co: GenCo,
18 path: &Path,
19 filter: Option<&Value>,
20) -> Result<Node, ErrorKind> {
21 let mut entries: Vec<walkdir::DirEntry> = vec![];
22 let mut it = walkdir::WalkDir::new(path)
23 .follow_links(false)
24 .follow_root_links(false)
25 .contents_first(false)
26 .into_iter();
27
28 entries.push(
30 it.next()
31 .ok_or_else(|| ErrorKind::IO {
32 path: Some(path.to_path_buf()),
33 error: std::io::Error::new(std::io::ErrorKind::NotFound, "No root node emitted")
34 .into(),
35 })?
36 .map_err(|err| ErrorKind::IO {
37 path: Some(path.to_path_buf()),
38 error: std::io::Error::from(err).into(),
39 })?,
40 );
41
42 while let Some(entry) = it.next() {
43 let entry = entry.map_err(|err| ErrorKind::IO {
45 path: err.path().map(|p| p.to_path_buf()),
46 error: std::io::Error::from(err).into(),
47 })?;
48
49 let file_type = if entry.file_type().is_dir() {
51 "directory"
52 } else if entry.file_type().is_file() {
53 "regular"
54 } else if entry.file_type().is_symlink() {
55 "symlink"
56 } else {
57 "unknown"
58 };
59
60 let should_keep: bool = if let Some(filter) = filter {
61 generators::request_force(
62 &co,
63 generators::request_call_with(
64 &co,
65 filter.clone(),
66 [
67 Value::String(entry.path().as_os_str().as_encoded_bytes().into()),
68 Value::String(file_type.into()),
69 ],
70 )
71 .await,
72 )
73 .await
74 .as_bool()?
75 } else {
76 true
77 };
78
79 if !should_keep {
80 if file_type == "directory" {
81 it.skip_current_dir();
82 }
83 continue;
84 }
85
86 entries.push(entry);
87 }
88
89 let dir_entries = entries.into_iter().rev().map(Ok);
90
91 state.tokio_handle.block_on(async {
92 let entries = snix_castore::import::fs::dir_entries_to_ingestion_stream::<'_, _, _, &[u8]>(
93 &state.build_state.blob_engine,
94 dir_entries,
95 path,
96 None, );
98 ingest_entries(&state.build_state.directory_service, entries)
99 .await
100 .map_err(|e| ErrorKind::IO {
101 path: Some(path.to_path_buf()),
102 error: Rc::new(std::io::Error::other(e)),
103 })
104 })
105}
106
107#[builtins(state = "Rc<SnixStoreIO>")]
108mod import_builtins {
109 use super::*;
110
111 use crate::builtins::ImportError;
112 use crate::snix_store_io::SnixStoreIO;
113 use bstr::ByteSlice;
114 use nix_compat::nixhash::{CAHash, HashAlgo, NixHash};
115 use nix_compat::store_path::{
116 StorePath, StorePathName, StorePathNameBuf, StorePathRef, build_ca_path,
117 build_text_path_from_content_digest,
118 };
119 use sha2::Digest;
120 use snix_castore::blob_engine::BlobEngine;
121 use snix_eval::builtins::coerce_value_to_path;
122 use snix_eval::generators::Gen;
123 use snix_eval::{AddContext, FileType, NixContext, NixContextElement, NixString};
124 use snix_eval::{ErrorKind, Value, generators::GenCo};
125 use snix_store::path_info::PathInfo;
126 use std::rc::Rc;
127 use std::sync::Arc;
128 use tokio::io::AsyncWriteExt;
129
130 fn copy_to_blob_engine<F>(
136 tokio_handle: tokio::runtime::Handle,
137 blob_engine: impl BlobEngine,
138 mut r: impl std::io::Read,
139 mut inspect_f: F,
140 ) -> std::io::Result<(snix_castore::B3Digest, u64)>
141 where
142 F: FnMut(&[u8]),
143 {
144 let mut blob_size = 0;
145
146 let mut blob_writer = tokio_handle.block_on(async { blob_engine.open_write().await });
147
148 {
151 let mut buf = [0u8; 4096];
152
153 loop {
154 let len = r.read(&mut buf)?;
156 if len == 0 {
157 break;
158 }
159 blob_size += len as u64;
160
161 let data = &buf[0..len];
162
163 tokio_handle.block_on(async { blob_writer.write_all(data).await })?;
165
166 inspect_f(data);
168 }
169
170 let blob_digest = tokio_handle.block_on(async { blob_writer.close().await })?;
171
172 Ok((blob_digest, blob_size))
173 }
174 }
175
176 async fn import_helper(
178 state: Rc<SnixStoreIO>,
179 co: GenCo,
180 path: std::path::PathBuf,
181 name: Option<&Value>,
182 filter: Option<&Value>,
183 recursive_ingestion: bool,
184 expected_sha256: Option<[u8; 32]>,
185 ) -> Result<Value, ErrorKind> {
186 let name: StorePathNameBuf = match name {
188 Some(name) => {
189 let nix_str = generators::request_force(&co, name.clone())
190 .await
191 .to_str()?;
192
193 StorePathName::from_slice(&nix_str)
194 .map_err(|err| {
195 ErrorKind::SnixError(Arc::new(
196 nix_compat::store_path::ParseStorePathError::from(err),
197 ))
198 })?
199 .to_owned()
200 }
201 None => {
202 let file_name = path.file_name().ok_or_else(|| {
203 std::io::Error::new(
204 std::io::ErrorKind::InvalidFilename,
205 "path without basename encountered",
206 )
207 })?;
208 StorePathName::from_os_str(file_name)
209 .map_err(|err| ErrorKind::SnixError(Arc::new(err)))?
210 .to_owned()
211 }
212 };
213 let (root_node, ca) = match std::fs::metadata(&path)?.file_type().into() {
216 FileType::Regular => {
221 let mut file = state.open(&path)?;
222 let mut h = (!recursive_ingestion).then(sha2::Sha256::new);
223
224 let (blob_digest, blob_size) = copy_to_blob_engine(
225 state.tokio_handle.clone(),
226 &state.build_state.blob_engine,
227 &mut file,
228 |data| {
229 if let Some(h) = h.as_mut() {
231 h.update(data)
232 }
233 },
234 )?;
235
236 (
237 Node::File {
238 digest: blob_digest,
239 size: blob_size,
240 executable: false,
241 },
242 h.map(|h| {
243 let actual_sha256 = h.finalize().into();
245
246 if let Some(expected_sha256) = expected_sha256
248 && actual_sha256 != expected_sha256
249 {
250 return Err(ImportError::HashMismatch(
251 path.clone(),
252 NixHash::Sha256(expected_sha256),
253 NixHash::Sha256(actual_sha256),
254 ));
255 }
256 Ok(CAHash::Flat(NixHash::Sha256(actual_sha256)))
257 })
258 .transpose()?,
259 )
260 }
261
262 FileType::Directory if !recursive_ingestion => {
263 return Err(ImportError::FlatImportOfNonFile(path))?;
264 }
265
266 FileType::Directory => (
268 filtered_ingest(state.clone(), co, path.as_ref(), filter).await?,
269 None,
270 ),
271 FileType::Symlink => {
272 return Err(snix_eval::ErrorKind::IO {
275 path: Some(path),
276 error: Rc::new(std::io::Error::new(
277 std::io::ErrorKind::Unsupported,
278 "builtins.path pointing to a symlink is ill-defined.",
279 )),
280 });
281 }
282 FileType::Unknown => {
283 return Err(snix_eval::ErrorKind::IO {
284 path: Some(path),
285 error: Rc::new(std::io::Error::new(
286 std::io::ErrorKind::Unsupported,
287 "unsupported file type",
288 )),
289 });
290 }
291 };
292
293 let (nar_size, nar_sha256) = state
295 .tokio_handle
296 .block_on(async {
297 state
298 .build_state
299 .nar_calculation_service
300 .as_ref()
301 .calculate_nar(&root_node)
302 .await
303 })
304 .map_err(|e| snix_eval::ErrorKind::SnixError(Arc::from(e)))?;
305
306 let ca = match ca {
309 None => {
310 if let Some(expected_nar_sha256) = expected_sha256
312 && expected_nar_sha256 != nar_sha256
313 {
314 return Err(ImportError::HashMismatch(
315 path,
316 NixHash::Sha256(expected_nar_sha256),
317 NixHash::Sha256(nar_sha256),
318 )
319 .into());
320 }
321 CAHash::Nar(NixHash::Sha256(nar_sha256))
322 }
323 Some(ca) => ca,
324 };
325
326 let store_path =
327 build_ca_path(&name, recursive_ingestion, &ca.hash(), [], false).to_owned();
328
329 let path_info = state
330 .tokio_handle
331 .block_on(async {
332 state
333 .build_state
334 .path_info_service
335 .as_ref()
336 .put(PathInfo {
337 store_path,
338 node: root_node,
339 references: vec![],
341 nar_size,
342 nar_sha256,
343 signatures: vec![],
344 deriver: None,
345 ca: Some(ca),
346 })
347 .await
348 })
349 .map_err(|e| snix_eval::ErrorKind::IO {
350 path: Some(path),
351 error: Rc::new(std::io::Error::other(e)),
352 })?;
353
354 let outpath = path_info.store_path.to_absolute_path();
356
357 Ok(
358 NixString::new_context_from(NixContextElement::Plain(outpath.clone()).into(), outpath)
359 .into(),
360 )
361 }
362
363 #[builtin("path")]
364 async fn builtin_path(
365 state: Rc<SnixStoreIO>,
366 co: GenCo,
367 args: Value,
368 ) -> Result<Value, ErrorKind> {
369 let args = args.to_attrs()?;
370
371 let path = match coerce_value_to_path(
372 &co,
373 generators::request_force(&co, args.select_required("path")?.clone()).await,
374 )
375 .await?
376 {
377 Ok(path) => path,
378 Err(cek) => return Ok(cek.into()),
379 };
380
381 let filter = args.select("filter");
382
383 let recursive_ingestion = args
385 .select("recursive")
386 .map(|r| r.as_bool())
387 .transpose()?
388 .unwrap_or(true); let expected_sha256 = args
391 .select("sha256")
392 .map(|h| {
393 h.to_str().and_then(|expected| {
394 match NixHash::from_str(expected.to_str()?, Some(HashAlgo::Sha256)) {
395 Ok(NixHash::Sha256(digest)) => Ok(digest),
396 Ok(_) => unreachable!(),
397 Err(e) => Err(ErrorKind::InvalidHash(e.to_string())),
398 }
399 })
400 })
401 .transpose()?;
402
403 import_helper(
404 state,
405 co,
406 path,
407 args.select("name"),
408 filter,
409 recursive_ingestion,
410 expected_sha256,
411 )
412 .await
413 }
414
415 #[builtin("filterSource")]
416 async fn builtin_filter_source(
417 state: Rc<SnixStoreIO>,
418 co: GenCo,
419 #[lazy] filter: Value,
420 path: Value,
421 ) -> Result<Value, ErrorKind> {
422 let path =
423 match coerce_value_to_path(&co, generators::request_force(&co, path).await).await? {
424 Ok(path) => path,
425 Err(cek) => return Ok(cek.into()),
426 };
427
428 import_helper(state, co, path, None, Some(&filter), true, None).await
429 }
430
431 #[builtin("storePath")]
432 async fn builtin_store_path(
433 state: Rc<SnixStoreIO>,
434 co: GenCo,
435 path: Value,
436 ) -> Result<Value, ErrorKind> {
437 let p = match &path {
438 Value::String(s) => Path::new(s.as_bytes().to_os_str()?),
439 Value::Path(p) => p.as_path(),
440 _ => {
441 return Err(ErrorKind::TypeError {
442 expected: "string or path",
443 actual: path.type_of(),
444 });
445 }
446 };
447
448 let (store_path, _sub_path) = StorePathRef::from_absolute_path_full(p)
450 .map_err(|_e| ImportError::PathNotAbsoluteOrInvalid(p.to_path_buf()))?;
451
452 if state.path_exists(p)? {
453 Ok(Value::String(NixString::new_context_from(
454 [NixContextElement::Plain(store_path.to_absolute_path())].into(),
455 p.as_os_str().as_encoded_bytes(),
456 )))
457 } else {
458 Err(ErrorKind::IO {
459 path: Some(p.to_path_buf()),
460 error: Rc::new(std::io::ErrorKind::NotFound.into()),
461 })
462 }
463 }
464
465 #[builtin("toFile")]
466 async fn builtin_to_file(
467 state: Rc<SnixStoreIO>,
468 co: GenCo,
469 name: Value,
470 content: Value,
471 ) -> Result<Value, ErrorKind> {
472 if name.is_catchable() {
473 return Ok(name);
474 }
475
476 if content.is_catchable() {
477 return Ok(content);
478 }
479
480 let name = name
481 .to_str()
482 .context("evaluating the `name` parameter of builtins.toFile")?;
483 let name = StorePathName::from_slice(&name)
484 .map_err(|e| {
485 nix_compat::derivation::DerivationError::InvalidDerivationName(
486 name.to_vec(),
487 e.into(),
488 )
489 })
490 .map_err(crate::builtins::DerivationError::InvalidDerivation)?;
491 let content = content
492 .to_contextful_str()
493 .context("evaluating the `content` parameter of builtins.toFile")?;
494
495 if content.iter_ctx_derivation().count() > 0
496 || content.iter_ctx_single_outputs().count() > 0
497 {
498 return Err(ErrorKind::UnexpectedContext);
499 }
500
501 let mut h = sha2::Sha256::new();
503 let (blob_digest, blob_size) = copy_to_blob_engine(
504 state.tokio_handle.clone(),
505 &state.build_state.blob_engine,
506 std::io::Cursor::new(&content),
507 |data| h.update(data),
508 )?;
509
510 let root_node = Node::File {
511 digest: blob_digest,
512 size: blob_size,
513 executable: false,
514 };
515
516 let (nar_size, nar_sha256) = state
518 .tokio_handle
519 .block_on(
520 state
521 .build_state
522 .nar_calculation_service
523 .calculate_nar(&root_node),
524 )
525 .map_err(|e| ErrorKind::SnixError(Arc::from(e)))?;
526
527 let content_digest: [u8; 32] = h.finalize().into();
528 let references = content.iter_ctx_plain().map(|sp| {
529 StorePathRef::from_absolute_path(sp.as_bytes())
530 .expect("Snix bug: must parse as store path")
531 });
532
533 let store_path = state
535 .tokio_handle
536 .block_on(
537 state.build_state.path_info_service.put(PathInfo {
538 store_path: build_text_path_from_content_digest(
539 name,
540 content_digest,
541 references,
542 )
543 .to_owned(),
544 node: root_node,
545 references: content
547 .iter_ctx_plain()
548 .map(|elem| StorePath::from_absolute_path(elem.as_bytes()))
549 .collect::<Result<_, _>>()
550 .map_err(|e| ErrorKind::SnixError(Arc::from(e)))?,
551 nar_size,
552 nar_sha256,
553 signatures: vec![],
554 deriver: None,
555 ca: Some(CAHash::Text(content_digest)),
556 }),
557 )
558 .map_err(|e| ErrorKind::SnixError(Arc::from(e)))
559 .map(|path_info| path_info.store_path)?;
560
561 let abs_path = store_path.to_absolute_path();
562 let context: NixContext = NixContextElement::Plain(abs_path.clone()).into();
563
564 Ok(Value::from(NixString::new_context_from(context, abs_path)))
565 }
566}
567
568pub use import_builtins::builtins as import_builtins;