snix_glue/
snix_build.rs

1//! This module contains glue code translating from
2//! [nix_compat::derivation::Derivation] to [snix_build::buildservice::BuildRequest].
3
4use std::collections::{BTreeMap, HashSet, VecDeque};
5use std::future::Future;
6use std::path::PathBuf;
7
8use async_stream::try_stream;
9use bstr::BString;
10use bytes::Bytes;
11use futures::Stream;
12use nix_compat::derivation::Output;
13use nix_compat::store_path::hash_placeholder;
14use nix_compat::{derivation::Derivation, nixbase32, store_path::StorePath};
15use sha2::{Digest, Sha256};
16use snix_build::buildservice::{AdditionalFile, BuildConstraints, BuildRequest, EnvVar};
17use snix_castore::Node;
18use snix_store::path_info::PathInfo;
19use tracing::warn;
20
21use crate::known_paths::KnownPaths;
22
23/// These are the environment variables that Nix sets in its sandbox for every
24/// build.
25const NIX_ENVIRONMENT_VARS: [(&str, &str); 12] = [
26    ("HOME", "/homeless-shelter"),
27    ("NIX_BUILD_CORES", "0"), // TODO: make this configurable?
28    ("NIX_BUILD_TOP", "/build"),
29    ("NIX_LOG_FD", "2"),
30    ("NIX_STORE", "/nix/store"),
31    ("PATH", "/path-not-set"),
32    ("PWD", "/build"),
33    ("TEMP", "/build"),
34    ("TEMPDIR", "/build"),
35    ("TERM", "xterm-256color"),
36    ("TMP", "/build"),
37    ("TMPDIR", "/build"),
38];
39
40/// Get a stream of a transitive input closure for a derivation.
41/// It's used for input propagation into the build and nixbase32 needle propagation
42/// for build output refscanning.
43pub(crate) fn get_all_inputs<'a, F, Fut>(
44    derivation: &'a Derivation,
45    known_paths: &'a KnownPaths,
46    get_path_info: F,
47) -> impl Stream<Item = Result<(StorePath<String>, Node), std::io::Error>> + use<F, Fut>
48where
49    F: Fn(StorePath<String>) -> Fut,
50    Fut: Future<Output = std::io::Result<Option<PathInfo>>>,
51{
52    let mut visited: HashSet<StorePath<String>> = HashSet::new();
53    let mut queue: VecDeque<StorePath<String>> = derivation
54        .input_sources
55        .iter()
56        .cloned()
57        .chain(
58            derivation
59                .input_derivations
60                .iter()
61                .flat_map(|(drv_path, outs)| {
62                    let drv = known_paths.get_drv_by_drvpath(drv_path).expect("drv Bug!!");
63                    outs.iter().map(move |output| {
64                        drv.outputs
65                            .get(output)
66                            .expect("No output bug!")
67                            .path
68                            .as_ref()
69                            .expect("output has no store path")
70                            .clone()
71                    })
72                }),
73        )
74        .collect();
75    try_stream! {
76        while let Some(store_path) = queue.pop_front() {
77                let info = get_path_info(store_path).await?.ok_or(std::io::Error::other("path_info not present"))?;
78                    for reference in info.references {
79                        if visited.insert(reference.clone()) {
80                            queue.push_back(reference);
81                        }
82                    }
83
84                    yield (info.store_path, info.node);
85
86
87        }
88    }
89}
90
91/// Takes a [Derivation] and turns it into a [snix_build::buildservice::BuildRequest].
92/// It assumes the Derivation has been validated, and all referenced output paths are present in `inputs`.
93pub(crate) fn derivation_to_build_request(
94    derivation: &Derivation,
95    inputs: &BTreeMap<StorePath<String>, Node>,
96) -> std::io::Result<BuildRequest> {
97    debug_assert!(derivation.validate(true).is_ok(), "drv must validate");
98
99    // produce command_args, which is builder and arguments in a Vec.
100    let mut command_args: Vec<String> = Vec::with_capacity(derivation.arguments.len() + 1);
101    command_args.push(derivation.builder.clone());
102    command_args.extend_from_slice(
103        derivation
104            .arguments
105            .iter()
106            .map(|arg| replace_placeholders(arg, &derivation.outputs))
107            .collect::<Vec<String>>()
108            .as_slice(),
109    );
110
111    // Produce environment_vars and additional files.
112    // We use a BTreeMap while producing, and only realize the resulting Vec
113    // while populating BuildRequest, so we don't need to worry about ordering.
114    let mut environment_vars: BTreeMap<String, Bytes> = BTreeMap::new();
115    let mut additional_files: BTreeMap<String, Bytes> = BTreeMap::new();
116
117    // Start with some the ones that nix magically sets:
118    environment_vars.extend(
119        NIX_ENVIRONMENT_VARS
120            .iter()
121            .map(|(k, v)| (k.to_string(), Bytes::from_static(v.as_bytes()))),
122    );
123
124    // extend / overwrite with the keys set in the derivation environment itself.
125    // TODO: check if this order is correct, and environment vars set in the
126    // *Derivation actually* have priority.
127    environment_vars.extend(derivation.environment.iter().map(|(k, v)| {
128        (
129            k.clone(),
130            Bytes::from(replace_placeholders_b(v, &derivation.outputs).to_vec()),
131        )
132    }));
133
134    handle_pass_as_file(&mut environment_vars, &mut additional_files)?;
135
136    // TODO: handle __json (structured attrs, provide JSON file and source-able bash script)
137
138    // Produce constraints.
139    let mut constraints = HashSet::from([
140        BuildConstraints::System(derivation.system.clone()),
141        BuildConstraints::ProvideBinSh,
142    ]);
143
144    if derivation.outputs.len() == 1
145        && derivation
146            .outputs
147            .get("out")
148            .expect("Snix bug: Derivation has no out output")
149            .is_fixed()
150    {
151        constraints.insert(BuildConstraints::NetworkAccess);
152    }
153
154    Ok(BuildRequest {
155        // Importantly, this must match the order of get_refscan_needles, since users may use that
156        // function to map back from the found needles to a store path
157        refscan_needles: derivation
158            .outputs
159            .values()
160            .filter_map(|output| output.path.as_ref())
161            .map(|path| nixbase32::encode(path.digest()))
162            .chain(inputs.keys().map(|path| nixbase32::encode(path.digest())))
163            .collect(),
164        command_args,
165
166        outputs: derivation
167            .outputs
168            .values()
169            .map(|e| PathBuf::from(e.path_str()[1..].to_owned()))
170            .collect(),
171
172        // Turn this into a sorted-by-key Vec<EnvVar>.
173        environment_vars: environment_vars
174            .into_iter()
175            .map(|(key, value)| EnvVar { key, value })
176            .collect(),
177        inputs: inputs
178            .iter()
179            .map(|(path, node)| {
180                (
181                    path.to_string()
182                        .as_str()
183                        .try_into()
184                        .expect("Snix bug: unable to convert store path basename to PathComponent"),
185                    node.clone(),
186                )
187            })
188            .collect(),
189        inputs_dir: nix_compat::store_path::STORE_DIR[1..].into(),
190        constraints,
191        working_dir: "build".into(),
192        scratch_paths: vec![
193            "build".into(),
194            // This is in here because Nix allows you to do
195            // `pkgs.runCommand "foo" {} "mkdir -p $out;touch /nix/store/aaaa"`
196            // (throwing away the /nix/store/aaaa post-build),
197            // not because it's a sane thing to do.
198            // FUTUREWORK: check if nothing exploits this.
199            "nix/store".into(),
200        ],
201        additional_files: additional_files
202            .into_iter()
203            .map(|(path, contents)| AdditionalFile {
204                path: PathBuf::from(path),
205                contents,
206            })
207            .collect(),
208    })
209}
210
211/// handle passAsFile, if set.
212/// For each env $x in that list, the original env is removed, and a $xPath
213/// environment var added instead, referring to a path inside the build with
214/// the contents from the original env var.
215fn handle_pass_as_file(
216    environment_vars: &mut BTreeMap<String, Bytes>,
217    additional_files: &mut BTreeMap<String, Bytes>,
218) -> std::io::Result<()> {
219    let pass_as_file = environment_vars.get("passAsFile").map(|v| {
220        // Convert pass_as_file to string.
221        // When it gets here, it contains a space-separated list of env var
222        // keys, which must be strings.
223        String::from_utf8(v.to_vec())
224    });
225
226    if let Some(pass_as_file) = pass_as_file {
227        let pass_as_file = pass_as_file.map_err(|_| {
228            std::io::Error::new(
229                std::io::ErrorKind::InvalidInput,
230                "passAsFile elements are no valid utf8 strings",
231            )
232        })?;
233
234        for x in pass_as_file.split(' ') {
235            match environment_vars.remove_entry(x) {
236                Some((k, contents)) => {
237                    let (new_k, path) = calculate_pass_as_file_env(&k);
238
239                    additional_files.insert(path[1..].to_string(), contents);
240                    environment_vars.insert(new_k, Bytes::from(path));
241                }
242                None => {
243                    return Err(std::io::Error::new(
244                        std::io::ErrorKind::InvalidData,
245                        "passAsFile refers to non-existent env key",
246                    ));
247                }
248            }
249        }
250    }
251
252    Ok(())
253}
254
255/// For a given key k in a derivation environment that's supposed to be passed as file,
256/// calculate the ${k}Path key and filepath value that it's being replaced with
257/// while preparing the build.
258/// The filepath is `/build/.attrs-${nixbase32(sha256(key))`.
259fn calculate_pass_as_file_env(k: &str) -> (String, String) {
260    (
261        format!("{}Path", k),
262        format!(
263            "/build/.attr-{}",
264            nixbase32::encode(&Sha256::new_with_prefix(k).finalize())
265        ),
266    )
267}
268
269/// Replace all references to `placeholder outputName` inside the derivation
270fn replace_placeholders(s: &str, outputs: &BTreeMap<String, Output>) -> String {
271    let mut s = s.to_owned();
272    for (out_name, output) in outputs {
273        let placeholder = hash_placeholder(out_name.as_str());
274        if let Some(path) = output.path.as_ref() {
275            s = s.replace(&placeholder, &path.to_absolute_path());
276        } else {
277            warn!(
278                output.name = out_name,
279                "output should have a path during placeholder replacement"
280            );
281        }
282    }
283    s
284}
285
286/// Replace all references to `placeholder outputName` inside the derivation
287fn replace_placeholders_b(s: &BString, outputs: &BTreeMap<String, Output>) -> BString {
288    use bstr::ByteSlice;
289    let mut s = s.clone();
290    for (out_name, output) in outputs {
291        let placeholder = hash_placeholder(out_name.as_str());
292        if let Some(path) = output.path.as_ref() {
293            s = s
294                .replace(placeholder.as_bytes(), path.to_absolute_path().as_bytes())
295                .into();
296        } else {
297            warn!(
298                output.name = out_name,
299                "output should have a path during placeholder replacement"
300            );
301        }
302    }
303    s
304}
305
306#[cfg(test)]
307mod test {
308    use bytes::Bytes;
309    use nix_compat::store_path::hash_placeholder;
310    use nix_compat::{derivation::Derivation, store_path::StorePath};
311    use snix_castore::fixtures::DUMMY_DIGEST;
312    use snix_castore::{Node, PathComponent};
313    use std::collections::{BTreeMap, HashSet};
314    use std::sync::LazyLock;
315
316    use snix_build::buildservice::{AdditionalFile, BuildConstraints, BuildRequest, EnvVar};
317
318    use crate::known_paths::KnownPaths;
319    use crate::snix_build::NIX_ENVIRONMENT_VARS;
320
321    use super::derivation_to_build_request;
322
323    static INPUT_NODE_FOO_NAME: LazyLock<Bytes> =
324        LazyLock::new(|| "mp57d33657rf34lzvlbpfa1gjfv5gmpg-bar".into());
325
326    static INPUT_NODE_FOO: LazyLock<Node> = LazyLock::new(|| Node::Directory {
327        digest: DUMMY_DIGEST.clone(),
328        size: 42,
329    });
330
331    #[test]
332    fn test_derivation_to_build_request() {
333        let aterm_bytes = include_bytes!("tests/ch49594n9avinrf8ip0aslidkc4lxkqv-foo.drv");
334
335        let dep_drv_bytes = include_bytes!("tests/ss2p4wmxijn652haqyd7dckxwl4c7hxx-bar.drv");
336
337        let derivation1 = Derivation::from_aterm_bytes(aterm_bytes).expect("drv1 must parse");
338        let drv_path1 =
339            StorePath::<String>::from_bytes("ch49594n9avinrf8ip0aslidkc4lxkqv-foo.drv".as_bytes())
340                .expect("drv path1 must parse");
341        let derivation2 = Derivation::from_aterm_bytes(dep_drv_bytes).expect("drv2 must parse");
342        let drv_path2 =
343            StorePath::<String>::from_bytes("ss2p4wmxijn652haqyd7dckxwl4c7hxx-bar.drv".as_bytes())
344                .expect("drv path2 must parse");
345
346        let mut known_paths = KnownPaths::default();
347
348        known_paths.add_derivation(drv_path2, derivation2);
349        known_paths.add_derivation(drv_path1, derivation1.clone());
350
351        let build_request = derivation_to_build_request(
352            &derivation1,
353            &BTreeMap::from([(
354                StorePath::<String>::from_bytes(&INPUT_NODE_FOO_NAME.clone()).unwrap(),
355                INPUT_NODE_FOO.clone(),
356            )]),
357        )
358        .expect("must succeed");
359
360        let mut expected_environment_vars = vec![
361            EnvVar {
362                key: "bar".into(),
363                value: "/nix/store/mp57d33657rf34lzvlbpfa1gjfv5gmpg-bar".into(),
364            },
365            EnvVar {
366                key: "builder".into(),
367                value: ":".into(),
368            },
369            EnvVar {
370                key: "name".into(),
371                value: "foo".into(),
372            },
373            EnvVar {
374                key: "out".into(),
375                value: "/nix/store/fhaj6gmwns62s6ypkcldbaj2ybvkhx3p-foo".into(),
376            },
377            EnvVar {
378                key: "system".into(),
379                value: ":".into(),
380            },
381        ];
382
383        expected_environment_vars.extend(NIX_ENVIRONMENT_VARS.iter().map(|(k, v)| EnvVar {
384            key: k.to_string(),
385            value: Bytes::from_static(v.as_bytes()),
386        }));
387
388        expected_environment_vars.sort_unstable_by_key(|e| e.key.to_owned());
389
390        assert_eq!(
391            BuildRequest {
392                command_args: vec![":".into()],
393                outputs: vec!["nix/store/fhaj6gmwns62s6ypkcldbaj2ybvkhx3p-foo".into()],
394                environment_vars: expected_environment_vars,
395                inputs: BTreeMap::from([(
396                    PathComponent::try_from(INPUT_NODE_FOO_NAME.clone()).unwrap(),
397                    INPUT_NODE_FOO.clone()
398                )]),
399                inputs_dir: "nix/store".into(),
400                constraints: HashSet::from([
401                    BuildConstraints::System(derivation1.system.clone()),
402                    BuildConstraints::ProvideBinSh
403                ]),
404                additional_files: vec![],
405                working_dir: "build".into(),
406                scratch_paths: vec!["build".into(), "nix/store".into()],
407                refscan_needles: vec![
408                    "fhaj6gmwns62s6ypkcldbaj2ybvkhx3p".into(),
409                    "mp57d33657rf34lzvlbpfa1gjfv5gmpg".into()
410                ],
411            },
412            build_request
413        );
414    }
415
416    #[test]
417    fn test_drv_with_placeholders_to_build_request() {
418        let aterm_bytes =
419            include_bytes!("tests/18m7y1d025lqgrzx8ypnhjbvq23z2kda-with-placeholders.drv");
420        let derivation = Derivation::from_aterm_bytes(aterm_bytes).expect("must parse");
421
422        let build_request =
423            derivation_to_build_request(&derivation, &BTreeMap::from([])).expect("must succeed");
424        let mut expected_environment_vars = vec![
425            EnvVar {
426                key: "FOO".into(),
427                value: "/nix/store/dgapb8kh5gis4w7hzfl5725sx5gam0nz-with-placeholders".into(),
428            },
429            EnvVar {
430                key: "BAR".into(),
431                // Non existent output placeholders should not get replaced.
432                value: hash_placeholder("non-existent").into(),
433            },
434            EnvVar {
435                key: "builder".into(),
436                value: "/bin/sh".into(),
437            },
438            EnvVar {
439                key: "name".into(),
440                value: "with-placeholders".into(),
441            },
442            EnvVar {
443                key: "out".into(),
444                value: "/nix/store/dgapb8kh5gis4w7hzfl5725sx5gam0nz-with-placeholders".into(),
445            },
446            EnvVar {
447                key: "system".into(),
448                value: "x86_64-linux".into(),
449            },
450        ];
451        expected_environment_vars.extend(NIX_ENVIRONMENT_VARS.iter().map(|(k, v)| EnvVar {
452            key: k.to_string(),
453            value: Bytes::from_static(v.as_bytes()),
454        }));
455
456        expected_environment_vars.sort_unstable_by_key(|e| e.key.to_owned());
457        assert_eq!(
458            BuildRequest {
459                command_args: vec![
460                    "/bin/sh".into(),
461                    "-c".into(),
462                    "/nix/store/dgapb8kh5gis4w7hzfl5725sx5gam0nz-with-placeholders".into(),
463                    // Non existent output placeholders should not get replaced.
464                    hash_placeholder("non-existent"),
465                ],
466                outputs: vec![
467                    "nix/store/dgapb8kh5gis4w7hzfl5725sx5gam0nz-with-placeholders".into()
468                ],
469                environment_vars: expected_environment_vars,
470                inputs: BTreeMap::new(),
471                inputs_dir: "nix/store".into(),
472                constraints: HashSet::from([
473                    BuildConstraints::System(derivation.system.clone()),
474                    BuildConstraints::ProvideBinSh
475                ]),
476                additional_files: vec![],
477                working_dir: "build".into(),
478                scratch_paths: vec!["build".into(), "nix/store".into()],
479                refscan_needles: vec!["dgapb8kh5gis4w7hzfl5725sx5gam0nz".into()],
480            },
481            build_request
482        );
483    }
484
485    #[test]
486    fn test_fod_to_build_request() {
487        let aterm_bytes = include_bytes!("tests/0hm2f1psjpcwg8fijsmr4wwxrx59s092-bar.drv");
488
489        let derivation = Derivation::from_aterm_bytes(aterm_bytes).expect("must parse");
490
491        let build_request =
492            derivation_to_build_request(&derivation, &BTreeMap::from([])).expect("must succeed");
493
494        let mut expected_environment_vars = vec![
495            EnvVar {
496                key: "builder".into(),
497                value: ":".into(),
498            },
499            EnvVar {
500                key: "name".into(),
501                value: "bar".into(),
502            },
503            EnvVar {
504                key: "out".into(),
505                value: "/nix/store/4q0pg5zpfmznxscq3avycvf9xdvx50n3-bar".into(),
506            },
507            EnvVar {
508                key: "outputHash".into(),
509                value: "08813cbee9903c62be4c5027726a418a300da4500b2d369d3af9286f4815ceba".into(),
510            },
511            EnvVar {
512                key: "outputHashAlgo".into(),
513                value: "sha256".into(),
514            },
515            EnvVar {
516                key: "outputHashMode".into(),
517                value: "recursive".into(),
518            },
519            EnvVar {
520                key: "system".into(),
521                value: ":".into(),
522            },
523        ];
524
525        expected_environment_vars.extend(NIX_ENVIRONMENT_VARS.iter().map(|(k, v)| EnvVar {
526            key: k.to_string(),
527            value: Bytes::from_static(v.as_bytes()),
528        }));
529
530        expected_environment_vars.sort_unstable_by_key(|e| e.key.to_owned());
531
532        assert_eq!(
533            BuildRequest {
534                command_args: vec![":".to_string()],
535                outputs: vec!["nix/store/4q0pg5zpfmznxscq3avycvf9xdvx50n3-bar".into()],
536                environment_vars: expected_environment_vars,
537                inputs: BTreeMap::new(),
538                inputs_dir: "nix/store".into(),
539                constraints: HashSet::from([
540                    BuildConstraints::System(derivation.system.clone()),
541                    BuildConstraints::NetworkAccess,
542                    BuildConstraints::ProvideBinSh
543                ]),
544                additional_files: vec![],
545                working_dir: "build".into(),
546                scratch_paths: vec!["build".into(), "nix/store".into()],
547                refscan_needles: vec!["4q0pg5zpfmznxscq3avycvf9xdvx50n3".into()],
548            },
549            build_request
550        );
551    }
552
553    #[test]
554    fn test_pass_as_file() {
555        // (builtins.derivation { "name" = "foo"; passAsFile = ["bar" "baz"]; bar = "baz"; baz = "bar"; system = ":"; builder = ":";}).drvPath
556        let aterm_bytes = r#"Derive([("out","/nix/store/pp17lwra2jkx8rha15qabg2q3wij72lj-foo","","")],[],[],":",":",[],[("bar","baz"),("baz","bar"),("builder",":"),("name","foo"),("out","/nix/store/pp17lwra2jkx8rha15qabg2q3wij72lj-foo"),("passAsFile","bar baz"),("system",":")])"#.as_bytes();
557
558        let derivation = Derivation::from_aterm_bytes(aterm_bytes).expect("must parse");
559
560        let build_request =
561            derivation_to_build_request(&derivation, &BTreeMap::from([])).expect("must succeed");
562
563        let mut expected_environment_vars = vec![
564            // Note how bar and baz are not present in the env anymore,
565            // but replaced with barPath, bazPath respectively.
566            EnvVar {
567                key: "barPath".into(),
568                value: "/build/.attr-1fcgpy7vc4ammr7s17j2xq88scswkgz23dqzc04g8sx5vcp2pppw".into(),
569            },
570            EnvVar {
571                key: "bazPath".into(),
572                value: "/build/.attr-15l04iksj1280dvhbzdq9ai3wlf8ac2188m9qv0gn81k9nba19ds".into(),
573            },
574            EnvVar {
575                key: "builder".into(),
576                value: ":".into(),
577            },
578            EnvVar {
579                key: "name".into(),
580                value: "foo".into(),
581            },
582            EnvVar {
583                key: "out".into(),
584                value: "/nix/store/pp17lwra2jkx8rha15qabg2q3wij72lj-foo".into(),
585            },
586            // passAsFile stays around
587            EnvVar {
588                key: "passAsFile".into(),
589                value: "bar baz".into(),
590            },
591            EnvVar {
592                key: "system".into(),
593                value: ":".into(),
594            },
595        ];
596
597        expected_environment_vars.extend(NIX_ENVIRONMENT_VARS.iter().map(|(k, v)| EnvVar {
598            key: k.to_string(),
599            value: Bytes::from_static(v.as_bytes()),
600        }));
601
602        expected_environment_vars.sort_unstable_by_key(|e| e.key.to_owned());
603
604        assert_eq!(
605            BuildRequest {
606                command_args: vec![":".to_string()],
607                outputs: vec!["nix/store/pp17lwra2jkx8rha15qabg2q3wij72lj-foo".into()],
608                environment_vars: expected_environment_vars,
609                inputs: BTreeMap::new(),
610                inputs_dir: "nix/store".into(),
611                constraints: HashSet::from([
612                    BuildConstraints::System(derivation.system.clone()),
613                    BuildConstraints::ProvideBinSh,
614                ]),
615                additional_files: vec![
616                    // baz env
617                    AdditionalFile {
618                        path: "build/.attr-15l04iksj1280dvhbzdq9ai3wlf8ac2188m9qv0gn81k9nba19ds"
619                            .into(),
620                        contents: "bar".into()
621                    },
622                    // bar env
623                    AdditionalFile {
624                        path: "build/.attr-1fcgpy7vc4ammr7s17j2xq88scswkgz23dqzc04g8sx5vcp2pppw"
625                            .into(),
626                        contents: "baz".into(),
627                    },
628                ],
629                working_dir: "build".into(),
630                scratch_paths: vec!["build".into(), "nix/store".into()],
631                refscan_needles: vec!["pp17lwra2jkx8rha15qabg2q3wij72lj".into()],
632            },
633            build_request
634        );
635    }
636}
snix_glue/snix_build.rs

snix_glue/
snix_build.rs
