snix_glue/builtins/
derivation.rs

1//! Implements `builtins.derivation`, the core of what makes Nix build packages.
2use crate::builtins::DerivationError;
3use crate::known_paths::KnownPaths;
4use crate::snix_store_io::SnixStoreIO;
5use bstr::BString;
6use nix_compat::derivation::{Derivation, Output};
7use nix_compat::nixhash::{CAHash, HashAlgo, NixHash};
8use nix_compat::store_path::{StorePath, StorePathRef};
9use snix_eval::builtin_macros::builtins;
10use snix_eval::generators::{self, GenCo, emit_warning_kind};
11use snix_eval::{
12    AddContext, ErrorKind, NixAttrs, NixContext, NixContextElement, Value, WarningKind,
13};
14use std::collections::{BTreeSet, btree_map};
15use std::rc::Rc;
16
17// Constants used for strangely named fields in derivation inputs.
18const STRUCTURED_ATTRS: &str = "__structuredAttrs";
19const IGNORE_NULLS: &str = "__ignoreNulls";
20
21/// Populate the inputs of a derivation from the build references
22/// found when scanning the derivation's parameters and extracting their contexts.
23fn populate_inputs(drv: &mut Derivation, full_context: NixContext, known_paths: &KnownPaths) {
24    for element in full_context.iter() {
25        match element {
26            NixContextElement::Plain(source) => {
27                let sp = StorePathRef::from_absolute_path(source.as_bytes())
28                    .expect("invalid store path")
29                    .to_owned();
30                drv.input_sources.insert(sp);
31            }
32
33            NixContextElement::Single {
34                name,
35                derivation: derivation_str,
36            } => {
37                // TODO: b/264
38                // We assume derivations to be passed validated, so ignoring rest
39                // and expecting parsing is ok.
40                let (derivation, _rest) =
41                    StorePath::from_absolute_path_full(derivation_str).expect("valid store path");
42
43                #[cfg(debug_assertions)]
44                assert!(
45                    _rest.iter().next().is_none(),
46                    "Extra path not empty for {}",
47                    derivation_str
48                );
49
50                match drv.input_derivations.entry(derivation.clone()) {
51                    btree_map::Entry::Vacant(entry) => {
52                        entry.insert(BTreeSet::from([name.clone()]));
53                    }
54
55                    btree_map::Entry::Occupied(mut entry) => {
56                        entry.get_mut().insert(name.clone());
57                    }
58                }
59            }
60
61            NixContextElement::Derivation(drv_path) => {
62                let (derivation, _rest) =
63                    StorePath::from_absolute_path_full(drv_path).expect("valid store path");
64
65                #[cfg(debug_assertions)]
66                assert!(
67                    _rest.iter().next().is_none(),
68                    "Extra path not empty for {}",
69                    drv_path
70                );
71
72                // We need to know all the outputs *names* of that derivation.
73                let output_names = known_paths
74                    .get_drv_by_drvpath(&derivation)
75                    .expect("no known derivation associated to that derivation path")
76                    .outputs
77                    .keys();
78
79                // FUTUREWORK(performance): ideally, we should be able to clone
80                // cheaply those outputs rather than duplicate them all around.
81                match drv.input_derivations.entry(derivation.clone()) {
82                    btree_map::Entry::Vacant(entry) => {
83                        entry.insert(output_names.cloned().collect());
84                    }
85
86                    btree_map::Entry::Occupied(mut entry) => {
87                        entry.get_mut().extend(output_names.cloned());
88                    }
89                }
90
91                drv.input_sources.insert(derivation);
92            }
93        }
94    }
95}
96
97/// Populate the output configuration of a derivation based on the
98/// parameters passed to the call, configuring a fixed-output derivation output
99/// if necessary.
100///
101/// This function handles all possible combinations of the
102/// parameters, including invalid ones.
103///
104/// Due to the support for SRI hashes, and how these are passed along to
105/// builtins.derivation, outputHash and outputHashAlgo can have values which
106/// need to be further modified before constructing the Derivation struct.
107///
108/// If outputHashAlgo is an SRI hash, outputHashAlgo must either be an empty
109/// string, or the hash algorithm as specified in the (single) SRI (entry).
110/// SRI strings with multiple hash algorithms are not supported.
111///
112/// In case an SRI string was used, the (single) fixed output is populated
113/// with the hash algo name, and the hash digest is populated with the
114/// (lowercase) hex encoding of the digest.
115///
116/// These values are only rewritten for the outputs, not what's passed to env.
117///
118/// The return value may optionally contain a warning.
119fn handle_fixed_output(
120    drv: &mut Derivation,
121    hash_str: Option<String>,      // in nix: outputHash
122    hash_algo_str: Option<String>, // in nix: outputHashAlgo
123    hash_mode_str: Option<String>, // in nix: outputHashmode
124) -> Result<Option<WarningKind>, ErrorKind> {
125    // If outputHash is provided, ensure hash_algo_str is compatible.
126    // If outputHash is not provided, do nothing.
127    if let Some(hash_str) = hash_str {
128        // treat an empty algo as None
129        let hash_algo_str = match hash_algo_str {
130            Some(s) if s.is_empty() => None,
131            Some(s) => Some(s),
132            None => None,
133        };
134
135        let hash_algo = hash_algo_str
136            .map(|s| HashAlgo::try_from(s.as_str()))
137            .transpose()
138            .map_err(DerivationError::InvalidOutputHash)?;
139
140        // construct a NixHash.
141        let nixhash =
142            NixHash::from_str(&hash_str, hash_algo).map_err(DerivationError::InvalidOutputHash)?;
143        let algo = nixhash.algo();
144
145        // construct the fixed output.
146        drv.outputs.insert(
147            "out".to_string(),
148            Output {
149                path: None,
150                ca_hash: match hash_mode_str.as_deref() {
151                    None | Some("flat") => Some(CAHash::Flat(nixhash)),
152                    Some("recursive") => Some(CAHash::Nar(nixhash)),
153                    Some(other) => {
154                        return Err(DerivationError::InvalidOutputHashMode(other.to_string()))?;
155                    }
156                },
157            },
158        );
159
160        // Peek at hash_str once more.
161        // If it was a SRI hash, but is not using the correct length, this means
162        // the padding was wrong. Emit a warning in that case.
163        let sri_prefix = format!("{}-", algo);
164        if let Some(rest) = hash_str.strip_prefix(&sri_prefix) {
165            if data_encoding::BASE64.encode_len(algo.digest_length()) != rest.len() {
166                return Ok(Some(WarningKind::SRIHashWrongPadding));
167            }
168        }
169    }
170    Ok(None)
171}
172
173#[builtins(state = "Rc<SnixStoreIO>")]
174pub(crate) mod derivation_builtins {
175    use std::collections::BTreeMap;
176
177    use bstr::ByteSlice;
178
179    use nix_compat::store_path::hash_placeholder;
180    use snix_eval::generators::Gen;
181    use snix_eval::{NixContext, NixContextElement, NixString};
182
183    use crate::builtins::utils::{select_string, strong_importing_coerce_to_string};
184    use crate::fetchurl::fetchurl_derivation_to_fetch;
185
186    use super::*;
187
188    #[builtin("placeholder")]
189    async fn builtin_placeholder(co: GenCo, input: Value) -> Result<Value, ErrorKind> {
190        if input.is_catchable() {
191            return Ok(input);
192        }
193
194        let placeholder = hash_placeholder(
195            input
196                .to_str()
197                .context("looking at output name in builtins.placeholder")?
198                .to_str()?,
199        );
200
201        Ok(placeholder.into())
202    }
203
204    /// Strictly construct a Nix derivation from the supplied arguments.
205    ///
206    /// This is considered an internal function, users usually want to
207    /// use the higher-level `builtins.derivation` instead.
208    #[builtin("derivationStrict")]
209    async fn builtin_derivation_strict(
210        state: Rc<SnixStoreIO>,
211        co: GenCo,
212        input: Value,
213    ) -> Result<Value, ErrorKind> {
214        if input.is_catchable() {
215            return Ok(input);
216        }
217
218        let input = input.to_attrs()?;
219        let name = generators::request_force(&co, input.select_required("name")?.clone()).await;
220
221        if name.is_catchable() {
222            return Ok(name);
223        }
224
225        let name = name.to_str().context("determining derivation name")?;
226        if name.is_empty() {
227            return Err(ErrorKind::Abort("derivation has empty name".to_string()));
228        }
229        let name = name.to_str()?;
230
231        let mut drv = Derivation::default();
232        drv.outputs.insert("out".to_string(), Default::default());
233        let mut input_context = NixContext::new();
234
235        /// Inserts a key and value into the drv.environment BTreeMap, and fails if the
236        /// key did already exist before.
237        fn insert_env(
238            drv: &mut Derivation,
239            k: &str, /* TODO: non-utf8 env keys */
240            v: BString,
241        ) -> Result<(), DerivationError> {
242            if drv.environment.insert(k.into(), v).is_some() {
243                return Err(DerivationError::DuplicateEnvVar(k.into()));
244            }
245            Ok(())
246        }
247
248        // Check whether null attributes should be ignored or passed through.
249        let ignore_nulls = match input.select(IGNORE_NULLS) {
250            Some(b) => generators::request_force(&co, b.clone()).await.as_bool()?,
251            None => false,
252        };
253
254        // peek at the STRUCTURED_ATTRS argument.
255        // If it's set and true, provide a BTreeMap that gets populated while looking at the arguments.
256        // We need it to be a BTreeMap, so iteration order of keys is reproducible.
257        let mut structured_attrs: Option<BTreeMap<String, serde_json::Value>> =
258            match input.select(STRUCTURED_ATTRS) {
259                Some(b) => generators::request_force(&co, b.clone())
260                    .await
261                    .as_bool()?
262                    .then_some(Default::default()),
263                None => None,
264            };
265
266        // Look at the arguments passed to builtins.derivationStrict.
267        // Some set special fields in the Derivation struct, some change
268        // behaviour of other functionality.
269        for (arg_name, arg_value) in input.clone().into_iter_sorted() {
270            let arg_name = arg_name.to_str()?;
271            // force the current value.
272            let value = generators::request_force(&co, arg_value).await;
273
274            // filter out nulls if ignore_nulls is set.
275            if ignore_nulls && matches!(value, Value::Null) {
276                continue;
277            }
278
279            match arg_name {
280                // Command line arguments to the builder.
281                // These are only set in drv.arguments.
282                "args" => {
283                    for arg in value.to_list()? {
284                        match strong_importing_coerce_to_string(&co, arg).await {
285                            Err(cek) => return Ok(Value::from(cek)),
286                            Ok(s) => {
287                                input_context.mimic(&s);
288                                drv.arguments.push(s.to_str()?.to_owned())
289                            }
290                        }
291                    }
292                }
293
294                // If outputs is set, remove the original default `out` output,
295                // and replace it with the list of outputs.
296                "outputs" => {
297                    let outputs = value
298                        .to_list()
299                        .context("looking at the `outputs` parameter of the derivation")?;
300
301                    // Remove the original default `out` output.
302                    drv.outputs.clear();
303
304                    let mut output_names = Vec::with_capacity(outputs.len());
305
306                    for output in outputs {
307                        let output_name = generators::request_force(&co, output)
308                            .await
309                            .to_str()
310                            .context("determining output name")?;
311
312                        input_context.mimic(&output_name);
313
314                        // Populate drv.outputs
315                        if drv
316                            .outputs
317                            .insert(output_name.to_str()?.to_owned(), Default::default())
318                            .is_some()
319                        {
320                            Err(DerivationError::DuplicateOutput(
321                                output_name.to_str_lossy().into_owned(),
322                            ))?
323                        }
324                        output_names.push(output_name.to_str()?.to_owned());
325                    }
326
327                    match structured_attrs.as_mut() {
328                        // add outputs to the json itself (as a list of strings)
329                        Some(structured_attrs) => {
330                            structured_attrs.insert(arg_name.into(), output_names.into());
331                        }
332                        // add drv.environment["outputs"] as a space-separated list
333                        None => {
334                            insert_env(&mut drv, arg_name, output_names.join(" ").into())?;
335                        }
336                    }
337                    // drv.environment[$output_name] is added after the loop,
338                    // with whatever is in drv.outputs[$output_name].
339                }
340
341                // handle builder and system.
342                "builder" | "system" => {
343                    match strong_importing_coerce_to_string(&co, value).await {
344                        Err(cek) => return Ok(Value::from(cek)),
345                        Ok(val_str) => {
346                            input_context.mimic(&val_str);
347
348                            if arg_name == "builder" {
349                                val_str.to_str()?.clone_into(&mut drv.builder);
350                            } else {
351                                val_str.to_str()?.clone_into(&mut drv.system);
352                            }
353
354                            // Either populate drv.environment or structured_attrs.
355                            if let Some(ref mut structured_attrs) = structured_attrs {
356                                // No need to check for dups, we only iterate over every attribute name once
357                                structured_attrs.insert(
358                                    arg_name.to_owned(),
359                                    val_str.to_str()?.to_owned().into(),
360                                );
361                            } else {
362                                insert_env(&mut drv, arg_name, val_str.as_bytes().into())?;
363                            }
364                        }
365                    }
366                }
367
368                // Don't add STRUCTURED_ATTRS if enabled.
369                STRUCTURED_ATTRS if structured_attrs.is_some() => continue,
370                // IGNORE_NULLS is always skipped, even if it's not set to true.
371                IGNORE_NULLS => continue,
372
373                // all other args.
374                _ => {
375                    // In SA case, force and add to structured attrs.
376                    // In non-SA case, coerce to string and add to env.
377                    if let Some(ref mut structured_attrs) = structured_attrs {
378                        let val = generators::request_force(&co, value).await;
379                        if val.is_catchable() {
380                            return Ok(val);
381                        }
382
383                        let (val_json, context) = val.into_contextful_json(&co).await?;
384                        input_context.extend(context.into_iter());
385
386                        // No need to check for dups, we only iterate over every attribute name once
387                        structured_attrs.insert(arg_name.to_owned(), val_json);
388                    } else {
389                        match strong_importing_coerce_to_string(&co, value).await {
390                            Err(cek) => return Ok(Value::from(cek)),
391                            Ok(val_str) => {
392                                input_context.mimic(&val_str);
393
394                                insert_env(&mut drv, arg_name, val_str.as_bytes().into())?;
395                            }
396                        }
397                    }
398                }
399            }
400        }
401        // end of per-argument loop
402
403        // Configure fixed-output derivations if required.
404        {
405            let output_hash = match select_string(&co, &input, "outputHash")
406                .await
407                .context("evaluating the `outputHash` parameter")?
408            {
409                Err(cek) => return Ok(Value::from(cek)),
410                Ok(s) => s,
411            };
412            let output_hash_algo = match select_string(&co, &input, "outputHashAlgo")
413                .await
414                .context("evaluating the `outputHashAlgo` parameter")?
415            {
416                Err(cek) => return Ok(Value::from(cek)),
417                Ok(s) => s,
418            };
419            let output_hash_mode = match select_string(&co, &input, "outputHashMode")
420                .await
421                .context("evaluating the `outputHashMode` parameter")?
422            {
423                Err(cek) => return Ok(Value::from(cek)),
424                Ok(s) => s,
425            };
426
427            if let Some(warning) =
428                handle_fixed_output(&mut drv, output_hash, output_hash_algo, output_hash_mode)?
429            {
430                emit_warning_kind(&co, warning).await;
431            }
432        }
433
434        // Each output name needs to exist in the environment, at this
435        // point initialised as an empty string, as the ATerm serialization of that is later
436        // used for the output path calculation (which will also update output
437        // paths post-calculation, both in drv.environment and drv.outputs)
438        for output in drv.outputs.keys() {
439            if drv
440                .environment
441                .insert(output.to_string(), String::new().into())
442                .is_some()
443            {
444                emit_warning_kind(&co, WarningKind::ShadowedOutput(output.to_string())).await;
445            }
446        }
447
448        if let Some(structured_attrs) = structured_attrs {
449            // configure __json
450            drv.environment.insert(
451                "__json".to_string(),
452                BString::from(serde_json::to_string(&structured_attrs)?),
453            );
454        }
455
456        let mut known_paths = state.as_ref().known_paths.borrow_mut();
457        populate_inputs(&mut drv, input_context, &known_paths);
458
459        // At this point, derivation fields are fully populated from
460        // eval data structures.
461        drv.validate(false)
462            .map_err(DerivationError::InvalidDerivation)?;
463
464        // Calculate the hash_derivation_modulo for the current derivation..
465        debug_assert!(
466            drv.outputs.values().all(|output| { output.path.is_none() }),
467            "outputs should still be unset"
468        );
469
470        // Mutate the Derivation struct and set output paths
471        drv.calculate_output_paths(
472            name,
473            // This one is still intermediate (so not added to known_paths),
474            // as the outputs are still unset.
475            &drv.hash_derivation_modulo(|drv_path| {
476                *known_paths
477                    .get_hash_derivation_modulo(&drv_path.to_owned())
478                    .unwrap_or_else(|| panic!("{} not found", drv_path))
479            }),
480        )
481        .map_err(DerivationError::InvalidDerivation)?;
482
483        let drv_path = drv
484            .calculate_derivation_path(name)
485            .map_err(DerivationError::InvalidDerivation)?;
486
487        // Assemble the attrset to return from this builtin.
488        let out = Value::Attrs(Box::new(NixAttrs::from_iter(
489            drv.outputs
490                .iter()
491                .map(|(name, output)| {
492                    (
493                        name.clone(),
494                        NixString::new_context_from(
495                            NixContextElement::Single {
496                                name: name.clone(),
497                                derivation: drv_path.to_absolute_path(),
498                            }
499                            .into(),
500                            output.path.as_ref().unwrap().to_absolute_path(),
501                        ),
502                    )
503                })
504                .chain(std::iter::once((
505                    "drvPath".to_owned(),
506                    NixString::new_context_from(
507                        NixContextElement::Derivation(drv_path.to_absolute_path()).into(),
508                        drv_path.to_absolute_path(),
509                    ),
510                ))),
511        )));
512
513        // If the derivation is a fake derivation (builtin:fetchurl),
514        // synthesize a [Fetch] and add it there, too.
515        if drv.builder == "builtin:fetchurl" {
516            let (name, fetch) =
517                fetchurl_derivation_to_fetch(&drv).map_err(|e| ErrorKind::SnixError(Rc::new(e)))?;
518
519            known_paths
520                .add_fetch(fetch, &name)
521                .map_err(|e| ErrorKind::SnixError(Rc::new(e)))?;
522        }
523
524        // Register the Derivation in known_paths.
525        known_paths.add_derivation(drv_path, drv);
526
527        Ok(out)
528    }
529}