nix_compat/derivation/
parser.rs

1//! This module constructs a [UnverifiedDerivation] by parsing its [ATerm][]
2//! serialization.
3//!
4//! [ATerm]: http://program-transformation.org/Tools/ATermFormat.html
5
6use nom::Parser;
7use nom::bytes::streaming::tag;
8use nom::character::streaming::char as nomchar;
9use nom::combinator::{all_consuming, consumed, map_res};
10use nom::multi::{separated_list0, separated_list1};
11use nom::sequence::{delimited, preceded, separated_pair, terminated};
12use std::collections::{BTreeMap, BTreeSet, btree_map};
13use thiserror;
14
15use crate::derivation::parse_error::{ErrorKind, NomError, NomResult, into_nomerror};
16use crate::derivation::{
17    DerivationBuilder, OutputHash, OutputName, Outputs, UnverifiedDerivation,
18    UnverifiedOutputsBuilder, write,
19};
20use crate::store_path::{self, StorePath};
21use crate::{aterm, nixhash};
22
23#[derive(Debug, thiserror::Error)]
24/// The error type for derivation parsing errors
25#[allow(missing_docs)]
26pub enum Error<I> {
27    #[error("parsing error: {0}")]
28    Parser(#[from] NomError<I>),
29    #[error("premature EOF")]
30    Incomplete,
31    #[error("validation error: {0}")]
32    Validation(super::DerivationError),
33}
34
35/// Convenience conversion of borrowed Error to an owned counterpart.
36impl From<Error<&[u8]>> for Error<Vec<u8>> {
37    fn from(value: Error<&[u8]>) -> Self {
38        match value {
39            Error::Parser(nom_error) => Error::Parser(NomError {
40                input: nom_error.input.to_vec(),
41                code: nom_error.code,
42            }),
43            Error::Incomplete => Error::Incomplete,
44            Error::Validation(e) => Error::Validation(e),
45        }
46    }
47}
48
49pub fn parse(i: &[u8]) -> Result<UnverifiedDerivation, Error<&[u8]>> {
50    match all_consuming(parse_derivation).parse(i) {
51        Ok((rest, (builder, outputs))) => {
52            // this shouldn't happen, as all_consuming shouldn't return.
53            debug_assert!(rest.is_empty());
54
55            // invoke validate
56            builder.build_unverified(outputs).map_err(Error::Validation)
57        }
58        Err(nom::Err::Incomplete(_)) => Err(Error::Incomplete),
59        Err(nom::Err::Error(e) | nom::Err::Failure(e)) => Err(e.into()),
60    }
61}
62
63/// This parses a derivation in streaming fashion.
64/// If the parse is successful, it returns the leftover bytes which were not used for the parsing.
65/// If the parse is unsuccessful, either it returns incomplete or an error with the input as
66/// leftover.
67#[allow(dead_code)]
68pub fn parse_streaming(i: &[u8]) -> (Result<UnverifiedDerivation, Error<&[u8]>>, &[u8]) {
69    match consumed(parse_derivation).parse(i) {
70        Ok((_, (rest, (builder, outputs)))) => {
71            // invoke validate
72            match builder.build_unverified(outputs) {
73                Ok(derivation) => (Ok(derivation), rest),
74                Err(e) => (Err(Error::Validation(e)), i),
75            }
76        }
77        Err(nom::Err::Incomplete(_)) => (Err(Error::Incomplete), i),
78        Err(nom::Err::Error(e) | nom::Err::Failure(e)) => (Err(e.into()), i),
79    }
80}
81
82/// Parse one output in ATerm. This is 4 string fields inside parans:
83/// output name, output path, algo (and mode), digest.
84/// Returns the [`OutputName`], output [`StorePath`] and possible FOD [`OutputHash`].
85fn parse_output(i: &[u8]) -> NomResult<&[u8], (OutputName, StorePath, Option<OutputHash>)> {
86    delimited(
87        nomchar('('),
88        map_res(
89            |i| {
90                (
91                    terminated(aterm::parse_string_field, nomchar(',')),
92                    terminated(aterm::parse_string_field, nomchar(',')),
93                    terminated(aterm::parse_string_field, nomchar(',')),
94                    aterm::parse_bytes_field,
95                )
96                    .parse(i)
97                    .map_err(into_nomerror)
98            },
99            |(output_name_str, output_path_str, algo_and_mode, encoded_digest)| -> Result<_, nom::Err<NomError<&[u8]>>> {
100                let output_name: OutputName = output_name_str.parse().map_err(|err| {
101                    nom::Err::Failure(NomError {
102                        input: i,
103                        code: ErrorKind::InvalidOutputName(err),
104                    })
105                })?;
106
107                // This can't be an empty string in ATerms written to disk.
108                // This being an empty string can only occur during output path calculation.
109                let path = string_to_store_path(i, &output_path_str)?;
110
111                let output_hash = if algo_and_mode.is_empty() && encoded_digest.is_empty() {
112                    None
113                } else {
114                    let digest =
115                        data_encoding::HEXLOWER
116                            .decode(&encoded_digest)
117                            .map_err(|err| {
118                                nom::Err::Failure(NomError {
119                                    input: i,
120                                    code: ErrorKind::NixHashError(
121                                        // TODO: do we still need the outer error?
122                                        nixhash::Error::InvalidBase16Encoding(err),
123                                    ),
124                                })
125                            })?;
126
127                    Some(
128                        OutputHash::from_mode_algo_and_digest(&algo_and_mode, digest)
129                            .map_err(|err| {
130                                nom::Err::Failure(NomError {
131                                    input: i,
132                                    code: ErrorKind::NixHashError(err),
133                                })
134                            })?,
135                    )
136                };
137                Ok((output_name, path, output_hash))
138            },
139        ),
140        nomchar(')'),
141    )
142    .parse(i)
143}
144
145/// Parse multiple outputs in ATerm. This is a list of things acccepted by
146/// parse_output, and takes care of turning the (String, Output) returned from
147/// it to a BTreeMap.
148/// We don't use parse_kv here, as it's dealing with 2-tuples, and these are
149/// 4-tuples.
150fn parse_outputs(i: &[u8]) -> NomResult<&[u8], Outputs> {
151    let res = delimited(
152        nomchar('['),
153        separated_list1(tag(","), parse_output),
154        nomchar(']'),
155    )
156    .parse(i);
157
158    match res {
159        Ok((rst, outputs_lst)) => {
160            let mut builder = UnverifiedOutputsBuilder::default();
161            for (output_name, store_path, output_hash) in outputs_lst {
162                builder
163                    .try_insert(output_name, store_path, output_hash)
164                    .map_err(|err| {
165                        nom::Err::Failure(NomError {
166                            input: i,
167                            code: ErrorKind::InvalidOutputs(err),
168                        })
169                    })?;
170            }
171            let outputs = builder.try_build().map_err(|err| {
172                nom::Err::Failure(NomError {
173                    input: i,
174                    code: ErrorKind::InvalidOutputs(err),
175                })
176            })?;
177            Ok((rst, outputs))
178        }
179        // pass regular parse errors along
180        Err(e) => Err(e),
181    }
182}
183
184fn parse_input_derivations(
185    i: &[u8],
186) -> NomResult<&[u8], BTreeMap<StorePath, BTreeSet<OutputName>>> {
187    let (i, input_derivations_list) = parse_kv(aterm::parse_string_list)(i)?;
188
189    // This is a HashMap of drv paths to a list of output names.
190    let mut input_derivations: BTreeMap<StorePath, BTreeSet<_>> = BTreeMap::new();
191
192    for (input_derivation, output_names_strings) in input_derivations_list {
193        let mut output_names = BTreeSet::<OutputName>::new();
194        for output_name_string in output_names_strings.into_iter() {
195            let output_name = OutputName::try_from(output_name_string).map_err(|err| {
196                nom::Err::Failure(NomError {
197                    input: i,
198                    code: ErrorKind::InvalidOutputName(err),
199                })
200            })?;
201
202            if output_names.contains(&output_name) {
203                return Err(nom::Err::Failure(NomError {
204                    input: i,
205                    code: ErrorKind::DuplicateInputDerivationOutputName(
206                        output_name,
207                        input_derivation,
208                    ),
209                }));
210            }
211            output_names.insert(output_name);
212        }
213
214        let input_derivation = string_to_store_path(i, input_derivation.as_str())?;
215
216        input_derivations.insert(input_derivation, output_names);
217    }
218
219    Ok((i, input_derivations))
220}
221
222fn parse_input_sources(i: &[u8]) -> NomResult<&[u8], BTreeSet<StorePath>> {
223    let (i, input_sources_lst) = aterm::parse_string_list(i).map_err(into_nomerror)?;
224
225    let mut input_sources: BTreeSet<_> = BTreeSet::new();
226    for input_source in input_sources_lst.into_iter() {
227        let input_source = string_to_store_path(i, input_source.as_str())?;
228        if input_sources.contains(&input_source) {
229            return Err(nom::Err::Failure(NomError {
230                input: i,
231                code: ErrorKind::DuplicateInputSource(input_source.to_owned()),
232            }));
233        } else {
234            input_sources.insert(input_source);
235        }
236    }
237
238    Ok((i, input_sources))
239}
240
241fn string_to_store_path<'i>(
242    i: &'i [u8],
243    path_str: &str,
244) -> Result<StorePath, nom::Err<NomError<&'i [u8]>>> {
245    let path = StorePath::from_absolute_path(path_str.as_bytes()).map_err(
246        |e: store_path::ParseStorePathError| {
247            nom::Err::Failure(NomError {
248                input: i,
249                code: e.into(),
250            })
251        },
252    )?;
253
254    #[cfg(debug_assertions)]
255    assert_eq!(path_str, path.to_absolute_path());
256
257    Ok(path)
258}
259
260pub fn parse_derivation(i: &[u8]) -> NomResult<&[u8], (DerivationBuilder, Outputs)> {
261    use nom::Parser;
262    preceded(
263        tag(write::DERIVATION_PREFIX),
264        delimited(
265            // inside parens
266            nomchar('('),
267            // tuple requires all errors to be of the same type, so we need to be a
268            // bit verbose here wrapping generic IResult into [NomATermResult].
269            (
270                // parse outputs
271                terminated(parse_outputs, nomchar(',')),
272                // // parse input derivations
273                terminated(parse_input_derivations, nomchar(',')),
274                // // parse input sources
275                terminated(parse_input_sources, nomchar(',')),
276                // // parse system
277                |i| {
278                    terminated(aterm::parse_string_field, nomchar(','))
279                        .parse(i)
280                        .map_err(into_nomerror)
281                },
282                // // parse builder
283                |i| {
284                    terminated(aterm::parse_string_field, nomchar(','))
285                        .parse(i)
286                        .map_err(into_nomerror)
287                },
288                // // parse arguments
289                |i| {
290                    terminated(aterm::parse_string_list, nomchar(','))
291                        .parse(i)
292                        .map_err(into_nomerror)
293                },
294                // parse environment
295                parse_kv(aterm::parse_bytes_field),
296            ),
297            nomchar(')'),
298        )
299        .map(
300            |(
301                outputs,
302                input_derivations,
303                input_sources,
304                system,
305                command,
306                arguments,
307                environment,
308            )| {
309                let outputs_builder = outputs.clone().into_builder();
310                let builder = DerivationBuilder {
311                    arguments,
312                    command,
313                    environment,
314                    input_derivations,
315                    input_sources,
316                    outputs: outputs_builder,
317                    system,
318                };
319                (builder, outputs)
320            },
321        ),
322    )
323    .parse(i)
324}
325
326/// Parse a list of key/value pairs into a BTreeMap.
327/// The parser for the values can be passed in.
328/// In terms of ATerm, this is just a 2-tuple,
329/// but we have the additional restriction that the first element needs to be
330/// unique across all tuples.
331pub(crate) fn parse_kv<'a, V, VF>(
332    vf: VF,
333) -> impl FnMut(&'a [u8]) -> NomResult<&'a [u8], BTreeMap<String, V>> + 'static
334where
335    VF: FnMut(&'a [u8]) -> nom::IResult<&'a [u8], V, nom::error::Error<&'a [u8]>> + Clone + 'static,
336{
337    move |i|
338    // inside brackets
339    delimited(
340        nomchar('['),
341        |ii| {
342            let res = separated_list0(
343                nomchar(','),
344                // inside parens
345                delimited(
346                    nomchar('('),
347                    separated_pair(
348                        aterm::parse_string_field,
349                        nomchar(','),
350                        vf.clone(),
351                    ),
352                    nomchar(')'),
353                ),
354            ).parse(ii).map_err(into_nomerror);
355
356            match res {
357                Ok((rest, pairs)) => {
358                    let mut kvs: BTreeMap<String, V> = BTreeMap::new();
359                    for (k, v) in pairs.into_iter() {
360                        // collect the 2-tuple to a BTreeMap,
361                        // and fail if the key was already seen before.
362                        match kvs.entry(k) {
363                            btree_map::Entry::Vacant(e) => { e.insert(v); },
364                            btree_map::Entry::Occupied(e) => {
365                                return Err(nom::Err::Failure(NomError {
366                                    input: i,
367                                    code: ErrorKind::DuplicateMapKey(e.key().clone()),
368                                }));
369                            }
370                        }
371                    }
372                    Ok((rest, kvs))
373                }
374                Err(e) => Err(e),
375            }
376        },
377        nomchar(']'),
378    ).parse(i)
379}
380
381#[cfg(test)]
382mod tests {
383    use super::OutputHash;
384    use crate::derivation::{OutputHashMode, OutputName, Outputs};
385    use crate::store_path::StorePathRef;
386    use crate::{
387        derivation::{NixHash, parse_error::ErrorKind},
388        store_path::StorePath,
389    };
390    use std::collections::{BTreeMap, BTreeSet};
391    use std::sync::LazyLock;
392
393    use bstr::{BString, ByteSlice};
394    use hex_literal::hex;
395    use rstest::rstest;
396
397    static EXP_MULTI_OUTPUTS: LazyLock<Outputs> = LazyLock::new(|| {
398        Outputs::input_addressed_from_iter([
399            (
400                OutputName::from_static("lib").unwrap(),
401                StorePath::from_bytes(b"2vixb94v0hy2xc6p7mbnxxcyc095yyia-has-multi-out-lib")
402                    .unwrap(),
403            ),
404            (
405                OutputName::out(),
406                StorePath::from_bytes(b"55lwldka5nyxa08wnvlizyqw02ihy8ic-has-multi-out".as_bytes())
407                    .unwrap(),
408            ),
409        ])
410        .expect("valid outputs")
411    });
412
413    static EXP_AB_MAP: LazyLock<BTreeMap<String, BString>> = LazyLock::new(|| {
414        let mut b = BTreeMap::new();
415        b.insert("a".to_string(), b"1".into());
416        b.insert("b".to_string(), b"2".into());
417        b
418    });
419
420    static EXP_INPUT_DERIVATIONS_SIMPLE: LazyLock<BTreeMap<StorePath, BTreeSet<OutputName>>> =
421        LazyLock::new(|| {
422            let mut b = BTreeMap::new();
423            b.insert(
424                StorePath::from_bytes(b"8bjm87p310sb7r2r0sg4xrynlvg86j8k-hello-2.12.1.tar.gz.drv")
425                    .unwrap(),
426                BTreeSet::from([OutputName::out()]),
427            );
428            b.insert(
429                StorePath::from_bytes(b"p3jc8aw45dza6h52v81j7lk69khckmcj-bash-5.2-p15.drv")
430                    .unwrap(),
431                BTreeSet::from([OutputName::out(), "lib".parse().expect("valid OutputName")]),
432            );
433            b
434        });
435
436    static EXP_INPUT_DERIVATIONS_SIMPLE_ATERM: LazyLock<String> = LazyLock::new(|| {
437        format!(
438            "[(\"{0}\",[\"out\"]),(\"{1}\",[\"out\",\"lib\"])]",
439            "/nix/store/8bjm87p310sb7r2r0sg4xrynlvg86j8k-hello-2.12.1.tar.gz.drv",
440            "/nix/store/p3jc8aw45dza6h52v81j7lk69khckmcj-bash-5.2-p15.drv"
441        )
442    });
443
444    static EXP_INPUT_SOURCES_SIMPLE: LazyLock<BTreeSet<String>> = LazyLock::new(|| {
445        let mut b = BTreeSet::new();
446        b.insert("/nix/store/55lwldka5nyxa08wnvlizyqw02ihy8ic-has-multi-out".to_string());
447        b.insert("/nix/store/2vixb94v0hy2xc6p7mbnxxcyc095yyia-has-multi-out-lib".to_string());
448        b
449    });
450
451    /// Ensure parsing KVs works
452    #[rstest]
453    #[case::empty(b"[]", &BTreeMap::new(), b"")]
454    #[case::simple(b"[(\"a\",\"1\"),(\"b\",\"2\")]", &EXP_AB_MAP, b"")]
455    fn parse_kv(
456        #[case] input: &'static [u8],
457        #[case] expected: &BTreeMap<String, BString>,
458        #[case] exp_rest: &[u8],
459    ) {
460        let (rest, parsed) =
461            super::parse_kv(crate::aterm::parse_bytes_field)(input).expect("must parse");
462        assert_eq!(exp_rest, rest, "expected remainder");
463        assert_eq!(*expected, parsed);
464    }
465
466    #[rstest]
467    #[case::incomplete_empty(b"[")]
468    #[case::incomplete_simple(b"[(\"a\",\"1\")")]
469    #[case::incomplete_complicated_escape(b"[(\"a")]
470    #[case::incomplete_complicated_sep(b"[(\"a\",")]
471    #[case::incomplete_complicated_multi_escape(b"[(\"a\",\"")]
472    #[case::incomplete_complicated_multi_outer_sep(b"[(\"a\",\"b\"),")]
473    fn parse_kv_incomplete(#[case] input: &'static [u8]) {
474        assert!(matches!(
475            super::parse_kv(crate::aterm::parse_bytes_field)(input),
476            Err(nom::Err::Incomplete(_))
477        ));
478    }
479
480    /// Ensures the kv parser complains about duplicate map keys
481    #[test]
482    fn parse_kv_fail_dup_keys() {
483        let input: &'static [u8] = b"[(\"a\",\"1\"),(\"a\",\"2\")]";
484        let e = super::parse_kv(crate::aterm::parse_bytes_field)(input).expect_err("must fail");
485
486        match e {
487            nom::Err::Failure(e) => {
488                assert_eq!(ErrorKind::DuplicateMapKey("a".to_string()), e.code);
489            }
490            _ => panic!("unexpected error"),
491        }
492    }
493
494    /// Ensure parsing input derivations works.
495    #[rstest]
496    #[case::empty(b"[]", &BTreeMap::new())]
497    #[case::simple(EXP_INPUT_DERIVATIONS_SIMPLE_ATERM.as_bytes(), &EXP_INPUT_DERIVATIONS_SIMPLE)]
498    fn parse_input_derivations(
499        #[case] input: &'static [u8],
500        #[case] expected: &BTreeMap<StorePath, BTreeSet<OutputName>>,
501    ) {
502        let (rest, parsed) = super::parse_input_derivations(input).expect("must parse");
503
504        assert_eq!(expected, &parsed, "parsed mismatch");
505        assert!(rest.is_empty(), "rest must be empty");
506    }
507
508    /// Ensures the input derivation parser complains about duplicate output names
509    #[test]
510    fn parse_input_derivations_fail_dup_output_names() {
511        let input_str = format!(
512            "[(\"{0}\",[\"out\"]),(\"{1}\",[\"out\",\"out\"])]",
513            "/nix/store/8bjm87p310sb7r2r0sg4xrynlvg86j8k-hello-2.12.1.tar.gz.drv",
514            "/nix/store/p3jc8aw45dza6h52v81j7lk69khckmcj-bash-5.2-p15.drv"
515        );
516        let e = super::parse_input_derivations(input_str.as_bytes()).expect_err("must fail");
517
518        match e {
519            nom::Err::Failure(e) => {
520                assert_eq!(
521                    ErrorKind::DuplicateInputDerivationOutputName(
522                        "out".parse().expect("Valid OutputName"),
523                        "/nix/store/p3jc8aw45dza6h52v81j7lk69khckmcj-bash-5.2-p15.drv".to_string(),
524                    ),
525                    e.code
526                );
527            }
528            _ => panic!("unexpected error"),
529        }
530    }
531
532    /// Ensure parsing input sources works
533    #[rstest]
534    #[case::empty(b"[]", &BTreeSet::new())]
535    #[case::simple(b"[\"/nix/store/55lwldka5nyxa08wnvlizyqw02ihy8ic-has-multi-out\",\"/nix/store/2vixb94v0hy2xc6p7mbnxxcyc095yyia-has-multi-out-lib\"]", &EXP_INPUT_SOURCES_SIMPLE)]
536    fn parse_input_sources(#[case] input: &'static [u8], #[case] expected: &BTreeSet<String>) {
537        let (rest, parsed) = super::parse_input_sources(input).expect("must parse");
538
539        assert_eq!(
540            expected,
541            &parsed
542                .iter()
543                .map(StorePath::to_absolute_path)
544                .collect::<BTreeSet<_>>(),
545            "parsed mismatch"
546        );
547        assert!(rest.is_empty(), "rest must be empty");
548    }
549
550    /// Ensures the input sources parser complains about duplicate input sources
551    #[test]
552    fn parse_input_sources_fail_dup_keys() {
553        let input: &'static [u8] = b"[\"/nix/store/55lwldka5nyxa08wnvlizyqw02ihy8ic-foo\",\"/nix/store/55lwldka5nyxa08wnvlizyqw02ihy8ic-foo\"]";
554        let e = super::parse_input_sources(input).expect_err("must fail");
555
556        match e {
557            nom::Err::Failure(e) => {
558                assert_eq!(
559                    ErrorKind::DuplicateInputSource(
560                        StorePathRef::from_absolute_path(
561                            "/nix/store/55lwldka5nyxa08wnvlizyqw02ihy8ic-foo".as_bytes()
562                        )
563                        .unwrap()
564                        .to_owned()
565                    ),
566                    e.code
567                );
568            }
569            _ => panic!("unexpected error"),
570        }
571    }
572
573    #[rstest]
574    #[case::simple(
575        br#"("out","/nix/store/5vyvcwah9l9kf07d52rcgdk70g2f4y13-foo","","")"#,
576        (
577            OutputName::out(),
578            StorePathRef::from_absolute_path("/nix/store/5vyvcwah9l9kf07d52rcgdk70g2f4y13-foo".as_bytes()).unwrap().to_owned(),
579            None
580        )
581    )]
582    #[case::fod(
583        br#"("out","/nix/store/4q0pg5zpfmznxscq3avycvf9xdvx50n3-bar","r:sha256","08813cbee9903c62be4c5027726a418a300da4500b2d369d3af9286f4815ceba")"#,
584        (
585            OutputName::out(),
586            StorePathRef::from_absolute_path(
587                "/nix/store/4q0pg5zpfmznxscq3avycvf9xdvx50n3-bar".as_bytes()).unwrap().to_owned(),
588            Some(OutputHash{
589                mode: OutputHashMode::Recursive,
590                hash: NixHash::Sha256(hex!("08813cbee9903c62be4c5027726a418a300da4500b2d369d3af9286f4815ceba")),
591            }),
592        )
593    )]
594    fn parse_output(
595        #[case] input: &[u8],
596        #[case] expected: (OutputName, StorePath, Option<OutputHash>),
597    ) {
598        let (rest, parsed) = super::parse_output(input).expect("must parse");
599        assert!(rest.is_empty());
600        assert_eq!(expected, parsed);
601    }
602
603    #[rstest]
604    #[case::multi_out(
605        br#"[("lib","/nix/store/2vixb94v0hy2xc6p7mbnxxcyc095yyia-has-multi-out-lib","",""),("out","/nix/store/55lwldka5nyxa08wnvlizyqw02ihy8ic-has-multi-out","","")]"#,
606        &EXP_MULTI_OUTPUTS
607    )]
608    fn parse_outputs(#[case] input: &[u8], #[case] expected: &Outputs) {
609        let (rest, parsed) = super::parse_outputs(input).expect("must parse");
610        assert!(rest.is_empty());
611        assert_eq!(*expected, parsed);
612    }
613}
nix_compat/derivation/parser.rs

nix_compat/derivation/
parser.rs
