snix_build_glue/
known_paths.rs

1//! This module implements logic required for persisting known paths
2//! during an evaluation.
3//!
4//! Snix needs to be able to keep track of each Nix store path that it
5//! knows about during the scope of a single evaluation and its
6//! related builds.
7//!
8//! This data is required to find the derivation needed to actually trigger the
9//! build, if necessary.
10
11use hashbrown::HashMap;
12use nix_compat::{
13    derivation::{Derivation, DerivationError, HashDerivationModuloLookup, UnverifiedDerivation},
14    nixhash::Sha256,
15    store_path::{ParseStorePathError, StorePath, StorePathRef},
16};
17
18use crate::fetchers::Fetch;
19
20/// Struct keeping track of all known Derivations in the current evaluation.
21/// This keeps both the Derivation struct, as well as the "Hash derivation
22/// modulo".
23#[derive(Debug, Default)]
24pub struct KnownPaths {
25    /// All known derivation or FOD hashes.
26    ///
27    /// Keys are derivation paths, values are a tuple of the "hash derivation
28    /// modulo" and the Derivation struct itself.
29    derivations: HashMap<StorePath, (Sha256, Derivation)>,
30
31    /// A map from output path to (one) drv path.
32    /// Note that in the case of FODs, multiple drvs can produce the same output
33    /// path. We use one of them.
34    outputs_to_drvpath: HashMap<StorePath, StorePath>,
35
36    /// A map from output path to fetches (and their names).
37    outputs_to_fetches: HashMap<StorePath, (String, Fetch)>,
38}
39
40impl KnownPaths {
41    /// Fetch the opaque "hash derivation modulo" for a given derivation path.
42    pub fn get_hash_derivation_modulo(&self, drv_path: &StorePathRef) -> Option<&Sha256> {
43        self.derivations
44            .get(drv_path)
45            .map(|(hash_derivation_modulo, _derivation)| hash_derivation_modulo)
46    }
47
48    /// Return a reference to the Derivation for a given drv path.
49    pub fn get_drv_by_drvpath(&self, drv_path: &StorePathRef) -> Option<&Derivation> {
50        self.derivations
51            .get(drv_path)
52            .map(|(_hash_derivation_modulo, derivation)| derivation)
53    }
54
55    /// Return the drv path of the derivation producing the passed output path.
56    /// Note there can be multiple Derivations producing the same output path in
57    /// flight; this function will only return one of them.
58    pub fn get_drv_path_for_output_path(&self, output_path: &StorePathRef) -> Option<&StorePath> {
59        self.outputs_to_drvpath.get(output_path)
60    }
61
62    pub fn add_derivation(
63        &mut self,
64        drv_path: StorePath,
65        drv: UnverifiedDerivation,
66    ) -> Result<(), DerivationError> {
67        let drv_name = drv_path.name().strip_suffix(".drv").ok_or_else(|| {
68            DerivationError::InvalidDerivationName(
69                drv_path.name().to_owned(),
70                ParseStorePathError::Name,
71            )
72        })?;
73        let drv = drv.verify(drv_name, &*self)?;
74
75        self.add_verified_derivation(drv)
76    }
77
78    /// Insert a new [Derivation] into this struct.
79    /// The Derivation struct must pass validation, and its output paths need to
80    /// be fully calculated.
81    /// All input derivations this refers to must also be inserted to this
82    /// struct.
83    pub fn add_verified_derivation(&mut self, drv: Derivation) -> Result<(), DerivationError> {
84        // check input derivations to have been inserted.
85        #[cfg(debug_assertions)]
86        {
87            for input_drv_path in drv.input_derivations().keys() {
88                debug_assert!(self.derivations.contains_key(input_drv_path));
89            }
90        }
91
92        // compute the hash derivation modulo
93        let hash_derivation_modulo = drv.hash_derivation_modulo(&*self)?;
94
95        // For all output paths, update our lookup table.
96        // We only write into the lookup table once.
97        for output_path in drv.outputs().store_paths() {
98            self.outputs_to_drvpath
99                .entry(output_path.clone())
100                .or_insert(drv.drv_path().to_owned());
101        }
102
103        // insert the derivation itself
104        #[allow(unused_variables)] // assertions on this only compiled in debug builds
105        let old = self
106            .derivations
107            .insert(drv.drv_path().to_owned(), (hash_derivation_modulo, drv));
108
109        #[cfg(debug_assertions)]
110        {
111            if let Some(old) = old {
112                debug_assert!(
113                    old.0 == hash_derivation_modulo,
114                    "hash derivation modulo for a given derivation should always be calculated the same"
115                );
116            }
117        }
118        Ok(())
119    }
120
121    /// Insert a new [Fetch] into this struct, which *must* have an expected
122    /// hash (otherwise we wouldn't be able to calculate the store path).
123    /// Fetches without a known hash need to be fetched inside builtins.
124    pub fn add_fetch<'a>(
125        &mut self,
126        fetch: Fetch,
127        name: &'a str,
128    ) -> Result<StorePathRef<'a>, ParseStorePathError> {
129        let store_path = fetch
130            .store_path(name)?
131            .expect("Snix bug: fetch must have an expected hash");
132        // insert the fetch.
133        self.outputs_to_fetches
134            .insert(store_path.to_owned(), (name.to_owned(), fetch));
135
136        Ok(store_path)
137    }
138
139    /// Return the name and fetch producing the passed output path.
140    /// Note there can also be (multiple) Derivations producing the same output path.
141    pub fn get_fetch_for_output_path(
142        &self,
143        output_path: &StorePathRef<'_>,
144    ) -> Option<(String, Fetch)> {
145        self.outputs_to_fetches
146            .get(output_path)
147            .map(|(name, fetch)| (name.to_owned(), fetch.to_owned()))
148    }
149
150    /// Returns an iterator over all known derivations and their store path.
151    pub fn get_derivations<'a>(
152        &'a self,
153    ) -> impl Iterator<Item = (StorePathRef<'a>, &'a Derivation)> {
154        self.derivations.iter().map(|(k, v)| (k.as_ref(), &v.1))
155    }
156}
157
158impl HashDerivationModuloLookup for &KnownPaths {
159    fn lookup_hdm(&self, drv_path: &StorePathRef<'_>) -> Option<Sha256> {
160        self.get_hash_derivation_modulo(drv_path).copied()
161    }
162}
163
164#[cfg(test)]
165mod tests {
166    use std::sync::LazyLock;
167
168    use hex_literal::hex;
169    use nix_compat::{
170        derivation::{Derivation, UnverifiedDerivation},
171        nixbase32,
172        nixhash::{NixHash, Sha256},
173        store_path::StorePath,
174    };
175    use url::Url;
176
177    use super::KnownPaths;
178    use crate::fetchers::Fetch;
179
180    static BAR_DRV: LazyLock<UnverifiedDerivation> = LazyLock::new(|| {
181        UnverifiedDerivation::from_aterm_bytes(include_bytes!(
182            "../test-data/ss2p4wmxijn652haqyd7dckxwl4c7hxx-bar.drv"
183        ))
184        .expect("must parse")
185    });
186
187    static FOO_DRV: LazyLock<UnverifiedDerivation> = LazyLock::new(|| {
188        UnverifiedDerivation::from_aterm_bytes(include_bytes!(
189            "../test-data/ch49594n9avinrf8ip0aslidkc4lxkqv-foo.drv"
190        ))
191        .expect("must parse")
192    });
193
194    static BAR_DRV_PATH: LazyLock<StorePath> = LazyLock::new(|| {
195        StorePath::from_bytes(b"ss2p4wmxijn652haqyd7dckxwl4c7hxx-bar.drv").expect("must parse")
196    });
197
198    static FOO_DRV_PATH: LazyLock<StorePath> = LazyLock::new(|| {
199        StorePath::from_bytes(b"ch49594n9avinrf8ip0aslidkc4lxkqv-foo.drv").expect("must parse")
200    });
201
202    static BAR_OUT_PATH: LazyLock<StorePath> = LazyLock::new(|| {
203        StorePath::from_bytes(b"mp57d33657rf34lzvlbpfa1gjfv5gmpg-bar").expect("must parse")
204    });
205
206    static FOO_OUT_PATH: LazyLock<StorePath> = LazyLock::new(|| {
207        StorePath::from_bytes(b"fhaj6gmwns62s6ypkcldbaj2ybvkhx3p-foo").expect("must parse")
208    });
209
210    static FETCH_URL: LazyLock<Fetch> = LazyLock::new(|| {
211        Fetch::URL {
212        url: Url::parse("https://raw.githubusercontent.com/aaptel/notmuch-extract-patch/f732a53e12a7c91a06755ebfab2007adc9b3063b/notmuch-extract-patch").unwrap(),
213        exp_hash: Some(NixHash::from_sri("sha256-Xa1Jbl2Eq5+L0ww+Ph1osA3Z/Dxe/RkN1/dITQCdXFk=").unwrap())
214    }
215    });
216
217    static FETCH_URL_OUT_PATH: LazyLock<StorePath> = LazyLock::new(|| {
218        StorePath::from_bytes(b"06qi00hylriyfm0nl827crgjvbax84mz-notmuch-extract-patch").unwrap()
219    });
220
221    static FETCH_TARBALL: LazyLock<Fetch> = LazyLock::new(|| {
222        Fetch::Tarball {
223        url: Url::parse("https://github.com/NixOS/nixpkgs/archive/91050ea1e57e50388fa87a3302ba12d188ef723a.tar.gz").unwrap(),
224        exp_nar_sha256: Some(nixbase32::decode_fixed("1hf6cgaci1n186kkkjq106ryf8mmlq9vnwgfwh625wa8hfgdn4dm").unwrap())
225    }
226    });
227
228    static FETCH_TARBALL_OUT_PATH: LazyLock<StorePath> = LazyLock::new(|| {
229        StorePath::from_bytes(b"7adgvk5zdfq4pwrhsm3n9lzypb12gw0g-source").unwrap()
230    });
231
232    /// Ensure that we don't allow adding a derivation that depends on another,
233    /// not-yet-added derivation.
234    #[test]
235    #[should_panic]
236    fn drv_reject_if_missing_input_drv() {
237        let known_paths = KnownPaths::default();
238
239        // FOO_DRV depends on BAR_DRV, which wasn't added.
240        FOO_DRV
241            .clone()
242            .verify("foo", &known_paths)
243            .expect("could not verify");
244    }
245
246    #[test]
247    fn drv_happy_path() {
248        let mut known_paths = KnownPaths::default();
249
250        // get_drv_by_drvpath should return None for non-existing Derivations,
251        // same as get_hash_derivation_modulo and get_drv_path_for_output_path
252        assert_eq!(None, known_paths.get_drv_by_drvpath(&BAR_DRV_PATH.as_ref()));
253        assert_eq!(
254            None,
255            known_paths.get_hash_derivation_modulo(&BAR_DRV_PATH.as_ref())
256        );
257        assert_eq!(
258            None,
259            known_paths.get_drv_path_for_output_path(&BAR_OUT_PATH.as_ref())
260        );
261
262        // Add BAR_DRV
263        known_paths
264            .add_derivation(BAR_DRV_PATH.clone(), BAR_DRV.clone())
265            .expect("must verify BAR_DRV");
266
267        // We should get it back
268        assert_eq!(
269            Some(&BAR_DRV.clone()),
270            known_paths
271                .get_drv_by_drvpath(&BAR_DRV_PATH.as_ref())
272                .map(Derivation::as_unverified)
273        );
274
275        // Test get_drv_path_for_output_path
276        assert_eq!(
277            Some(&BAR_DRV_PATH.clone()),
278            known_paths.get_drv_path_for_output_path(&BAR_OUT_PATH.as_ref())
279        );
280
281        // It should be possible to get the hash derivation modulo.
282        assert_eq!(
283            Some(&Sha256::new(hex!(
284                "c79aebd0ce3269393d4a1fde2cbd1d975d879b40f0bf40a48f550edc107fd5df"
285            ))),
286            known_paths.get_hash_derivation_modulo(&BAR_DRV_PATH.as_ref())
287        );
288
289        // Now insert FOO_DRV too. It shouldn't panic, as BAR_DRV is already
290        // added.
291        known_paths
292            .add_derivation(FOO_DRV_PATH.clone(), FOO_DRV.clone())
293            .expect("must verify FOO_DRV");
294
295        assert_eq!(
296            Some(&FOO_DRV.clone()),
297            known_paths
298                .get_drv_by_drvpath(&FOO_DRV_PATH.as_ref())
299                .map(|v| &**v)
300        );
301        assert_eq!(
302            Some(&Sha256::new(hex!(
303                "af030d36d63d3d7f56a71adaba26b36f5fa1f9847da5eed953ed62e18192762f"
304            ))),
305            known_paths.get_hash_derivation_modulo(&FOO_DRV_PATH.as_ref())
306        );
307
308        // Test get_drv_path_for_output_path
309        assert_eq!(
310            Some(&FOO_DRV_PATH.clone()),
311            known_paths.get_drv_path_for_output_path(&FOO_OUT_PATH.as_ref())
312        );
313    }
314
315    #[test]
316    fn fetch_happy_path() {
317        let mut known_paths = KnownPaths::default();
318
319        // get_fetch_for_output_path should return None for new fetches.
320        assert!(
321            known_paths
322                .get_fetch_for_output_path(&FETCH_TARBALL_OUT_PATH.as_ref())
323                .is_none()
324        );
325
326        // add_fetch should return the properly calculated store paths.
327        assert_eq!(
328            *FETCH_TARBALL_OUT_PATH,
329            known_paths
330                .add_fetch(FETCH_TARBALL.clone(), "source")
331                .unwrap()
332                .to_owned()
333        );
334
335        assert_eq!(
336            *FETCH_URL_OUT_PATH,
337            known_paths
338                .add_fetch(FETCH_URL.clone(), "notmuch-extract-patch")
339                .unwrap()
340                .to_owned()
341        );
342    }
343
344    #[test]
345    fn get_derivations_working() {
346        let mut known_paths = KnownPaths::default();
347
348        // Add BAR_DRV
349        known_paths
350            .add_derivation(BAR_DRV_PATH.clone(), BAR_DRV.clone())
351            .expect("must verify BAR_DRV");
352
353        // We should be able to find BAR_DRV_PATH and BAR_DRV as a pair in get_derivations.
354        assert_eq!(
355            Some((BAR_DRV_PATH.as_ref(), &BAR_DRV.clone())),
356            known_paths
357                .get_derivations()
358                .find(|(s, d)| (s, d.as_unverified()) == (&BAR_DRV_PATH.as_ref(), &*BAR_DRV))
359                .map(|(s, d)| (s, d.as_unverified()))
360        );
361    }
362
363    // TODO: add test panicking about missing digest
364}
snix_build_glue/known_paths.rs

snix_build_glue/
known_paths.rs
