snix_glue/builtins/
fetchers.rs

1//! Contains builtins that fetch paths from the Internet, or local filesystem.
2
3use super::utils::select_string;
4use crate::snix_store_io::SnixStoreIO;
5use nix_compat::nixhash::{HashAlgo, NixHash};
6use snix_eval::builtin_macros::builtins;
7use snix_eval::generators::Gen;
8use snix_eval::generators::GenCo;
9use snix_eval::{CatchableErrorKind, ErrorKind, Value, try_cek};
10use std::{rc::Rc, sync::Arc};
11use url::Url;
12
13// Used as a return type for extract_fetch_args, which is sharing some
14// parsing code between the fetchurl and fetchTarball builtins.
15struct NixFetchArgs {
16    url: Url,
17    name: Option<String>,
18    sha256: Option<[u8; 32]>,
19}
20
21// `fetchurl` and `fetchTarball` accept a single argument, which can either be the URL (as string),
22// or an attrset, where `url`, `sha256` and `name` keys are allowed.
23async fn extract_fetch_args(
24    co: &GenCo,
25    args: Value,
26) -> Result<Result<NixFetchArgs, CatchableErrorKind>, ErrorKind> {
27    if let Ok(url_str) = args.to_str() {
28        // Get the raw bytes, not the ToString repr.
29        let url_str =
30            String::from_utf8(url_str.as_bytes().to_vec()).map_err(|_| ErrorKind::Utf8)?;
31
32        // Parse the URL.
33        let url = Url::parse(&url_str).map_err(|e| ErrorKind::SnixError(Arc::from(e)))?;
34
35        return Ok(Ok(NixFetchArgs {
36            url,
37            name: None,
38            sha256: None,
39        }));
40    }
41
42    let attrs = args.to_attrs().map_err(|_| ErrorKind::TypeError {
43        expected: "attribute set or contextless string",
44        actual: args.type_of(),
45    })?;
46
47    // Reject disallowed attrset keys, to match Nix' behaviour.
48    // We complain about the first unexpected key we find in the list.
49    const VALID_KEYS: [&[u8]; 3] = [b"url", b"name", b"sha256"];
50    if let Some(first_invalid_key) = attrs.keys().find(|k| !&VALID_KEYS.contains(&k.as_bytes())) {
51        return Err(ErrorKind::UnexpectedArgumentBuiltin(
52            first_invalid_key.clone(),
53        ));
54    }
55
56    let url_str = try_cek!(select_string(co, &attrs, "url").await?)
57        .ok_or_else(|| ErrorKind::AttributeNotFound { name: "url".into() })?;
58    let name = try_cek!(select_string(co, &attrs, "name").await?);
59    let sha256_str = try_cek!(select_string(co, &attrs, "sha256").await?);
60
61    Ok(Ok(NixFetchArgs {
62        url: Url::parse(&url_str).map_err(|e| ErrorKind::SnixError(Arc::from(e)))?,
63        name,
64        // parse the sha256 string into a digest, and bail out if it's not sha256.
65        sha256: sha256_str
66            .map(
67                |sha256_str| match NixHash::from_str(&sha256_str, Some(HashAlgo::Sha256)) {
68                    Ok(NixHash::Sha256(digest)) => Ok(digest),
69                    _ => Err(ErrorKind::InvalidHash(sha256_str)),
70                },
71            )
72            .transpose()?,
73    }))
74}
75
76#[allow(unused_variables)] // for the `state` arg, for now
77#[builtins(state = "Rc<SnixStoreIO>")]
78pub(crate) mod fetcher_builtins {
79    use bstr::ByteSlice;
80    use nix_compat::{flakeref, nixhash::NixHash};
81    use snix_build_glue::fetchers::Fetch;
82    use snix_eval::{NixContext, NixString, try_cek_to_value};
83    use std::collections::BTreeMap;
84
85    use super::*;
86
87    /// Attempts to mimic `nix::libutil::baseNameOf`
88    fn url_basename(url: &Url) -> &str {
89        let s = url.path().trim_end_matches('/');
90
91        match s.rsplit_once('/') {
92            None => url.host_str().unwrap_or_default(),
93            Some((_, basename)) => basename,
94        }
95    }
96
97    /// Consumes a fetch.
98    /// If there is enough info to calculate the store path without fetching,
99    /// queue the fetch to be fetched lazily, and return the store path.
100    /// If there's not enough info to calculate it, do the fetch now, and then
101    /// return the store path.
102    /// Note the builtins.typeof of fetchurl and fetchTarball are *not* "path", but "string",
103    /// to stay bug-compatible with Nix.
104    fn fetch_lazy(state: Rc<SnixStoreIO>, name: String, fetch: Fetch) -> Result<Value, ErrorKind> {
105        let store_path = match fetch
106            .store_path(&name)
107            .map_err(|e| ErrorKind::SnixError(Arc::from(e)))?
108        {
109            Some(store_path) => {
110                // Move the fetch to KnownPaths, so it can be actually fetched later.
111                let sp = state
112                    .build_state
113                    .known_paths
114                    .borrow_mut()
115                    .add_fetch(fetch, &name)
116                    .expect("Snix bug: should only fail if the store path cannot be calculated");
117
118                debug_assert_eq!(
119                    sp, store_path,
120                    "calculated store path by KnownPaths should match"
121                );
122                sp
123            }
124            None => {
125                // If we don't have enough info, do the fetch now.
126                let (store_path, _path_info) = state
127                    .tokio_handle
128                    .block_on(async {
129                        state
130                            .build_state
131                            .fetcher
132                            .ingest_and_persist(&name, fetch)
133                            .await
134                    })
135                    .map_err(|e| ErrorKind::SnixError(Arc::from(e)))?;
136
137                store_path
138            }
139        };
140
141        let s = store_path.to_absolute_path();
142
143        // Emit the calculated Store Path, which needs to have context.
144        let context = NixContext::new().append(snix_eval::NixContextElement::Plain(s.clone()));
145        Ok(Value::String(NixString::new_context_from(context, s)))
146    }
147
148    #[builtin("fetchurl")]
149    async fn builtin_fetchurl(
150        state: Rc<SnixStoreIO>,
151        co: GenCo,
152        args: Value,
153    ) -> Result<Value, ErrorKind> {
154        let args = try_cek_to_value!(extract_fetch_args(&co, args).await?);
155
156        // Derive the name from the URL basename if not set explicitly.
157        let name = args
158            .name
159            .unwrap_or_else(|| url_basename(&args.url).to_owned());
160
161        fetch_lazy(
162            state,
163            name,
164            Fetch::URL {
165                url: args.url,
166                exp_hash: args.sha256.map(NixHash::Sha256),
167            },
168        )
169    }
170
171    #[builtin("fetchTarball")]
172    async fn builtin_fetch_tarball(
173        state: Rc<SnixStoreIO>,
174        co: GenCo,
175        args: Value,
176    ) -> Result<Value, ErrorKind> {
177        let args = try_cek_to_value!(extract_fetch_args(&co, args).await?);
178
179        // Name defaults to "source" if not set explicitly.
180        const DEFAULT_NAME_FETCH_TARBALL: &str = "source";
181        let name = args
182            .name
183            .unwrap_or_else(|| DEFAULT_NAME_FETCH_TARBALL.to_owned());
184
185        fetch_lazy(
186            state,
187            name,
188            Fetch::Tarball {
189                url: args.url,
190                exp_nar_sha256: args.sha256,
191            },
192        )
193    }
194
195    #[builtin("fetchGit")]
196    async fn builtin_fetch_git(
197        state: Rc<SnixStoreIO>,
198        co: GenCo,
199        args: Value,
200    ) -> Result<Value, ErrorKind> {
201        Err(ErrorKind::NotImplemented("fetchGit"))
202    }
203
204    // FUTUREWORK: make it a feature flag once #64 is implemented
205    #[builtin("parseFlakeRef")]
206    async fn builtin_parse_flake_ref(
207        state: Rc<SnixStoreIO>,
208        co: GenCo,
209        value: Value,
210    ) -> Result<Value, ErrorKind> {
211        let flake_ref = value.to_str()?;
212        let flake_ref_str = flake_ref.to_str()?;
213
214        let fetch_args = flake_ref_str
215            .parse()
216            .map_err(|err| ErrorKind::SnixError(Arc::new(err)))?;
217
218        // Convert the FlakeRef to our Value format
219        let mut attrs = BTreeMap::new();
220
221        // Extract type and url based on the variant
222        match fetch_args {
223            flakeref::FlakeRef::Git { url, .. } => {
224                attrs.insert("type".into(), Value::from("git"));
225                attrs.insert("url".into(), Value::from(url.to_string()));
226            }
227            flakeref::FlakeRef::GitHub {
228                owner, repo, r#ref, ..
229            } => {
230                attrs.insert("type".into(), Value::from("github"));
231                attrs.insert("owner".into(), Value::from(owner));
232                attrs.insert("repo".into(), Value::from(repo));
233                if let Some(ref_name) = r#ref {
234                    attrs.insert("ref".into(), Value::from(ref_name));
235                }
236            }
237            flakeref::FlakeRef::GitLab { owner, repo, .. } => {
238                attrs.insert("type".into(), Value::from("gitlab"));
239                attrs.insert("owner".into(), Value::from(owner));
240                attrs.insert("repo".into(), Value::from(repo));
241            }
242            flakeref::FlakeRef::File { url, .. } => {
243                attrs.insert("type".into(), Value::from("file"));
244                attrs.insert("url".into(), Value::from(url.to_string()));
245            }
246            flakeref::FlakeRef::Tarball { url, .. } => {
247                attrs.insert("type".into(), Value::from("tarball"));
248                attrs.insert("url".into(), Value::from(url.to_string()));
249            }
250            flakeref::FlakeRef::Path { path, .. } => {
251                attrs.insert("type".into(), Value::from("path"));
252                attrs.insert(
253                    "path".into(),
254                    Value::from(path.to_string_lossy().into_owned()),
255                );
256            }
257            _ => {
258                // For all other ref types, return a simple type/url attributes
259                attrs.insert("type".into(), Value::from("indirect"));
260                attrs.insert("url".into(), Value::from(flake_ref_str));
261            }
262        }
263
264        Ok(Value::Attrs(attrs.into()))
265    }
266
267    #[cfg(test)]
268    mod tests {
269        mod url_basename {
270            use super::super::*;
271            use rstest::rstest;
272
273            #[rstest]
274            #[case::empty_path("", "localhost")]
275            #[case::path_on_root("/dir", "dir")]
276            #[case::relative_path("dir/foo", "foo")]
277            #[case::root_with_trailing_slash("/", "localhost")]
278            #[case::root_with_many_trailing_slashes("///", "localhost")]
279            #[case::trailing_slash("/dir/", "dir")]
280            #[case::many_trailing_slashes("/dir//", "dir")]
281            fn test_url_basename(#[case] url_path: &str, #[case] exp_basename: &str) {
282                let mut url = Url::parse("http://localhost").expect("invalid url");
283                url.set_path(url_path);
284                assert_eq!(url_basename(&url), exp_basename);
285            }
286        }
287    }
288}
snix_glue/builtins/fetchers.rs

snix_glue/builtins/
fetchers.rs
