snix_glue/builtins/
fetchers.rs

1//! Contains builtins that fetch paths from the Internet, or local filesystem.
2
3use super::utils::select_string;
4use crate::snix_store_io::SnixStoreIO;
5use nix_compat::nixhash::{HashAlgo, NixHash};
6use nix_compat::store_path::StorePathNameBuf;
7use snix_build_glue::fetchers::FetcherError;
8use snix_eval::builtin_macros::builtins;
9use snix_eval::generators::Gen;
10use snix_eval::generators::GenCo;
11use snix_eval::{CatchableErrorKind, ErrorKind, Value, try_cek};
12use std::{rc::Rc, sync::Arc};
13use url::Url;
14
15// Used as a return type for extract_fetch_args, which is sharing some
16// parsing code between the fetchurl and fetchTarball builtins.
17struct NixFetchArgs {
18    url: Url,
19    name: Option<StorePathNameBuf>,
20    sha256: Option<[u8; 32]>,
21}
22
23// `fetchurl` and `fetchTarball` accept a single argument, which can either be the URL (as string),
24// or an attrset, where `url`, `sha256` and `name` keys are allowed.
25async fn extract_fetch_args(
26    co: &GenCo,
27    args: Value,
28) -> Result<Result<NixFetchArgs, CatchableErrorKind>, ErrorKind> {
29    if let Ok(url_str) = args.to_str() {
30        // Get the raw bytes, not the ToString repr.
31        let url_str =
32            String::from_utf8(url_str.as_bytes().to_vec()).map_err(|_| ErrorKind::Utf8)?;
33
34        // Parse the URL.
35        let url = Url::parse(&url_str).map_err(|e| ErrorKind::SnixError(Arc::from(e)))?;
36
37        return Ok(Ok(NixFetchArgs {
38            url,
39            name: None,
40            sha256: None,
41        }));
42    }
43
44    let attrs = args.to_attrs().map_err(|_| ErrorKind::TypeError {
45        expected: "attribute set or contextless string",
46        actual: args.type_of(),
47    })?;
48
49    // Reject disallowed attrset keys, to match Nix' behaviour.
50    // We complain about the first unexpected key we find in the list.
51    const VALID_KEYS: [&[u8]; 3] = [b"url", b"name", b"sha256"];
52    if let Some(first_invalid_key) = attrs.keys().find(|k| !&VALID_KEYS.contains(&k.as_bytes())) {
53        return Err(ErrorKind::UnexpectedArgumentBuiltin(
54            first_invalid_key.clone(),
55        ));
56    }
57
58    let url_str = try_cek!(select_string(co, &attrs, "url").await?)
59        .ok_or_else(|| ErrorKind::AttributeNotFound { name: "url".into() })?;
60    let name = try_cek!(select_string(co, &attrs, "name").await?);
61    let name = name
62        .map(|n| n.parse::<StorePathNameBuf>())
63        .transpose()
64        .map_err(|e| ErrorKind::SnixError(Arc::from(FetcherError::StorePath(e))))?;
65    let sha256_str = try_cek!(select_string(co, &attrs, "sha256").await?);
66
67    Ok(Ok(NixFetchArgs {
68        url: Url::parse(&url_str).map_err(|e| ErrorKind::SnixError(Arc::from(e)))?,
69        name,
70        // parse the sha256 string into a digest, and bail out if it's not sha256.
71        sha256: sha256_str
72            .map(
73                |sha256_str| match NixHash::from_str(&sha256_str, Some(HashAlgo::Sha256)) {
74                    Ok(NixHash::Sha256(digest)) => Ok(digest),
75                    _ => Err(ErrorKind::InvalidHash(sha256_str)),
76                },
77            )
78            .transpose()?,
79    }))
80}
81
82#[allow(unused_variables)] // for the `state` arg, for now
83#[builtins(state = "Rc<SnixStoreIO>")]
84pub(crate) mod fetcher_builtins {
85    use bstr::ByteSlice;
86    use nix_compat::{flakeref, nixhash::NixHash, store_path::StorePathNameBuf};
87    use snix_build_glue::fetchers::Fetch;
88    use snix_eval::{NixContext, NixString, try_cek_to_value};
89    use std::collections::BTreeMap;
90
91    use super::*;
92
93    /// Attempts to mimic `nix::libutil::baseNameOf`
94    fn url_basename(url: &Url) -> &str {
95        let s = url.path().trim_end_matches('/');
96
97        match s.rsplit_once('/') {
98            None => url.host_str().unwrap_or_default(),
99            Some((_, basename)) => basename,
100        }
101    }
102
103    /// Consumes a fetch.
104    /// If there is enough info to calculate the store path without fetching,
105    /// queue the fetch to be fetched lazily, and return the store path.
106    /// If there's not enough info to calculate it, do the fetch now, and then
107    /// return the store path.
108    /// Note the builtins.typeof of fetchurl and fetchTarball are *not* "path", but "string",
109    /// to stay bug-compatible with Nix.
110    fn fetch_lazy(
111        state: Rc<SnixStoreIO>,
112        name: StorePathNameBuf,
113        fetch: Fetch,
114    ) -> Result<Value, ErrorKind> {
115        let store_path = match fetch.store_path(&name) {
116            Some(store_path) => {
117                // Move the fetch to KnownPaths, so it can be actually fetched later.
118                let sp = state
119                    .build_state
120                    .known_paths
121                    .write()
122                    .add_fetch(fetch, &name);
123
124                debug_assert_eq!(
125                    sp, store_path,
126                    "calculated store path by KnownPaths should match"
127                );
128                sp
129            }
130            None => {
131                // If we don't have enough info, do the fetch now.
132                let (store_path, _path_info) = state
133                    .tokio_handle
134                    .block_on(async {
135                        state
136                            .build_state
137                            .fetcher
138                            .ingest_and_persist(&name, fetch)
139                            .await
140                    })
141                    .map_err(|e| ErrorKind::SnixError(Arc::from(e)))?;
142
143                store_path
144            }
145        };
146
147        let s = store_path.to_absolute_path();
148
149        // Emit the calculated Store Path, which needs to have context.
150        let context = NixContext::new().append(snix_eval::NixContextElement::Plain(s.clone()));
151        Ok(Value::String(NixString::new_context_from(context, s)))
152    }
153
154    #[builtin("fetchurl")]
155    async fn builtin_fetchurl(
156        state: Rc<SnixStoreIO>,
157        co: GenCo,
158        args: Value,
159    ) -> Result<Value, ErrorKind> {
160        let args = try_cek_to_value!(extract_fetch_args(&co, args).await?);
161
162        // Derive the name from the URL basename if not set explicitly.
163        let name = args
164            .name
165            .map_or_else(|| url_basename(&args.url).parse::<StorePathNameBuf>(), Ok)
166            .map_err(|e| ErrorKind::SnixError(Arc::from(FetcherError::StorePath(e))))?;
167
168        fetch_lazy(
169            state,
170            name,
171            Fetch::URL {
172                url: args.url,
173                exp_hash: args.sha256.map(NixHash::Sha256),
174            },
175        )
176    }
177
178    #[builtin("fetchTarball")]
179    async fn builtin_fetch_tarball(
180        state: Rc<SnixStoreIO>,
181        co: GenCo,
182        args: Value,
183    ) -> Result<Value, ErrorKind> {
184        let args = try_cek_to_value!(extract_fetch_args(&co, args).await?);
185
186        // Name defaults to "source" if not set explicitly.
187        let name = args.name.unwrap_or_default();
188
189        fetch_lazy(
190            state,
191            name,
192            Fetch::Tarball {
193                url: args.url,
194                exp_nar_sha256: args.sha256,
195            },
196        )
197    }
198
199    #[builtin("fetchGit")]
200    async fn builtin_fetch_git(
201        state: Rc<SnixStoreIO>,
202        co: GenCo,
203        args: Value,
204    ) -> Result<Value, ErrorKind> {
205        Err(ErrorKind::NotImplemented("fetchGit"))
206    }
207
208    // FUTUREWORK: make it a feature flag once #64 is implemented
209    #[builtin("parseFlakeRef")]
210    async fn builtin_parse_flake_ref(
211        state: Rc<SnixStoreIO>,
212        co: GenCo,
213        value: Value,
214    ) -> Result<Value, ErrorKind> {
215        let flake_ref = value.to_str()?;
216        let flake_ref_str = flake_ref.to_str()?;
217
218        let fetch_args = flake_ref_str
219            .parse()
220            .map_err(|err| ErrorKind::SnixError(Arc::new(err)))?;
221
222        // Convert the FlakeRef to our Value format
223        let mut attrs = BTreeMap::new();
224
225        // Extract type and url based on the variant
226        match fetch_args {
227            flakeref::FlakeRef::Git { url, .. } => {
228                attrs.insert("type".into(), Value::from("git"));
229                attrs.insert("url".into(), Value::from(url.to_string()));
230            }
231            flakeref::FlakeRef::GitHub {
232                owner, repo, r#ref, ..
233            } => {
234                attrs.insert("type".into(), Value::from("github"));
235                attrs.insert("owner".into(), Value::from(owner));
236                attrs.insert("repo".into(), Value::from(repo));
237                if let Some(ref_name) = r#ref {
238                    attrs.insert("ref".into(), Value::from(ref_name));
239                }
240            }
241            flakeref::FlakeRef::GitLab { owner, repo, .. } => {
242                attrs.insert("type".into(), Value::from("gitlab"));
243                attrs.insert("owner".into(), Value::from(owner));
244                attrs.insert("repo".into(), Value::from(repo));
245            }
246            flakeref::FlakeRef::File { url, .. } => {
247                attrs.insert("type".into(), Value::from("file"));
248                attrs.insert("url".into(), Value::from(url.to_string()));
249            }
250            flakeref::FlakeRef::Tarball { url, .. } => {
251                attrs.insert("type".into(), Value::from("tarball"));
252                attrs.insert("url".into(), Value::from(url.to_string()));
253            }
254            flakeref::FlakeRef::Path { path, .. } => {
255                attrs.insert("type".into(), Value::from("path"));
256                attrs.insert(
257                    "path".into(),
258                    Value::from(path.to_string_lossy().into_owned()),
259                );
260            }
261            _ => {
262                // For all other ref types, return a simple type/url attributes
263                attrs.insert("type".into(), Value::from("indirect"));
264                attrs.insert("url".into(), Value::from(flake_ref_str));
265            }
266        }
267
268        Ok(Value::Attrs(attrs.into()))
269    }
270
271    #[cfg(test)]
272    mod tests {
273        mod url_basename {
274            use super::super::*;
275            use rstest::rstest;
276
277            #[rstest]
278            #[case::empty_path("", "localhost")]
279            #[case::path_on_root("/dir", "dir")]
280            #[case::relative_path("dir/foo", "foo")]
281            #[case::root_with_trailing_slash("/", "localhost")]
282            #[case::root_with_many_trailing_slashes("///", "localhost")]
283            #[case::trailing_slash("/dir/", "dir")]
284            #[case::many_trailing_slashes("/dir//", "dir")]
285            fn test_url_basename(#[case] url_path: &str, #[case] exp_basename: &str) {
286                let mut url = Url::parse("http://localhost").expect("invalid url");
287                url.set_path(url_path);
288                assert_eq!(url_basename(&url), exp_basename);
289            }
290        }
291    }
292}
snix_glue/builtins/fetchers.rs

snix_glue/builtins/
fetchers.rs
