Skip to main content

snix_castore/directoryservice/
object_store.rs

1use std::collections::HashMap;
2use std::sync::Arc;
3
4use data_encoding::HEXLOWER;
5use futures::SinkExt;
6use futures::StreamExt;
7use futures::TryStreamExt;
8use futures::stream::BoxStream;
9use object_store::ObjectStoreExt;
10use object_store::{ObjectStore, path::Path};
11use prost::Message;
12use tokio::io::AsyncWriteExt;
13use tokio_util::codec::LengthDelimitedCodec;
14use tonic::async_trait;
15use tracing::{Level, instrument, trace, warn};
16
17use super::{Directory, DirectoryPutter, DirectoryService};
18use crate::composition::{CompositionContext, ServiceBuilder};
19use crate::directoryservice::directory_graph::DirectoryGraphBuilder;
20use crate::directoryservice::order_validator::{self, LeavesToRoot, OrderValidator, RootToLeaves};
21use crate::{B3Digest, Node, proto};
22
23/// Stores directory closures in an object store.
24/// Notably, this makes use of the option to disallow accessing child directories except when
25/// fetching them recursively via the top-level directory, since all batched writes
26/// (using `put_multiple_start`) are stored in a single object.
27/// Directories are stored in a length-delimited format with a 1MiB limit. The length field is a
28/// u32 and the directories are stored in root-to-leaves topological order, the same way they will
29/// be returned to the client in get_recursive.
30#[derive(Clone)]
31pub struct ObjectStoreDirectoryService {
32    instance_name: String,
33    object_store: Arc<dyn ObjectStore>,
34    base_path: Path,
35}
36
37#[instrument(level=Level::TRACE, skip_all,fields(base_path=%base_path,blob.digest=%digest),ret(Display))]
38fn derive_dirs_path(base_path: &Path, digest: &B3Digest) -> Path {
39    base_path
40        .clone()
41        .join("dirs")
42        .join("b3")
43        .join(HEXLOWER.encode(&digest.as_slice()[..2]))
44        .join(HEXLOWER.encode(digest.as_slice()))
45}
46
47/// Helper function, parsing protobuf-encoded Directories into [crate::Directory].
48fn parse_proto_directory(encoded_directory: &[u8]) -> Result<crate::Directory, Error> {
49    let directory_proto = proto::Directory::decode(encoded_directory)?;
50
51    Ok(Directory::try_from(directory_proto)?)
52}
53
54#[allow(clippy::identity_op)]
55const MAX_FRAME_LENGTH: usize = 1 * 1024 * 1024 * 1000; // 1 MiB
56
57impl ObjectStoreDirectoryService {
58    pub fn new(instance_name: String, object_store: Arc<dyn ObjectStore>, base_path: Path) -> Self {
59        Self {
60            instance_name,
61            object_store,
62            base_path,
63        }
64    }
65}
66
67#[async_trait]
68impl DirectoryService for ObjectStoreDirectoryService {
69    /// This is the same steps as for get_recursive anyways, so we just call get_recursive and
70    /// return the first element of the stream and drop the request.
71    #[instrument(level = "trace", skip_all, fields(directory.digest = %digest, instance_name = %self.instance_name))]
72    async fn get(&self, digest: &B3Digest) -> Result<Option<Directory>, super::Error> {
73        self.get_recursive(digest).take(1).next().await.transpose()
74    }
75
76    #[instrument(level = "trace", skip_all, fields(directory.digest = %directory.digest(), instance_name = %self.instance_name))]
77    async fn put(&self, directory: Directory) -> Result<B3Digest, super::Error> {
78        // Ensure the directory doesn't contain other directory children
79        if directory
80            .nodes()
81            .any(|(_, e)| matches!(e, Node::Directory { .. }))
82        {
83            Err(Error::PutForDirectoryWithChildren)?
84        }
85
86        let mut handle = self.put_multiple_start();
87        handle.put(directory).await?;
88        handle.close().await
89    }
90
91    #[instrument(level = "trace", skip_all, fields(directory.digest = %root_directory_digest, instance_name = %self.instance_name))]
92    fn get_recursive(
93        &self,
94        root_directory_digest: &B3Digest,
95    ) -> BoxStream<'_, Result<Directory, super::Error>> {
96        // Check that we are not passing on bogus from the object store to the client, and that the
97        // trust chain from the root digest to the leaves is intact.
98        let dir_path = derive_dirs_path(&self.base_path, root_directory_digest);
99        let object_store = &self.object_store;
100        let root_directory_digest = *root_directory_digest;
101
102        async_stream::try_stream! {
103                let bytes_stream = match object_store.get(&dir_path).await {
104                    Ok(v) => v.into_stream(),
105                    Err(object_store::Error::NotFound { .. }) => {
106                        return;
107                    }
108                    Err(e) => Err(Error::ObjectStore(e))?,
109                };
110
111                // get a reader of the response body.
112                let r = tokio_util::io::StreamReader::new(bytes_stream);
113                let decompressed_stream = async_compression::tokio::bufread::ZstdDecoder::new(r);
114
115                // the subdirectories are stored in a length delimited format
116                let mut encoded_directories = LengthDelimitedCodec::builder()
117                    .max_frame_length(MAX_FRAME_LENGTH)
118                    .length_field_type::<u32>()
119                    .new_read(decompressed_stream)
120                    .err_into::<Error>();
121
122                let mut order_validator = RootToLeaves::new_with_root_digest(root_directory_digest);
123                while let Some(encoded_directory) = encoded_directories.try_next().await? {
124                    // hash the encoded proto message, only proceed if we would accept a directory with this digest.
125                    let digest = B3Digest::from(blake3::hash(&encoded_directory));
126                    if !order_validator.would_accept(&digest) {
127                        Err(Error::UnexpectedDigest(digest))?;
128                    }
129
130                    // only then proceed with parsing
131                    let directory = parse_proto_directory(&encoded_directory)?;
132
133                    // The directory can still be rejected for other reasons.
134                    order_validator.try_accept(&directory).map_err(Error::DirectoryOrdering)?;
135
136                    yield directory;
137                }
138
139                order_validator.finalize().map_err(Error::DirectoryOrdering)?;
140        }
141        .boxed()
142    }
143
144    #[instrument(skip_all)]
145    fn put_multiple_start(&self) -> Box<dyn DirectoryPutter + '_>
146    where
147        Self: Clone,
148    {
149        Box::new(ObjectStoreDirectoryPutter::new(
150            self.object_store.clone(),
151            &self.base_path,
152        ))
153    }
154}
155
156#[derive(thiserror::Error, Debug)]
157enum Error {
158    #[error("put() may only be used for directories without children")]
159    PutForDirectoryWithChildren,
160
161    #[error("Directory Graph ordering error")]
162    DirectoryOrdering(#[from] order_validator::OrderingError),
163    #[error("next directory in batch has unexpected digest {0}")]
164    UnexpectedDigest(B3Digest),
165    #[error("failed to decode protobuf: {0}")]
166    ProtobufDecode(#[from] prost::DecodeError),
167    #[error("failed to validate directory: {0}")]
168    DirectoryValidation(#[from] crate::DirectoryError),
169
170    #[error("DirectoryPutter already closed")]
171    DirectoryPutterAlreadyClosed,
172
173    #[error("ObjectStore error: {0}")]
174    ObjectStore(#[from] object_store::Error),
175
176    #[error("io error: {0}")]
177    IO(#[from] std::io::Error),
178}
179impl From<Error> for super::Error {
180    fn from(value: Error) -> Self {
181        Self(Box::new(value))
182    }
183}
184
185#[derive(serde::Deserialize)]
186#[serde(deny_unknown_fields)]
187pub struct ObjectStoreDirectoryServiceConfig {
188    object_store_url: url::Url,
189    #[serde(default)]
190    object_store_options: HashMap<String, String>,
191}
192
193impl TryFrom<url::Url> for ObjectStoreDirectoryServiceConfig {
194    type Error = Box<dyn std::error::Error + Send + Sync>;
195    fn try_from(url: url::Url) -> Result<Self, Self::Error> {
196        let trimmed_url = crate::object_store::trim_objectstore_prefix(&url)?;
197
198        Ok(ObjectStoreDirectoryServiceConfig {
199            object_store_url: trimmed_url,
200            object_store_options: url
201                .query_pairs()
202                .into_iter()
203                .map(|(k, v)| (k.to_string(), v.to_string()))
204                .collect(),
205        })
206    }
207}
208
209#[async_trait]
210impl ServiceBuilder for ObjectStoreDirectoryServiceConfig {
211    type Output = dyn DirectoryService;
212    async fn build<'a>(
213        &'a self,
214        instance_name: &str,
215        _context: &CompositionContext,
216    ) -> Result<Arc<Self::Output>, Box<dyn std::error::Error + Send + Sync>> {
217        let (object_store, path) = crate::object_store::setup_object_store(
218            &self.object_store_url,
219            &self.object_store_options,
220        )
221        .await?;
222
223        Ok(Arc::new(ObjectStoreDirectoryService::new(
224            instance_name.to_string(),
225            Arc::new(object_store),
226            path,
227        )))
228    }
229}
230
231struct ObjectStoreDirectoryPutter<'a> {
232    object_store: Arc<dyn ObjectStore>,
233    base_path: &'a Path,
234
235    builder: Option<DirectoryGraphBuilder<LeavesToRoot>>,
236}
237
238impl<'a> ObjectStoreDirectoryPutter<'a> {
239    fn new(object_store: Arc<dyn ObjectStore>, base_path: &'a Path) -> Self {
240        Self {
241            object_store,
242            base_path,
243            builder: Some(DirectoryGraphBuilder::<LeavesToRoot>::new()),
244        }
245    }
246}
247
248#[async_trait]
249impl DirectoryPutter for ObjectStoreDirectoryPutter<'_> {
250    #[instrument(level = "trace", skip_all, fields(directory.digest=%directory.digest()), err)]
251    async fn put(&mut self, directory: Directory) -> Result<(), super::Error> {
252        let builder = self
253            .builder
254            .as_mut()
255            .ok_or_else(|| Error::DirectoryPutterAlreadyClosed)?;
256
257        builder
258            .try_insert(directory)
259            .map_err(Error::DirectoryOrdering)?;
260
261        Ok(())
262    }
263
264    #[instrument(level = "trace", skip_all, ret, err)]
265    async fn close(&mut self) -> Result<B3Digest, super::Error> {
266        let builder = self
267            .builder
268            .take()
269            .ok_or_else(|| Error::DirectoryPutterAlreadyClosed)?;
270
271        // Retrieve the validated directories.
272        let directory_graph = builder.build().map_err(Error::DirectoryOrdering)?;
273        let root_digest = directory_graph.root().digest();
274
275        let dir_path = derive_dirs_path(self.base_path, &root_digest);
276
277        match self.object_store.head(&dir_path).await {
278            // directory tree already exists, nothing to do
279            Ok(_) => {
280                trace!("directory tree already exists");
281            }
282
283            // directory tree does not yet exist, compress and upload.
284            Err(object_store::Error::NotFound { .. }) => {
285                trace!("uploading directory tree");
286
287                let object_store_writer =
288                    object_store::buffered::BufWriter::new(self.object_store.clone(), dir_path);
289                let compressed_writer =
290                    async_compression::tokio::write::ZstdEncoder::new(object_store_writer);
291                let mut directories_sink = LengthDelimitedCodec::builder()
292                    .max_frame_length(MAX_FRAME_LENGTH)
293                    .length_field_type::<u32>()
294                    .new_write(compressed_writer);
295
296                // Drain the graph in *Root-To-Leaves*, order, as that's how we write it to storage.
297                for directory in directory_graph.drain_root_to_leaves() {
298                    directories_sink
299                        .send(proto::Directory::from(directory).encode_to_vec().into())
300                        .await
301                        .map_err(Error::IO)?;
302                }
303
304                let mut compressed_writer = directories_sink.into_inner();
305                compressed_writer.shutdown().await.map_err(Error::IO)?;
306            }
307            // other error
308            Err(err) => Err(Error::ObjectStore(err))?,
309        }
310
311        Ok(root_digest)
312    }
313}