snix_castore/directoryservice/
object_store.rs1use std::collections::HashMap;
2use std::sync::Arc;
3
4use data_encoding::HEXLOWER;
5use futures::SinkExt;
6use futures::StreamExt;
7use futures::TryStreamExt;
8use futures::stream::BoxStream;
9use object_store::ObjectStoreExt;
10use object_store::{ObjectStore, path::Path};
11use prost::Message;
12use tokio::io::AsyncWriteExt;
13use tokio_util::codec::LengthDelimitedCodec;
14use tonic::async_trait;
15use tracing::{Level, instrument, trace, warn};
16
17use super::{Directory, DirectoryPutter, DirectoryService};
18use crate::composition::{CompositionContext, ServiceBuilder};
19use crate::directoryservice::directory_graph::DirectoryGraphBuilder;
20use crate::directoryservice::order_validator::{self, LeavesToRoot, OrderValidator, RootToLeaves};
21use crate::{B3Digest, Node, proto};
22
23#[derive(Clone)]
31pub struct ObjectStoreDirectoryService {
32 instance_name: String,
33 object_store: Arc<dyn ObjectStore>,
34 base_path: Path,
35}
36
37#[instrument(level=Level::TRACE, skip_all,fields(base_path=%base_path,blob.digest=%digest),ret(Display))]
38fn derive_dirs_path(base_path: &Path, digest: &B3Digest) -> Path {
39 base_path
40 .clone()
41 .join("dirs")
42 .join("b3")
43 .join(HEXLOWER.encode(&digest.as_slice()[..2]))
44 .join(HEXLOWER.encode(digest.as_slice()))
45}
46
47fn parse_proto_directory(encoded_directory: &[u8]) -> Result<crate::Directory, Error> {
49 let directory_proto = proto::Directory::decode(encoded_directory)?;
50
51 Ok(Directory::try_from(directory_proto)?)
52}
53
54#[allow(clippy::identity_op)]
55const MAX_FRAME_LENGTH: usize = 1 * 1024 * 1024 * 1000; impl ObjectStoreDirectoryService {
58 pub fn new(instance_name: String, object_store: Arc<dyn ObjectStore>, base_path: Path) -> Self {
59 Self {
60 instance_name,
61 object_store,
62 base_path,
63 }
64 }
65}
66
67#[async_trait]
68impl DirectoryService for ObjectStoreDirectoryService {
69 #[instrument(level = "trace", skip_all, fields(directory.digest = %digest, instance_name = %self.instance_name))]
72 async fn get(&self, digest: &B3Digest) -> Result<Option<Directory>, super::Error> {
73 self.get_recursive(digest).take(1).next().await.transpose()
74 }
75
76 #[instrument(level = "trace", skip_all, fields(directory.digest = %directory.digest(), instance_name = %self.instance_name))]
77 async fn put(&self, directory: Directory) -> Result<B3Digest, super::Error> {
78 if directory
80 .nodes()
81 .any(|(_, e)| matches!(e, Node::Directory { .. }))
82 {
83 Err(Error::PutForDirectoryWithChildren)?
84 }
85
86 let mut handle = self.put_multiple_start();
87 handle.put(directory).await?;
88 handle.close().await
89 }
90
91 #[instrument(level = "trace", skip_all, fields(directory.digest = %root_directory_digest, instance_name = %self.instance_name))]
92 fn get_recursive(
93 &self,
94 root_directory_digest: &B3Digest,
95 ) -> BoxStream<'_, Result<Directory, super::Error>> {
96 let dir_path = derive_dirs_path(&self.base_path, root_directory_digest);
99 let object_store = &self.object_store;
100 let root_directory_digest = *root_directory_digest;
101
102 async_stream::try_stream! {
103 let bytes_stream = match object_store.get(&dir_path).await {
104 Ok(v) => v.into_stream(),
105 Err(object_store::Error::NotFound { .. }) => {
106 return;
107 }
108 Err(e) => Err(Error::ObjectStore(e))?,
109 };
110
111 let r = tokio_util::io::StreamReader::new(bytes_stream);
113 let decompressed_stream = async_compression::tokio::bufread::ZstdDecoder::new(r);
114
115 let mut encoded_directories = LengthDelimitedCodec::builder()
117 .max_frame_length(MAX_FRAME_LENGTH)
118 .length_field_type::<u32>()
119 .new_read(decompressed_stream)
120 .err_into::<Error>();
121
122 let mut order_validator = RootToLeaves::new_with_root_digest(root_directory_digest);
123 while let Some(encoded_directory) = encoded_directories.try_next().await? {
124 let digest = B3Digest::from(blake3::hash(&encoded_directory));
126 if !order_validator.would_accept(&digest) {
127 Err(Error::UnexpectedDigest(digest))?;
128 }
129
130 let directory = parse_proto_directory(&encoded_directory)?;
132
133 order_validator.try_accept(&directory).map_err(Error::DirectoryOrdering)?;
135
136 yield directory;
137 }
138
139 order_validator.finalize().map_err(Error::DirectoryOrdering)?;
140 }
141 .boxed()
142 }
143
144 #[instrument(skip_all)]
145 fn put_multiple_start(&self) -> Box<dyn DirectoryPutter + '_>
146 where
147 Self: Clone,
148 {
149 Box::new(ObjectStoreDirectoryPutter::new(
150 self.object_store.clone(),
151 &self.base_path,
152 ))
153 }
154}
155
156#[derive(thiserror::Error, Debug)]
157enum Error {
158 #[error("put() may only be used for directories without children")]
159 PutForDirectoryWithChildren,
160
161 #[error("Directory Graph ordering error")]
162 DirectoryOrdering(#[from] order_validator::OrderingError),
163 #[error("next directory in batch has unexpected digest {0}")]
164 UnexpectedDigest(B3Digest),
165 #[error("failed to decode protobuf: {0}")]
166 ProtobufDecode(#[from] prost::DecodeError),
167 #[error("failed to validate directory: {0}")]
168 DirectoryValidation(#[from] crate::DirectoryError),
169
170 #[error("DirectoryPutter already closed")]
171 DirectoryPutterAlreadyClosed,
172
173 #[error("ObjectStore error: {0}")]
174 ObjectStore(#[from] object_store::Error),
175
176 #[error("io error: {0}")]
177 IO(#[from] std::io::Error),
178}
179impl From<Error> for super::Error {
180 fn from(value: Error) -> Self {
181 Self(Box::new(value))
182 }
183}
184
185#[derive(serde::Deserialize)]
186#[serde(deny_unknown_fields)]
187pub struct ObjectStoreDirectoryServiceConfig {
188 object_store_url: url::Url,
189 #[serde(default)]
190 object_store_options: HashMap<String, String>,
191}
192
193impl TryFrom<url::Url> for ObjectStoreDirectoryServiceConfig {
194 type Error = Box<dyn std::error::Error + Send + Sync>;
195 fn try_from(url: url::Url) -> Result<Self, Self::Error> {
196 let trimmed_url = crate::object_store::trim_objectstore_prefix(&url)?;
197
198 Ok(ObjectStoreDirectoryServiceConfig {
199 object_store_url: trimmed_url,
200 object_store_options: url
201 .query_pairs()
202 .into_iter()
203 .map(|(k, v)| (k.to_string(), v.to_string()))
204 .collect(),
205 })
206 }
207}
208
209#[async_trait]
210impl ServiceBuilder for ObjectStoreDirectoryServiceConfig {
211 type Output = dyn DirectoryService;
212 async fn build<'a>(
213 &'a self,
214 instance_name: &str,
215 _context: &CompositionContext,
216 ) -> Result<Arc<Self::Output>, Box<dyn std::error::Error + Send + Sync>> {
217 let (object_store, path) = crate::object_store::setup_object_store(
218 &self.object_store_url,
219 &self.object_store_options,
220 )
221 .await?;
222
223 Ok(Arc::new(ObjectStoreDirectoryService::new(
224 instance_name.to_string(),
225 Arc::new(object_store),
226 path,
227 )))
228 }
229}
230
231struct ObjectStoreDirectoryPutter<'a> {
232 object_store: Arc<dyn ObjectStore>,
233 base_path: &'a Path,
234
235 builder: Option<DirectoryGraphBuilder<LeavesToRoot>>,
236}
237
238impl<'a> ObjectStoreDirectoryPutter<'a> {
239 fn new(object_store: Arc<dyn ObjectStore>, base_path: &'a Path) -> Self {
240 Self {
241 object_store,
242 base_path,
243 builder: Some(DirectoryGraphBuilder::<LeavesToRoot>::new()),
244 }
245 }
246}
247
248#[async_trait]
249impl DirectoryPutter for ObjectStoreDirectoryPutter<'_> {
250 #[instrument(level = "trace", skip_all, fields(directory.digest=%directory.digest()), err)]
251 async fn put(&mut self, directory: Directory) -> Result<(), super::Error> {
252 let builder = self
253 .builder
254 .as_mut()
255 .ok_or_else(|| Error::DirectoryPutterAlreadyClosed)?;
256
257 builder
258 .try_insert(directory)
259 .map_err(Error::DirectoryOrdering)?;
260
261 Ok(())
262 }
263
264 #[instrument(level = "trace", skip_all, ret, err)]
265 async fn close(&mut self) -> Result<B3Digest, super::Error> {
266 let builder = self
267 .builder
268 .take()
269 .ok_or_else(|| Error::DirectoryPutterAlreadyClosed)?;
270
271 let directory_graph = builder.build().map_err(Error::DirectoryOrdering)?;
273 let root_digest = directory_graph.root().digest();
274
275 let dir_path = derive_dirs_path(self.base_path, &root_digest);
276
277 match self.object_store.head(&dir_path).await {
278 Ok(_) => {
280 trace!("directory tree already exists");
281 }
282
283 Err(object_store::Error::NotFound { .. }) => {
285 trace!("uploading directory tree");
286
287 let object_store_writer =
288 object_store::buffered::BufWriter::new(self.object_store.clone(), dir_path);
289 let compressed_writer =
290 async_compression::tokio::write::ZstdEncoder::new(object_store_writer);
291 let mut directories_sink = LengthDelimitedCodec::builder()
292 .max_frame_length(MAX_FRAME_LENGTH)
293 .length_field_type::<u32>()
294 .new_write(compressed_writer);
295
296 for directory in directory_graph.drain_root_to_leaves() {
298 directories_sink
299 .send(proto::Directory::from(directory).encode_to_vec().into())
300 .await
301 .map_err(Error::IO)?;
302 }
303
304 let mut compressed_writer = directories_sink.into_inner();
305 compressed_writer.shutdown().await.map_err(Error::IO)?;
306 }
307 Err(err) => Err(Error::ObjectStore(err))?,
309 }
310
311 Ok(root_digest)
312 }
313}