Skip to main content

snix_castore/directoryservice/
grpc.rs

1use super::{Directory, DirectoryPutter, DirectoryService};
2use crate::B3Digest;
3use crate::composition::{CompositionContext, ServiceBuilder};
4use crate::directoryservice::order_validator::{self, OrderValidator, RootToLeaves};
5use crate::proto::{self, get_directory_request::ByWhat};
6use futures::StreamExt;
7use futures::stream::BoxStream;
8use std::sync::Arc;
9use tokio::spawn;
10use tokio::sync::mpsc::UnboundedSender;
11use tokio::task::JoinHandle;
12use tokio_stream::wrappers::UnboundedReceiverStream;
13use tonic::{Code, Status, async_trait};
14use tracing::{Instrument as _, instrument, warn};
15
16/// Maximum size of proto messages endoded and decoded.
17/// Unfortunately there's directories out there larger than 4MiB, so bump this
18/// by a bit.
19pub const MAX_DECODING_MESSAGE_SIZE: usize = 10 * 1024 * 1024;
20
21/// Connects to a (remote) snix-store DirectoryService over gRPC.
22#[derive(Clone)]
23pub struct GRPCDirectoryService<T> {
24    instance_name: String,
25    /// The internal reference to a gRPC client.
26    /// Cloning it is cheap, and it internally handles concurrent requests.
27    grpc_client: proto::directory_service_client::DirectoryServiceClient<T>,
28}
29
30impl<T> GRPCDirectoryService<T> {
31    /// construct a [GRPCDirectoryService] from a [proto::directory_service_client::DirectoryServiceClient].
32    /// panics if called outside the context of a tokio runtime.
33    pub fn from_client(
34        instance_name: String,
35        grpc_client: proto::directory_service_client::DirectoryServiceClient<T>,
36    ) -> Self {
37        Self {
38            instance_name,
39            grpc_client,
40        }
41    }
42}
43
44#[async_trait]
45impl<T> DirectoryService for GRPCDirectoryService<T>
46where
47    T: tonic::client::GrpcService<tonic::body::Body> + Send + Sync + Clone + 'static,
48    T::ResponseBody: tonic::codegen::Body<Data = tonic::codegen::Bytes> + Send + 'static,
49    <T::ResponseBody as tonic::codegen::Body>::Error: Into<tonic::codegen::StdError> + Send,
50    T::Future: Send,
51{
52    #[instrument(level = "trace", skip_all, fields(directory.digest = %digest, instance_name = %self.instance_name))]
53    async fn get(&self, digest: &B3Digest) -> Result<Option<Directory>, super::Error> {
54        // clone the client, as it takes a &mut.
55        // We set recursive to false, so the stream holds this Directory only.
56        let mut stream = self
57            .grpc_client
58            .clone()
59            .get(proto::GetDirectoryRequest {
60                recursive: false,
61                by_what: Some(ByWhat::Digest((*digest).into())),
62            })
63            .await
64            .map_err(Error::Tonic)?
65            .into_inner();
66        let proto_directory = match stream.message().await {
67            Ok(Some(proto_directory)) => proto_directory,
68            Ok(None) => return Ok(None),
69            Err(e) if e.code() == Code::NotFound => return Ok(None),
70            Err(e) => Err(Error::Tonic(e))?,
71        };
72        // Check that the stream ends after this Directory. Dropping it before
73        // its end resets the HTTP/2 stream, and once too many resets are
74        // pending, the connection fails with all requests on it.
75        if stream.message().await.map_err(Error::Tonic)?.is_some() {
76            Err(Error::UnexpectedMessage)?
77        }
78
79        // Validate the retrieved Directory indeed has the
80        // digest we expect it to have, to detect corruptions.
81        let actual_digest = proto_directory.digest();
82        if &actual_digest != digest {
83            Err(Error::WrongDigest {
84                expected: *digest,
85                actual: actual_digest,
86            })?
87        }
88        let directory = Directory::try_from(proto_directory).map_err(Error::DirectoryValidation)?;
89        Ok(Some(directory))
90    }
91
92    #[instrument(level = "trace", skip_all, fields(directory.digest = %directory.digest(), instance_name = %self.instance_name))]
93    async fn put(&self, directory: Directory) -> Result<B3Digest, super::Error> {
94        let resp = self
95            .grpc_client
96            .clone()
97            .put(tokio_stream::once(proto::Directory::from(directory)))
98            .await
99            .map_err(Error::Tonic)?;
100
101        let digest = resp
102            .into_inner()
103            .root_digest
104            .try_into()
105            .map_err(|_| Error::InvalidDigestLen)?;
106
107        Ok(digest)
108    }
109
110    #[instrument(level = "trace", skip_all, fields(directory.digest = %root_directory_digest, instance_name = %self.instance_name))]
111    fn get_recursive(
112        &self,
113        root_directory_digest: &B3Digest,
114    ) -> BoxStream<'static, Result<Directory, super::Error>> {
115        let mut grpc_client = self.grpc_client.clone();
116        let root_directory_digest = *root_directory_digest;
117
118        let mut order_validator = RootToLeaves::new_with_root_digest(root_directory_digest);
119
120        async_stream::try_stream! {
121            let mut directories = grpc_client
122                .get(proto::GetDirectoryRequest {
123                    recursive: true,
124                    by_what: Some(ByWhat::Digest((root_directory_digest).into())),
125                })
126                .await
127                .map_err(Error::Tonic)?
128                .into_inner();
129
130            while let Some(proto_directory) = directories.message().await.map_err(Error::Tonic)? {
131                let directory = Directory::try_from(proto_directory).map_err(Error::DirectoryValidation)?;
132                order_validator.try_accept(&directory).map_err(Error::DirectoryOrdering)?;
133
134                yield directory;
135            }
136        }
137        .boxed()
138    }
139
140    #[instrument(skip_all)]
141    fn put_multiple_start(&self) -> Box<dyn DirectoryPutter + 'static> {
142        let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
143
144        let task = spawn({
145            let mut grpc_client = self.grpc_client.clone();
146
147            async move {
148                Ok::<_, Status>(
149                    grpc_client
150                        .put(UnboundedReceiverStream::new(rx))
151                        .await?
152                        .into_inner(),
153                )
154            }
155            // instrument the task with the current span, this is not done by default
156            .in_current_span()
157        });
158
159        Box::new(GRPCPutter {
160            rq: Some((task, tx)),
161        })
162    }
163}
164
165/// Allows uploading multiple Directory messages in the same gRPC stream.
166pub struct GRPCPutter {
167    /// Data about the current request - a handle to the task, and the tx part
168    /// of the channel.
169    /// The tx part of the pipe is used to send [proto::Directory] to the ongoing request.
170    /// The task will yield a [proto::PutDirectoryResponse] once the stream is closed.
171    #[allow(clippy::type_complexity)] // lol
172    rq: Option<(
173        JoinHandle<Result<proto::PutDirectoryResponse, Status>>,
174        UnboundedSender<proto::Directory>,
175    )>,
176}
177
178#[async_trait]
179impl DirectoryPutter for GRPCPutter {
180    #[instrument(level = "trace", skip_all, fields(directory.digest=%directory.digest()), err)]
181    async fn put(&mut self, directory: Directory) -> Result<(), super::Error> {
182        let (_, directory_sender) = self
183            .rq
184            .as_ref()
185            .ok_or_else(|| Error::DirectoryPutterAlreadyClosed)?;
186        // If we're not already closed, send the directory to directory_sender.
187        if directory_sender.send(directory.into()).is_err() {
188            // If the channel has been prematurely closed, invoke close (so we can peek at the error code)
189            // That error code is much more helpful, because it
190            // contains the error message from the server.
191            self.close().await?;
192        }
193        Ok(())
194    }
195
196    /// Closes the stream for sending, and returns the value.
197    #[instrument(level = "trace", skip_all, ret, err)]
198    async fn close(&mut self) -> Result<B3Digest, super::Error> {
199        // get self.rq, and replace it with None.
200        // This ensures we can only close it once.
201        let (task, directory_sender) =
202            std::mem::take(&mut self.rq).ok_or_else(|| Error::DirectoryPutterAlreadyClosed)?;
203
204        // close directory_sender, so blocking on task will finish.
205        drop(directory_sender);
206
207        let resp = task
208            .await
209            .map_err(Error::TokioJoin)?
210            .map_err(Error::Tonic)?;
211
212        Ok(B3Digest::try_from(resp.root_digest).map_err(|_| Error::InvalidDigestLen)?)
213    }
214}
215
216#[derive(thiserror::Error, Debug)]
217pub enum Error {
218    #[error("Directory Graph ordering error: {0}")]
219    DirectoryOrdering(#[from] order_validator::OrderingError),
220
221    #[error("DirectoryPutter already closed")]
222    DirectoryPutterAlreadyClosed,
223
224    #[error("requested directory has wrong digest, expected {expected}, actual {actual}")]
225    WrongDigest {
226        expected: B3Digest,
227        actual: B3Digest,
228    },
229
230    #[error("more than one directory returned for a non-recursive get")]
231    UnexpectedMessage,
232
233    #[error("tonic status: {0}")]
234    Tonic(#[from] tonic::Status),
235
236    #[error("invalid digest length returned from put")]
237    InvalidDigestLen,
238
239    #[error("failed to decode protobuf: {0}")]
240    ProtobufDecode(#[from] prost::DecodeError),
241    #[error("failed to validate directory: {0}")]
242    DirectoryValidation(#[from] crate::DirectoryError),
243
244    #[error("join error: {0}")]
245    TokioJoin(#[from] tokio::task::JoinError),
246    #[error("io error: {0}")]
247    IO(#[from] std::io::Error),
248}
249
250impl From<Error> for super::Error {
251    fn from(value: Error) -> Self {
252        Self(Box::new(value))
253    }
254}
255
256#[derive(serde::Deserialize, Debug)]
257#[serde(deny_unknown_fields)]
258pub struct GRPCDirectoryServiceConfig {
259    url: url::Url,
260}
261
262impl TryFrom<url::Url> for GRPCDirectoryServiceConfig {
263    type Error = Box<dyn std::error::Error + Send + Sync>;
264    fn try_from(url: url::Url) -> Result<Self, Self::Error> {
265        Ok(GRPCDirectoryServiceConfig { url })
266    }
267}
268
269#[async_trait]
270impl ServiceBuilder for GRPCDirectoryServiceConfig {
271    type Output = dyn DirectoryService;
272    async fn build<'a>(
273        &'a self,
274        instance_name: &str,
275        _context: &CompositionContext,
276    ) -> Result<Arc<Self::Output>, Box<dyn std::error::Error + Send + Sync>> {
277        let client = proto::directory_service_client::DirectoryServiceClient::with_interceptor(
278            crate::tonic::channel_from_url(&self.url).await?,
279            snix_tracing::propagate::tonic::send_trace,
280        );
281
282        let client = client.max_decoding_message_size(MAX_DECODING_MESSAGE_SIZE);
283
284        Ok(Arc::new(GRPCDirectoryService::from_client(
285            instance_name.to_string(),
286            client,
287        )))
288    }
289}
290#[cfg(test)]
291mod tests {
292    use std::time::Duration;
293    use tempfile::TempDir;
294    use tokio::net::UnixListener;
295    use tokio_retry::{Retry, strategy::ExponentialBackoff};
296    use tokio_stream::wrappers::UnixListenerStream;
297
298    use crate::{
299        directoryservice::{DirectoryService, GRPCDirectoryService},
300        fixtures,
301        proto::{GRPCDirectoryServiceWrapper, directory_service_client::DirectoryServiceClient},
302        utils::gen_test_directory_service,
303    };
304
305    /// This ensures connecting via gRPC works as expected.
306    #[tokio::test]
307    async fn test_valid_unix_path_ping_pong() {
308        let tmpdir = TempDir::new().unwrap();
309        let socket_path = tmpdir.path().join("daemon");
310
311        let path_clone = socket_path.clone();
312
313        // Spin up a server
314        tokio::spawn(async {
315            let uds = UnixListener::bind(path_clone).unwrap();
316            let uds_stream = UnixListenerStream::new(uds);
317
318            // spin up a new server
319            let mut server = tonic::transport::Server::builder();
320            let router = server.add_service(
321                crate::proto::directory_service_server::DirectoryServiceServer::new(
322                    GRPCDirectoryServiceWrapper::new(Box::new(gen_test_directory_service())),
323                ),
324            );
325            router.serve_with_incoming(uds_stream).await
326        });
327
328        // wait for the socket to be created
329        Retry::start(
330            ExponentialBackoff::from_millis(20).max_delay(Duration::from_secs(10)),
331            || async {
332                if socket_path.exists() {
333                    Ok(())
334                } else {
335                    Err(())
336                }
337            },
338        )
339        .await
340        .expect("failed to wait for socket");
341
342        // prepare a client
343        let grpc_client = {
344            let url = url::Url::parse(&format!(
345                "grpc+unix:{}?wait-connect=1",
346                socket_path.display()
347            ))
348            .expect("must parse");
349            let client = DirectoryServiceClient::new(
350                crate::tonic::channel_from_url(&url)
351                    .await
352                    .expect("must succeed"),
353            );
354            GRPCDirectoryService::from_client("test-instance".into(), client)
355        };
356
357        assert!(
358            grpc_client
359                .get(&fixtures::DIRECTORY_A.digest())
360                .await
361                .expect("must not fail")
362                .is_none()
363        )
364    }
365}