snix_castore/
utils.rs

1use std::collections::HashMap;
2use std::sync::Arc;
3use url::Url;
4
5use crate::blob_engine::{self, BlobEngine};
6use crate::blobstore::BlobStore;
7use crate::chunkstore::ChunkStore;
8use crate::composition::{
9    Composition, DeserializeWithRegistry, REG, ServiceBuilder, with_registry,
10};
11use crate::directoryservice::DirectoryService;
12
13#[derive(serde::Deserialize, Default)]
14pub struct CompositionConfigs {
15    pub blob_stores:
16        HashMap<String, DeserializeWithRegistry<Box<dyn ServiceBuilder<Output = dyn BlobStore>>>>,
17    pub chunk_stores:
18        HashMap<String, DeserializeWithRegistry<Box<dyn ServiceBuilder<Output = dyn ChunkStore>>>>,
19    pub directoryservices: HashMap<
20        String,
21        DeserializeWithRegistry<Box<dyn ServiceBuilder<Output = dyn DirectoryService>>>,
22    >,
23}
24
25/// Provides a set of clap arguments to configure snix-castore services.
26///
27/// This particular variant has defaults tailored for usecases accessing data
28/// directly locally, like the `snix store daemon` command.
29#[derive(clap::Parser, Clone)]
30#[group(id = "CastoreServiceUrls")]
31pub struct ServiceUrls {
32    #[arg(
33        long,
34        env,
35        default_value = "objectstore+file:/var/lib/snix-castore/blobs"
36    )]
37    pub blob_store_addr: Url,
38
39    #[arg(
40        long,
41        env,
42        default_value = "objectstore+file:/var/lib/snix-castore/blobs"
43    )]
44    pub chunk_store_addr: Url,
45
46    #[arg(
47        long,
48        env,
49        default_value = "redb:/var/lib/snix-castore/directories.redb"
50    )]
51    pub directory_service_addr: Url,
52
53    /// Path to a TOML file describing the way the services should be composed
54    /// Experimental because the format is not final.
55    /// If specified, the other service addrs are ignored.
56    #[cfg(feature = "xp-composition-cli")]
57    #[arg(long, env)]
58    pub experimental_store_composition: Option<String>,
59}
60
61/// Provides a set of clap arguments to configure snix-castore services.
62///
63/// This particular variant has defaults tailored for usecases accessing data
64/// from another running snix daemon, via gRPC.
65#[derive(clap::Parser, Clone)]
66#[group(id = "CastoreServiceUrlsGrpc")]
67pub struct ServiceUrlsGrpc {
68    #[arg(long, env, default_value = "grpc+http://[::1]:8000")]
69    blob_store_addr: Url,
70
71    #[arg(long, env, default_value = "grpc+http://[::1]:8000")]
72    chunk_store_addr: Url,
73
74    #[arg(long, env, default_value = "grpc+http://[::1]:8000")]
75    directory_service_addr: Url,
76
77    #[cfg(feature = "xp-composition-cli")]
78    #[arg(long, env)]
79    experimental_store_composition: Option<String>,
80}
81
82/// Provides a set of clap arguments to configure snix-castore services.
83///
84/// This particular variant has defaults tailored for usecases keeping all data
85/// in memory.
86/// It's currently used in snix-cli, as we don't really care about persistency
87/// there yet, and using something else here might make some perf output harder
88/// to interpret.
89#[derive(clap::Parser, Clone)]
90#[group(id = "CastoreServiceUrlsMemory")]
91pub struct ServiceUrlsMemory {
92    #[arg(long, env, default_value = "memory:")]
93    blob_store_addr: Url,
94
95    #[arg(long, env, default_value = "memory:")]
96    chunk_store_addr: Url,
97
98    #[arg(long, env, default_value = "redb+memory:")]
99    directory_service_addr: Url,
100
101    #[cfg(feature = "xp-composition-cli")]
102    #[arg(long, env)]
103    experimental_store_composition: Option<String>,
104}
105
106impl From<ServiceUrlsGrpc> for ServiceUrls {
107    fn from(
108        ServiceUrlsGrpc {
109            blob_store_addr,
110            chunk_store_addr,
111            directory_service_addr,
112            #[cfg(feature = "xp-composition-cli")]
113            experimental_store_composition,
114        }: ServiceUrlsGrpc,
115    ) -> ServiceUrls {
116        ServiceUrls {
117            blob_store_addr,
118            chunk_store_addr,
119            directory_service_addr,
120            #[cfg(feature = "xp-composition-cli")]
121            experimental_store_composition,
122        }
123    }
124}
125
126impl From<ServiceUrlsMemory> for ServiceUrls {
127    fn from(
128        ServiceUrlsMemory {
129            blob_store_addr,
130            chunk_store_addr,
131            directory_service_addr,
132            #[cfg(feature = "xp-composition-cli")]
133            experimental_store_composition,
134        }: ServiceUrlsMemory,
135    ) -> ServiceUrls {
136        ServiceUrls {
137            blob_store_addr,
138            chunk_store_addr,
139            directory_service_addr,
140            #[cfg(feature = "xp-composition-cli")]
141            experimental_store_composition,
142        }
143    }
144}
145
146/// Deserializes service addresses into composition config, configuring each
147/// service as the single "root".
148/// If the `xp-composition-cli` feature is enabled, and a file specified in the
149/// `--experimental-store-composition` parameter, this is used instead.
150pub async fn addrs_to_configs(
151    urls: impl Into<ServiceUrls>,
152) -> Result<CompositionConfigs, Box<dyn std::error::Error + Send + Sync>> {
153    let ServiceUrls {
154        blob_store_addr,
155        chunk_store_addr,
156        directory_service_addr,
157        #[cfg(feature = "xp-composition-cli")]
158        experimental_store_composition,
159    } = urls.into();
160
161    #[cfg(feature = "xp-composition-cli")]
162    if let Some(conf_path) = experimental_store_composition {
163        let conf_text = tokio::fs::read_to_string(conf_path).await?;
164        return Ok(with_registry(&REG, || toml::from_str(&conf_text))?);
165    }
166
167    let mut configs: CompositionConfigs = Default::default();
168
169    configs.blob_stores.insert(
170        "root".into(),
171        with_registry(&REG, || blob_store_addr.try_into())?,
172    );
173    configs.chunk_stores.insert(
174        "root".into(),
175        with_registry(&REG, || chunk_store_addr.try_into())?,
176    );
177    configs.directoryservices.insert(
178        "root".into(),
179        with_registry(&REG, || directory_service_addr.try_into())?,
180    );
181    Ok(configs)
182}
183
184/// Construct the castore handles from their addrs.
185pub async fn construct_services(
186    urls: impl Into<ServiceUrls>,
187) -> Result<
188    (
189        Arc<dyn BlobStore>,
190        Arc<dyn ChunkStore>,
191        Arc<dyn BlobEngine>,
192        Arc<dyn DirectoryService>,
193    ),
194    Box<dyn std::error::Error + Send + Sync>,
195> {
196    let configs = addrs_to_configs(urls).await?;
197    construct_services_from_configs(configs).await
198}
199
200/// Construct the castore handles from their addrs.
201pub async fn construct_services_from_configs(
202    configs: CompositionConfigs,
203) -> Result<
204    (
205        Arc<dyn BlobStore>,
206        Arc<dyn ChunkStore>,
207        Arc<dyn BlobEngine>,
208        Arc<dyn DirectoryService>,
209    ),
210    Box<dyn std::error::Error + Send + Sync>,
211> {
212    let mut comp = Composition::new(&REG);
213
214    comp.extend(configs.blob_stores);
215    comp.extend(configs.chunk_stores);
216    comp.extend(configs.directoryservices);
217
218    let blob_store: Arc<dyn BlobStore> = comp.build("root").await?;
219    let chunk_store: Arc<dyn ChunkStore> = comp.build("root").await?;
220    let blob_engine: Arc<dyn BlobEngine> = Arc::new(
221        blob_engine::ChunkingBlobEngine::builder()
222            .blob_store(blob_store.clone())
223            .chunk_store(chunk_store.clone())
224            .build(),
225    );
226    let directory_service: Arc<dyn DirectoryService> = comp.build("root").await?;
227
228    Ok((blob_store, chunk_store, blob_engine, directory_service))
229}
230
231/// Returns a new [DirectoryService]. Should only be used for tests.
232#[cfg(any(test, feature = "mocks"))]
233pub fn gen_test_directory_service() -> impl DirectoryService + Clone {
234    crate::directoryservice::RedbDirectoryService::new_temporary("test".to_string())
235}
236
237/// Returns a new [BlobEngine]. Should only be used for tests.
238///
239/// We wrap the underlying stores into Arc to match the signature of
240/// [construct_services_from_configs], so the downcast in SnixStoreFs works.
241///
242/// [BlobEngine]: crate::blob_engine::BlobEngine
243/// [construct_services_from_configs]: snix_castore::utils::construct_services_from_configs]
244#[cfg(any(test, feature = "mocks"))]
245pub fn gen_test_blob_engine()
246-> Arc<crate::blob_engine::ChunkingBlobEngine<Arc<dyn BlobStore>, Arc<dyn ChunkStore>>> {
247    use crate::{
248        blob_engine::ChunkingBlobEngine, blobstore::memory::MemoryBlobStore,
249        chunkstore::memory::MemoryChunkStore,
250    };
251
252    let blob_store = Arc::new(MemoryBlobStore::default()) as Arc<dyn BlobStore>;
253    let chunk_store = Arc::new(MemoryChunkStore::default()) as Arc<dyn ChunkStore>;
254
255    let blob_engine = ChunkingBlobEngine::builder()
256        .blob_store(blob_store)
257        .chunk_store(chunk_store)
258        // very small on purpose here, for tests.
259        .avg_chunk_size(512)
260        .build();
261
262    Arc::new(blob_engine)
263}
snix_castore/utils.rs

snix_castore/
utils.rs
