object_store/
multipart.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Cloud Multipart Upload
19//!
20//! This crate provides an asynchronous interface for multipart file uploads to
21//! cloud storage services. It's designed to offer efficient, non-blocking operations,
22//! especially useful when dealing with large files or high-throughput systems.
23
24use async_trait::async_trait;
25
26use crate::path::Path;
27use crate::{MultipartId, PutPayload, PutResult, Result};
28
29/// Represents a part of a file that has been successfully uploaded in a multipart upload process.
30#[derive(Debug, Clone)]
31pub struct PartId {
32    /// Id of this part
33    pub content_id: String,
34}
35
36/// A low-level interface for interacting with multipart upload APIs
37///
38/// Most use-cases should prefer [`ObjectStore::put_multipart`] as this is supported by more
39/// backends, including [`LocalFileSystem`], and automatically handles uploading fixed
40/// size parts of sufficient size in parallel
41///
42/// [`ObjectStore::put_multipart`]: crate::ObjectStore::put_multipart
43/// [`LocalFileSystem`]: crate::local::LocalFileSystem
44#[async_trait]
45pub trait MultipartStore: Send + Sync + 'static {
46    /// Creates a new multipart upload, returning the [`MultipartId`]
47    async fn create_multipart(&self, path: &Path) -> Result<MultipartId>;
48
49    /// Uploads a new part with index `part_idx`
50    ///
51    /// `part_idx` should be an integer in the range `0..N` where `N` is the number of
52    /// parts in the upload. Parts may be uploaded concurrently and in any order.
53    ///
54    /// Most stores require that all parts excluding the last are at least 5 MiB, and some
55    /// further require that all parts excluding the last be the same size, e.g. [R2].
56    /// [`WriteMultipart`] performs writes in fixed size blocks of 5 MiB, and clients wanting
57    /// to maximise compatibility should look to do likewise.
58    ///
59    /// [R2]: https://developers.cloudflare.com/r2/objects/multipart-objects/#limitations
60    /// [`WriteMultipart`]: crate::upload::WriteMultipart
61    async fn put_part(
62        &self,
63        path: &Path,
64        id: &MultipartId,
65        part_idx: usize,
66        data: PutPayload,
67    ) -> Result<PartId>;
68
69    /// Completes a multipart upload
70    ///
71    /// The `i`'th value of `parts` must be a [`PartId`] returned by a call to [`Self::put_part`]
72    /// with a `part_idx` of `i`, and the same `path` and `id` as provided to this method. Calling
73    /// this method with out of sequence or repeated [`PartId`], or [`PartId`] returned for other
74    /// values of `path` or `id`, will result in implementation-defined behaviour
75    async fn complete_multipart(
76        &self,
77        path: &Path,
78        id: &MultipartId,
79        parts: Vec<PartId>,
80    ) -> Result<PutResult>;
81
82    /// Aborts a multipart upload
83    async fn abort_multipart(&self, path: &Path, id: &MultipartId) -> Result<()>;
84}
object_store/multipart.rs

object_store/
multipart.rs
