diff --git a/CHANGELOG.md b/CHANGELOG.md index 30dab3d..e19fc9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## 0.7.1 — 2026-10-02 + +### Fixed + +- **S3 requests no longer end after 30 seconds.** object_store's default + request timeout runs from connecting to the end of the response body, + so 0.7.0's single PUTs (up to 1 GiB) and any large GET failed on an + ordinary uplink: at 85 Mbit/s about 320 MB fits in 30 s, less with jobs + sharing it. Every S3 client bigstore builds now allows six hours per + request and keeps the 5-second connect limit, with no per-read limit + (a PUT's response read waits for the whole upload). A caller that builds + its own client for large transfers needs the same. + +### Changed + +- **Single PUTs go up to 4 GiB** (`backend::SINGLE_PUT_MAX`, was 1 GiB), + inside the 5 GB that S3 and Wasabi document. Tracking runs write tables + of up to 2 GB; at 1 GiB those went multipart, whose ETag is never the + md5, so every S3 scrub read them back whole. The process-wide budget + for single-PUT bodies held in memory is the same 4 GiB, counted in MiB + units so it fits the semaphore. + ## 0.7.0 — 2026-10-02 ### Added diff --git a/Cargo.lock b/Cargo.lock index 6bc3f00..bef7311 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -596,7 +596,7 @@ dependencies = [ [[package]] name = "git-bigstore-rs" -version = "0.7.0" +version = "0.7.1" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index eeabaf7..82f95d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "git-bigstore-rs" -version = "0.7.0" +version = "0.7.1" edition = "2021" rust-version = "1.89" description = "Large files in git, your bucket, one binary." diff --git a/README.md b/README.md index b1b309d..25d929d 100644 --- a/README.md +++ b/README.md @@ -487,12 +487,15 @@ What it guarantees: cloned into an existing file. A hard link is one file under two names: write it in place and the store's object changes too (a scrub then finds it damaged), so use it only for files nothing rewrites. -- **Uploads are checked.** Objects up to 1 GiB go up in one PUT, whose S3 +- **Uploads are checked.** Objects up to 4 GiB go up in one PUT, whose S3 ETag is their md5: that proves the write, and later scrubs prove the object from the listing alone. Larger objects go up in parts and are read back whole, after the upload and on every S3 scrub. A single PUT - holds its object in memory; all of them share a 1 GiB budget per - process, so a push needs about 1 GiB of RAM at most, whatever `--jobs`. + holds its object in memory; all of them share a 4 GiB budget per + process, so a push needs about 4 GiB of RAM at most, whatever `--jobs`. + An S3 request may take up to six hours (object_store's default is 30 + seconds, which a large single PUT or GET cannot meet); the 5-second + connect limit stays. - **History without git.** Every push that changes an output records a version under `bigstore-history//` on the remote; a push that changes nothing adds nothing. History is a graph, not a timeline: each version diff --git a/src/backend/mod.rs b/src/backend/mod.rs index f1230a0..31cebb2 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -358,22 +358,30 @@ async fn rclone_into( /// Objects up to this size are written with one PUT, whose S3 ETag is the /// md5 of the bytes, so the write (and every later S3 scrub of it) is /// proven without reading it back; larger ones by multipart upload, read -/// back whole after writing and on every S3 scrub. A PUT holds the object -/// in memory, so this stays well below S3's 5 GiB limit, and the bodies of -/// single PUTs that bigstore reads into memory (by -/// [`Store::put_file`] and `folder::integrity::replace`) share one budget -/// of this many bytes across the process, whatever the job count: small -/// objects go up side by side, large ones one after another. -pub const SINGLE_PUT_MAX: u64 = 1 << 30; - -/// The single-PUT budget: one permit per byte. +/// back whole after writing and on every S3 scrub. 4 GiB stays clearly +/// inside the single-PUT limit S3 and Wasabi document as 5 GB, and holds +/// every table a tracking run writes (the largest seen is 2 GB). A PUT +/// holds the object in memory, so the bodies of single PUTs that bigstore +/// reads into memory (by [`Store::put_file`] and +/// `folder::integrity::replace`) share one budget of this many bytes +/// across the process, whatever the job count: small objects go up side by +/// side, large ones one after another. +pub const SINGLE_PUT_MAX: u64 = 4 << 30; + +/// The budget's unit: a permit stands for this many bytes of body, so the +/// whole budget fits the semaphore's `u32` count. +const BUDGET_UNIT: u64 = 1 << 20; + +/// The single-PUT budget: one permit per [`BUDGET_UNIT`] of body. static PUT_BUDGET: tokio::sync::Semaphore = - tokio::sync::Semaphore::const_new(SINGLE_PUT_MAX as usize); + tokio::sync::Semaphore::const_new((SINGLE_PUT_MAX / BUDGET_UNIT) as usize); /// Room in the single-PUT budget for a body of `size` bytes (at most -/// [`SINGLE_PUT_MAX`]), held until the permit drops. +/// [`SINGLE_PUT_MAX`], rounded up to whole [`BUDGET_UNIT`]s), held until +/// the permit drops. pub(crate) async fn put_budget(size: u64) -> tokio::sync::SemaphorePermit<'static> { - let permits = u32::try_from(size.min(SINGLE_PUT_MAX)).expect("1 GiB fits in u32"); + let units = size.min(SINGLE_PUT_MAX).div_ceil(BUDGET_UNIT); + let permits = u32::try_from(units).expect("the budget's units fit in u32"); PUT_BUDGET .acquire_many(permits) .await diff --git a/src/backend/store.rs b/src/backend/store.rs index 6a1be6b..afb111b 100644 --- a/src/backend/store.rs +++ b/src/backend/store.rs @@ -26,6 +26,25 @@ fn cloud_crypto() -> Result<()> { Ok(()) } +/// The time limits for an S3 client that moves whole store files. +/// +/// object_store's default request timeout is 30 seconds from connecting to +/// the end of the response body, which a single PUT of up to +/// [`SINGLE_PUT_MAX`](super::SINGLE_PUT_MAX), or a GET of a large object, +/// cannot meet on an ordinary uplink (85 Mbit/s moves ~320 MB in 30 s, and +/// jobs share it). The whole-request limit is therefore six hours, which a +/// 4 GiB transfer meets at about 1.6 Mbit/s; the 5-second connect limit +/// stays. No per-read limit is set: while a PUT body is being sent the +/// response read waits for the whole upload, and a per-read limit would end +/// a slow upload that is making progress. A transfer that stops making +/// progress is ended by the connection itself or by cancelling the run. +/// Every S3 client bigstore builds uses these limits. +fn with_transfer_limits(builder: AmazonS3Builder) -> AmazonS3Builder { + use object_store::aws::AmazonS3ConfigKey as Key; + use object_store::ClientConfigKey; + builder.with_config(Key::Client(ClientConfigKey::Timeout), "6h") +} + /// Build an ObjectStore client from backend config. Used by both bigstore /// and the LFS transfer adapter. pub fn build_object_store(backend: &BackendConfig) -> Result> { @@ -37,7 +56,8 @@ pub fn build_object_store(backend: &BackendConfig) -> Result { cloud_crypto()?; - let mut builder = AmazonS3Builder::from_env().with_bucket_name(bucket); + let mut builder = + with_transfer_limits(AmazonS3Builder::from_env()).with_bucket_name(bucket); if let Some(ep) = endpoint { builder = builder @@ -152,7 +172,7 @@ pub fn build_strict_s3( (Some(k), Some(s)) if !k.is_empty() && !s.is_empty() => (k, s), _ => return Err(super::Error::CredentialsMissing.into()), }; - let mut builder = AmazonS3Builder::new() + let mut builder = with_transfer_limits(AmazonS3Builder::new()) .with_bucket_name(bucket) .with_endpoint(endpoint) .with_virtual_hosted_style_request(false) diff --git a/src/folder/integrity.rs b/src/folder/integrity.rs index f70dc21..84ebd0c 100644 --- a/src/folder/integrity.rs +++ b/src/folder/integrity.rs @@ -27,7 +27,7 @@ //! same store (time as `20261001T120000.123456789Z`): a [`Kind::Other`] //! key, which no scrub, listing or transfer touches. //! -//! Objects up to [`SINGLE_PUT_MAX`](backend::SINGLE_PUT_MAX) (1 GiB) are +//! Objects up to [`SINGLE_PUT_MAX`](backend::SINGLE_PUT_MAX) (4 GiB) are //! written with one PUT, so on S3 their ETag is their md5 and a scrub //! proves them from the listing alone. A larger object goes up in parts: //! its ETag never is its md5, so it is read back whole when written and @@ -116,7 +116,7 @@ impl Unreadable { /// manifests whose listed ETag is their name's md5 are good unread, unless /// `opts.deep`; every other file (records, any other ETag, none) is read /// and checked against its name. So on S3 an object written by multipart -/// upload (over 1 GiB, [the module docs](self)) is read whole on every +/// upload (over 4 GiB, [the module docs](self)) is read whole on every /// scrub. A listing that fails is the error; a file that fails is /// [`Unreadable`] in the report. Reads only. pub fn scrub(store: &dyn ObjectStore, opts: &ScrubOptions) -> Result { @@ -302,7 +302,7 @@ pub enum Replaced { /// [quarantine](self) first (on a `LocalFileSystem`, a hard link: the /// store must be on a filesystem that has them). Then `source` replaces it /// in one atomic write (on a `LocalFileSystem`, a temp file renamed over -/// the name): one PUT up to 1 GiB, held in memory meanwhile within the +/// the name): one PUT up to 4 GiB, held in memory meanwhile within the /// process's single-PUT budget ([`SINGLE_PUT_MAX`]), else a multipart /// upload, streamed from a file or sent from `Source::Bytes` a part at a /// time ([the module docs](self)). Last, the write is checked: an ETag @@ -708,7 +708,7 @@ mod tests { format!("files/md5/{}/{}", &md5[..2], &md5[2..]) } - /// Sources over the single-PUT ceiling (1 MiB here, 1 GiB for real) + /// Sources over the single-PUT ceiling (1 MiB here, 4 GiB for real) /// go up in parts, checked as they go and read back after. const CEILING: u64 = 1 << 20;