WIP: report checksum validation level

Author: graebm

aws-s3-transfer-manager/src/operation/download.rs

@@ -16,6 +16,10 @@ pub mod builders;
 mod body;
 pub use body::{Body, ChunkOutput};
 
+/// Abstractions for checksum validation
+mod checksums;
+pub use checksums::ChecksumValidationLevel;
+
 mod discovery;
 
 mod handle;
@@ -31,6 +35,10 @@ pub use object_meta::ObjectMetadata;
 
 mod service;
 
+/// Provides metadata that isn't known until the download completes
+mod trailing_meta;
+pub use trailing_meta::TrailingMetadata;
+
 use crate::error;
 use crate::io::AggregatedBytes;
 use crate::runtime::scheduler::OwnedWorkPermit;
@@ -91,6 +99,7 @@ impl Download {
             discovery,
             object_meta_rx: Mutex::new(Some(object_meta_rx)),
             object_meta: OnceCell::new(),
+            trailing_meta: OnceCell::new(),
         })
     }
 }

aws-s3-transfer-manager/src/operation/download/checksums.rs

@@ -0,0 +1,47 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/// The level of checksum validation performed on a download
+#[derive(Debug, Clone, Default, PartialEq, Eq, PartialOrd, Ord)]
+pub enum ChecksumValidationLevel {
+    /// Checksum validation was not performed on all downloaded data.
+    ///
+    /// Note this DOES NOT mean the checksum didn't match. That would have failed the download with a
+    /// [ChecksumMismatch] error.
+    ///
+    /// There are many reasons for `NotValidated`, including:
+    /// -   The object had no checksum.
+    ///     -   Objects uploaded before 2025 are unlikely to have a checksum.
+    ///         In late-2024/early-2025 Amazon S3 began automatically calculating and storing checksums ([blog post]).
+    ///         The exact date for this varies by region.
+    ///     -   Third parties that mimic the Amazon S3 API may not provide a checksum.
+    /// -   The object was downloaded in chunks, and one or more chunks had no checksum.
+    ///     -   This happens when a large object with a [FullObject checksum][ChecksumTypes]
+    ///         is downloaded in multiple chunks.
+    ///     -   This happens when an object with a [Composite checksum][ChecksumTypes]
+    ///         is downloaded in chunks that don't align with the [PartSize] it was uploaded with.
+    /// -   Checksum validation was disabled in the underlying The S3 client,
+    ///     by configuring it with the non-default
+    ///     [`aws_sdk_s3::config::ResponseChecksumValidation::WhenRequired`].
+    ///
+    /// [ChecksumMismatch]: https://docs.rs/aws-smithy-checksums/latest/aws_smithy_checksums/body/validate/enum.Error.html#variant.ChecksumMismatch
+    /// [ChecksumTypes]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html#ChecksumTypes
+    /// [blog post]: https://aws.amazon.com/blogs/aws/introducing-default-data-integrity-protections-for-new-objects-in-amazon-s3/
+    /// [PartSize]: crate::types::PartSize
+    #[default]
+    NotValidated,
+    /// The checksum of each downloaded chunk was validated, but the
+    /// [FullObject or Composite checksum][ChecksumTypes] for the whole object
+    /// was not validated.
+    ///
+    /// This can happen if:
+    /// - A large object is downloaded in multiple chunks.
+    /// - You requested a range of the object to download, not the full object.
+    ///
+    /// [ChecksumTypes]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html#ChecksumTypes
+    AllChunks,
+    /// The full object was downloaded, and its checksum was validated.
+    FullObject,
+}

aws-s3-transfer-manager/src/operation/download/handle.rs

@@ -1,3 +1,7 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
 use std::sync::Arc;
 
 use crate::error::{self, ErrorKind};
@@ -6,13 +10,9 @@ use tokio::{
     task,
 };
 
-/*
- * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
- * SPDX-License-Identifier: Apache-2.0
- */
 use crate::operation::download::body::Body;
 
-use super::object_meta::ObjectMetadata;
+use super::{ObjectMetadata, TrailingMetadata};
 
 /// Response type for a single download object request.
 #[derive(Debug)]
@@ -23,6 +23,9 @@ pub struct DownloadHandle {
     /// Object metadata.
     pub(crate) object_meta: OnceCell<ObjectMetadata>,
 
+    /// Metadata that isn't available until the download completes.
+    pub(crate) trailing_meta: OnceCell<TrailingMetadata>,
+
     /// The object content, in chunks, and the metadata for each chunk
     pub(crate) body: Body,
 
@@ -74,6 +77,14 @@ impl DownloadHandle {
         tasks.abort_all();
         while (tasks.join_next().await).is_some() {}
     }
+
+    /// Get metadata about the completed download.
+    /// This won't return `Some` until all [`body`] chunks have been successfully received.
+    ///
+    /// [`body`]: method@Self::body
+    pub fn trailing_meta(&self) -> Option<&TrailingMetadata> {
+        self.trailing_meta.get()
+    }
 }
 
 #[cfg(test)]

aws-s3-transfer-manager/src/operation/download/trailing_meta.rs

@@ -0,0 +1,12 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+use super::ChecksumValidationLevel;
+
+#[derive(Debug)]
+/// Metadata that isn't available until the download completes.
+pub struct TrailingMetadata {
+    /// The level of checksum validation performed on this download.
+    pub checksum_validation_level: ChecksumValidationLevel,
+}