docs(bindings): add example for kms pkey offload

bindings/rust-examples/Cargo.toml

@@ -1,6 +1,8 @@
 [workspace]
 members = [
-    "client-hello-config-resolution", "tokio-server-client",
+    "async-pkey-offload",
+    "client-hello-config-resolution", 
+    "tokio-server-client",
 ]
 resolver = "2"
 

bindings/rust-examples/async-pkey-offload/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "async-pkey-offload"
+version.workspace = true
+authors.workspace = true
+publish.workspace = true
+license.workspace = true
+edition.workspace = true
+
+[dependencies]
+aws-config = "1.5.8"
+aws-sdk-kms = "1.47.0"
+clap = { version = "4", features = ["derive"] }
+pin-project = "1.1.6"
+rcgen = "0.13.1"
+s2n-tls = { path = "../../rust/extended/s2n-tls" }
+s2n-tls-tokio = { path = "../../rust/extended/s2n-tls-tokio" }
+tokio = { version = "1", features = ["full"] }
+tracing = "0.1.41"
+yasna = "0.5.2"

bindings/rust-examples/async-pkey-offload/README.md

@@ -0,0 +1,88 @@
+# PKey Offload with KMS
+
+This example shows how to use s2n-tls pkey offload functionality to create TLS connections with a private key that is stored in KMS.
+
+It will
+1. generate an asymmetric key in KMS
+2. create a public (self-signed) x509 certificate corresponding to the private key in KMS
+3. handle TLS connections for that certificate, offloading all private key operations to KMS
+
+```
+                      server (s2n-tls)                  KMS      
+                     ┌───────────────┐            ┌─────────────┐
+                     │               │            │             │
+Client──────────────►│  Public Key   ┼───────────►│ Private Key │
+            ▲        │ (certificate) │      ▲     │             │
+            │        │               │      │     └─────────────┘
+            │        └───────────────┘      │                    
+     TLS Connection                    pkey offload              
+                                      through AWS SDK            
+```
+
+The client will talk to an s2n-tls server. This server only contains the public key in the form of an x509 certificate. The server does _not_ hold a copy of a private key. The only copy of the key is stored in KMS, and it can not be removed from KMS. The advantage of this is that if an attacker were able to compromise the server, they could not steal the private key. 
+
+Because the server does not have a copy of the private key, it must delegate cryptographic operations to KMS, and return those results to the clients. s2n-tls offers a "pkey offload" feature to accomplish this behavior. This example will use s2n-tls pkey offload functionality along with the AWS SDK to successfully complete a TLS handshake with the client, while never actually holding the private key.
+
+### Running the demo
+You will need to have access to IAM credentials with KMS permissions to create, list, describe, sign, and delete keys.
+
+Once those are available in the environment, you can run the demo - which is structured as a test - with `cargo test -- --nocapture`.
+
+```
+creating new key
+Using KMS Key: "b6a9ff77-f672-46a1-8d59-8fa0eb1136ed"
+client successfully connected
+TlsStream {
+    connection: Connection {
+        handshake_type: "NEGOTIATED|FULL_HANDSHAKE|MIDDLEBOX_COMPAT",
+        cipher_suite: "TLS_AES_128_GCM_SHA256",
+        actual_protocol_version: TLS13,
+        selected_curve: "secp256r1",
+        ..
+    },
+}
+test handshake ... ok
+```
+
+You can clean up the test resources by running `cargo run --bin delete_demo_keys`.
+
+### Self Signed Cert Generation
+The example will use a self signed cert with an asymmetric key that is stored in KMS. First we generate a private key in KMS. This will be the private key of the certificate. We use [rcgen](https://github.com/rustls/rcgen) and its associated [KeyPair::from_remote](https://docs.rs/rcgen/latest/rcgen/trait.RemoteKeyPair.html) functionality to actually generate the cert. Below you can see what the certificate looked like when I ran it on my own machine.
+
+```
+Certificate:
+    Data:
+        Version: 3 (0x2)
+        Serial Number:
+            16:5c:dd:a4:d0:01:34:1a:82:16:03:2f:3b:d6:08:95:94:a0:6e:c3
+        Signature Algorithm: ecdsa-with-SHA384
+        Issuer: CN = rcgen self signed cert
+        Validity
+            Not Before: Jan  1 00:00:00 1975 GMT
+            Not After : Jan  1 00:00:00 4096 GMT
+        Subject: CN = rcgen self signed cert
+        Subject Public Key Info:
+            Public Key Algorithm: id-ecPublicKey
+                Public-Key: (384 bit)
+                pub:
+                    04:d4:40:4c:1a:77:c2:2a:d2:04:f6:11:17:e2:e5:
+                    7b:d7:14:9b:47:4a:fb:58:0e:09:a8:7e:c0:45:00:
+                    51:55:22:52:1e:51:46:98:e5:57:08:7c:31:36:d5:
+                    03:81:21:67:cf:88:75:43:21:c2:91:ec:bb:8f:67:
+                    12:76:67:df:44:a0:2f:55:57:af:89:57:66:38:ad:
+                    0d:0f:55:bb:2f:70:24:f8:46:67:5e:5b:d0:b5:ba:
+                    79:6e:48:a7:f3:c7:9c
+                ASN1 OID: secp384r1
+                NIST CURVE: P-384
+        X509v3 extensions:
+            X509v3 Subject Alternative Name: 
+                DNS:async-pkey.demo.s2n
+    Signature Algorithm: ecdsa-with-SHA384
+    Signature Value:
+        30:64:02:30:5f:8d:89:d2:ee:f1:2c:fc:88:43:3b:b4:31:6a:
+        7c:61:8e:6a:bb:b3:97:15:68:2d:77:c3:3e:08:c6:48:71:2f:
+        2d:ba:96:14:40:f0:66:7d:05:ba:47:27:12:83:d9:78:02:30:
+        27:df:5a:73:f6:3a:42:25:e2:7e:e4:e4:65:88:bc:56:98:7a:
+        47:92:bd:56:b7:1e:12:44:3a:e4:a1:63:32:f4:35:75:ac:e9:
+        94:d6:5d:2b:c5:c4:6d:3b:43:23:a4:b8
+```

bindings/rust-examples/async-pkey-offload/rust-toolchain

@@ -0,0 +1 @@
+stable

bindings/rust-examples/async-pkey-offload/src/bin/delete_demo_keys.rs

@@ -0,0 +1,35 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use async_pkey_offload::{get_demo_keys, DEMO_REGION};
+use aws_config::{BehaviorVersion, Region};
+use aws_sdk_kms::Client;
+
+/// This is a small helper script used to delete any keys that might have been
+/// created by the demo.
+///
+/// It will iterate over all the KMS keys and schedule the deletion of any keys
+/// where the key description is [crate::KEY_DESCRIPTION]
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let shared_config = aws_config::defaults(BehaviorVersion::v2024_03_28())
+        .region(Region::from_static(DEMO_REGION))
+        .load()
+        .await;
+
+    let client = Client::new(&shared_config);
+
+    let demo_key_ids = get_demo_keys(&client).await?;
+
+    if demo_key_ids.is_empty() {
+        // no keys to delete, can immediately return
+        return Ok(());
+    }
+
+    for k in demo_key_ids {
+        println!("scheduling {:?} for deletion", k);
+        client.schedule_key_deletion().key_id(k).send().await?;
+    }
+
+    Ok(())
+}

bindings/rust-examples/async-pkey-offload/src/lib.rs

@@ -0,0 +1,356 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use std::{
+    future::Future,
+    pin::Pin,
+    task::{Context, Poll},
+};
+
+use aws_sdk_kms::{
+    primitives::Blob,
+    types::{KeySpec, SigningAlgorithmSpec},
+    Client,
+};
+use pin_project::pin_project;
+use rcgen::CertificateParams;
+use s2n_tls::{
+    callbacks::{OperationType, PrivateKeyOperation},
+    connection::Connection,
+};
+use yasna::ASN1Result;
+
+pub const KEY_DESCRIPTION: &str = "KMS Asymmetric Key for s2n-tls pkey offload demo";
+pub const DEMO_REGION: &str = "us-west-2";
+pub const DEMO_DOMAIN: &str = "async-pkey.demo.s2n";
+
+/// Return a list of available demo keys.
+///
+/// There might be multiple keys if a pending deletion is manually cancelled.
+pub async fn get_demo_keys(client: &Client) -> Result<Vec<String>, Box<dyn std::error::Error>> {
+    let key_list = client.list_keys().send().await?;
+    if key_list.truncated {
+        // assumption: key list should be small enough to not require pagination
+        return Err("key list should not be truncated".into());
+    }
+
+    let key_list = match key_list.keys {
+        Some(list) => list,
+        None => return Ok(Vec::new()),
+    };
+
+    let mut matching_keys = Vec::new();
+    for k in key_list {
+        let describe_output = client
+            .describe_key()
+            .key_id(k.key_id().unwrap())
+            .send()
+            .await?;
+
+        let metadata = match describe_output.key_metadata {
+            Some(metadata) => metadata,
+            None => continue,
+        };
+
+        // this key is already scheduled for deletion
+        if metadata.deletion_date.is_some() {
+            continue;
+        }
+
+        if metadata.description() == Some(KEY_DESCRIPTION) {
+            matching_keys.push(k.key_id().unwrap().to_owned());
+        }
+    }
+    Ok(matching_keys)
+}
+
+/// Get a key from KMS, returning an existing key if found, or creating a new one.
+///
+/// It will return the first key where
+/// - it is not scheduled for deletion
+/// - the key description matches [KEY_DESCRIPTION]
+pub async fn get_key(client: &Client) -> Result<String, Box<dyn std::error::Error>> {
+    let mut demo_keys = get_demo_keys(client).await?;
+    if let Some(key_id) = demo_keys.pop() {
+        return Ok(key_id);
+    }
+
+    // no keys were found, so create one.
+    let create_key_resp = client
+        .create_key()
+        .key_spec(KeySpec::EccNistP384)
+        .key_usage(aws_sdk_kms::types::KeyUsageType::SignVerify)
+        .description(KEY_DESCRIPTION)
+        .send()
+        .await?;
+    println!("creating new key");
+    let key = create_key_resp.key_metadata.unwrap().key_id;
+    Ok(key)
+}
+
+/// KmsAsymmetricKey is a container used to implement application-specific traits.
+///
+/// It implements [rcgen::RemoteKeyPair] which allows us to create a self-signed
+/// x509 cert corresponding to the key pair.
+///
+/// It implements [s2n_tls::callbacks::PrivateKeyCallback] which allows us to offload
+/// cryptographic operations from s2n-tls to the KMS key.
+#[derive(Debug, Clone)]
+pub struct KmsAsymmetricKey {
+    /// AWS KMS SDK client.
+    kms_client: Client,
+    /// A copy of the public key in "raw" format
+    public_key: Vec<u8>,
+    /// The KMS key id
+    key_id: String,
+}
+
+impl KmsAsymmetricKey {
+    const EXPECTED_SIG: s2n_tls::enums::SignatureAlgorithm =
+        s2n_tls::enums::SignatureAlgorithm::ECDSA;
+
+    /// Encapsulate an existing KmsAsymmetricKey
+    ///
+    /// This method does not create a new key in KMS. It will retrieve the public
+    /// key of an existing key to be used locally.
+    pub async fn new(client: Client, key_id: String) -> Result<Self, Box<dyn std::error::Error>> {
+        let public_key_output = client
+            .get_public_key()
+            .key_id(key_id.clone())
+            .send()
+            .await?;
+        // > The public key that AWS KMS returns is a DER-encoded X.509 public key,
+        // > also known as SubjectPublicKeyInfo (SPKI), as defined in RFC 5280.
+        // > When you use the HTTP API or the AWS CLI, the value is Base64-encoded.
+        // > Otherwise, it is not Base64-encoded.
+        // https://docs.aws.amazon.com/kms/latest/developerguide/download-public-key.html
+        // Note that the rust sdk seems to handle common encoding tasks for
+        // us, so `encoded_public_key` is binary, not base64 encoded.
+        let encoded_public_key = public_key_output.public_key.unwrap().into_inner();
+        let raw_public_key = extract_ex_public_key(&encoded_public_key)?;
+
+        Ok(Self {
+            kms_client: client,
+            public_key: raw_public_key,
+            key_id,
+        })
+    }
+
+    /// Perform an async pkey offload.
+    ///
+    /// s2n-tls requires that future have 'static bounds, so this function can not
+    /// operation on `&self`. Instead we clone all of the necessary elements and
+    /// capture them in the closure.
+    async fn async_pkey_offload(
+        kms_client: Client,
+        key_id: String,
+        operation: s2n_tls::callbacks::PrivateKeyOperation,
+        kms_key_spec: SigningAlgorithmSpec,
+    ) -> Result<(PrivateKeyOperation, Vec<u8>), s2n_tls::error::Error> {
+        //> If this is an OperationType::Sign operation, then this input has
+        //> already been hashed and is the resultant digest.
+        let mut data_to_sign = vec![0; operation.input_size().unwrap()];
+        operation.input(&mut data_to_sign).unwrap();
+
+        // This is necessary as ConnectionFuture requires Sync
+        // but this is not implemented by many Futures, including
+        // those returned by the aws_sdk_kms client
+        let spawned_result = tokio::spawn(async move {
+            kms_client
+                .sign()
+                .key_id(key_id.clone())
+                .message_type(aws_sdk_kms::types::MessageType::Digest)
+                .message(Blob::new(data_to_sign))
+                .signing_algorithm(kms_key_spec)
+                .send()
+                .await
+                .unwrap()
+        });
+        let signature_output = spawned_result.await.unwrap();
+
+        let signature = signature_output.signature.unwrap().into_inner();
+
+        Ok((operation, signature))
+    }
+}
+
+#[pin_project]
+pub struct PrivateKeyFuture<F> {
+    #[pin]
+    fut: F,
+}
+
+impl<F> PrivateKeyFuture<F>
+where
+    F: 'static
+        + Send
+        + Future<Output = Result<(PrivateKeyOperation, Vec<u8>), s2n_tls::error::Error>>,
+{
+    pub fn new(fut: F) -> Self {
+        PrivateKeyFuture { fut }
+    }
+}
+
+impl<F> s2n_tls::callbacks::ConnectionFuture for PrivateKeyFuture<F>
+where
+    F: 'static
+        + Send
+        + Sync
+        + Future<Output = Result<(PrivateKeyOperation, Vec<u8>), s2n_tls::error::Error>>,
+{
+    fn poll(
+        self: Pin<&mut Self>,
+        connection: &mut Connection,
+        ctx: &mut Context,
+    ) -> Poll<Result<(), s2n_tls::error::Error>> {
+        let this = self.project();
+        let (op, out) = match this.fut.poll(ctx) {
+            Poll::Ready(out) => out?,
+            Poll::Pending => return Poll::Pending,
+        };
+        op.set_output(connection, &out)?;
+        Poll::Ready(Ok(()))
+    }
+}
+
+impl s2n_tls::callbacks::PrivateKeyCallback for KmsAsymmetricKey {
+    fn handle_operation(
+        &self,
+        _connection: &mut s2n_tls::connection::Connection,
+        operation: s2n_tls::callbacks::PrivateKeyOperation,
+    ) -> Result<
+        Option<std::pin::Pin<Box<dyn s2n_tls::callbacks::ConnectionFuture>>>,
+        s2n_tls::error::Error,
+    > {
+        let hash = match operation.kind()? {
+            // success!
+            OperationType::Sign(Self::EXPECTED_SIG, hash_algorithm) => Ok(hash_algorithm),
+
+            // errors
+            OperationType::Sign(s, _) => Err(s2n_tls::error::Error::application(
+                format!("Unsupported signature type: {:?}", s).into(),
+            )),
+            OperationType::Decrypt => Err(s2n_tls::error::Error::application(
+                "Decrypt operation not supported".into(),
+            )),
+            _ => Err(s2n_tls::error::Error::application(
+                format!("Unrecognized operation type: {:?}", operation.kind()).into(),
+            )),
+        }?;
+
+        // the hash must be available in KMS
+        let kms_key_spec = match hash {
+            s2n_tls::enums::HashAlgorithm::SHA256 => {
+                aws_sdk_kms::types::SigningAlgorithmSpec::EcdsaSha256
+            }
+            s2n_tls::enums::HashAlgorithm::SHA384 => {
+                aws_sdk_kms::types::SigningAlgorithmSpec::EcdsaSha384
+            }
+            s2n_tls::enums::HashAlgorithm::SHA512 => {
+                aws_sdk_kms::types::SigningAlgorithmSpec::EcdsaSha512
+            }
+            h => {
+                return Err(s2n_tls::error::Error::application(
+                    format!("requested hash type {:?} is not supported by KMS", h).into(),
+                ))
+            }
+        };
+
+        // This is the async closure that will actually call out to KMS.
+        let signing_future = KmsAsymmetricKey::async_pkey_offload(
+            self.kms_client.clone(),
+            self.key_id.clone(),
+            operation,
+            kms_key_spec,
+        );
+
+        // We wrap the async closure in a PrivateKeyFuture. PrivateKeyFuture implements
+        // s2n_tls::callbacks::ConnectionFuture, so s2n-tls knows how to poll
+        // this type to completion.
+        let wrapped_future = PrivateKeyFuture::new(signing_future);
+
+        // Finally we pin the future, allowing it to be safely polled.
+        Ok(Some(Box::pin(wrapped_future)))
+    }
+}
+
+impl rcgen::RemoteKeyPair for KmsAsymmetricKey {
+    fn public_key(&self) -> &[u8] {
+        &self.public_key
+    }
+
+    fn sign(&self, msg: &[u8]) -> Result<Vec<u8>, rcgen::Error> {
+        let signature: Result<Vec<u8>, Box<dyn std::error::Error>> =
+            // This trait require a "sync" function. Use `block_in_place` to run 
+            // the async function inside a sync context.
+            tokio::task::block_in_place(|| {
+                let current_runtime = tokio::runtime::Handle::current();
+
+                current_runtime.block_on(async {
+                    let output = self
+                        .kms_client
+                        .sign()
+                        .key_id(self.key_id.clone())
+                        .message(Blob::new(msg.to_owned()))
+                        .signing_algorithm(aws_sdk_kms::types::SigningAlgorithmSpec::EcdsaSha384)
+                        .send()
+                        .await?;
+                    let signature = output.signature.unwrap().into_inner();
+                    Ok(signature)
+                })
+            });
+        Ok(signature.unwrap())
+    }
+
+    fn algorithm(&self) -> &'static rcgen::SignatureAlgorithm {
+        &rcgen::PKCS_ECDSA_P384_SHA384
+    }
+}
+
+/// return a pem encoded self-signed certificate
+pub fn create_self_signed_cert(
+    kms_key: KmsAsymmetricKey,
+) -> Result<String, Box<dyn std::error::Error>> {
+    let key_pair = rcgen::KeyPair::from_remote(Box::new(kms_key))?;
+
+    let params = CertificateParams::new(vec![DEMO_DOMAIN.to_owned()])?.self_signed(&key_pair)?;
+    Ok(params.pem())
+}
+
+/// Parse a der-encoded SubjectPublicKeyInfo into a raw public key.
+///  
+/// A SubjectPublicKeyInfo is defined as follows
+/// ```text
+/// SubjectPublicKeyInfo  ::=  SEQUENCE  {
+///     algorithm            AlgorithmIdentifier,
+///     subjectPublicKey     BIT STRING  }
+/// ```
+/// This function just skips over the algorithm identifier and returns the raw
+/// subjectPublicKey field.
+pub fn extract_ex_public_key(spki_der: &[u8]) -> ASN1Result<Vec<u8>> {
+    yasna::parse_der(spki_der, |reader| {
+        reader.read_sequence(|reader| {
+            // read the algorithm identifier (ECDSA, etc.)
+            reader.next().read_sequence(|reader| {
+                // Read past the OID identifying the algorithm (e.g., ECDSA with SHA-256)
+                let _algorithm_oid = reader.next().read_oid()?;
+
+                // Read past the second OID which identifies the curve (e.g., prime256v1)
+                let _curve_oid = reader.next().read_oid()?;
+
+                Ok(())
+            })?;
+            // Read the BIT STRING (the actual public key)
+            let (public_key_bytes, _size) = reader.next().read_bitvec_bytes()?;
+
+            // The public key inside the BIT STRING should be in uncompressed format (0x04 || x || y)
+            assert_eq!(
+                public_key_bytes[0], 0x04,
+                "Public Key should use an uncompressed format"
+            );
+
+            // Return the raw public key
+            Ok(public_key_bytes)
+        })
+    })
+}

bindings/rust-examples/async-pkey-offload/tests/client_server.rs

@@ -0,0 +1,112 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use async_pkey_offload::{
+    create_self_signed_cert, get_key, KmsAsymmetricKey, DEMO_DOMAIN, DEMO_REGION,
+};
+use aws_config::{BehaviorVersion, Region};
+use aws_sdk_kms::Client;
+use s2n_tls::security;
+use s2n_tls_tokio::{TlsAcceptor, TlsConnector};
+use tokio::{
+    io::{AsyncReadExt, AsyncWriteExt},
+    net::{TcpListener, TcpStream},
+    task::JoinHandle,
+};
+
+const MESSAGE: &[u8] = b"hello world";
+
+// we need multiple threads, because block_on can only be used in multi-threaded
+// runtimes
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn handshake() -> Result<(), Box<dyn std::error::Error>> {
+    let kms_key = {
+        let shared_config = aws_config::defaults(BehaviorVersion::v2024_03_28())
+            .region(Region::from_static(DEMO_REGION))
+            .load()
+            .await;
+        let kms_client = Client::new(&shared_config);
+        let key_id = get_key(&kms_client).await?;
+        println!("Using KMS Key: {:?}", key_id);
+        KmsAsymmetricKey::new(kms_client.clone(), key_id)
+            .await
+            .unwrap()
+    };
+
+    let self_signed_cert = create_self_signed_cert(kms_key.clone())?;
+    // async blocks are marked `move`, so we need another copy
+    let cert_copy = self_signed_cert.clone();
+
+    // Bind to an address and listen for connections.
+    // ":0" can be used to automatically assign a port.
+    let listener = TcpListener::bind("127.0.0.1:0").await?;
+    let addr = listener
+        .local_addr()
+        .map(|x| x.to_string())
+        .unwrap_or_else(|_| "UNKNOWN".to_owned());
+
+    let server_loop: JoinHandle<Result<(), Box<dyn std::error::Error + Send + Sync>>> =
+        tokio::spawn(async move {
+            let mut server_config = s2n_tls::config::Config::builder();
+            server_config.set_security_policy(&security::DEFAULT_TLS13)?;
+            server_config.load_public_pem(self_signed_cert.as_bytes())?;
+            server_config.set_private_key_callback(kms_key)?;
+
+            let server = TlsAcceptor::new(server_config.build()?);
+
+            loop {
+                let (stream, _peer_addr) = listener.accept().await?;
+
+                let server = server.clone();
+                tokio::spawn(async move {
+                    let mut tls = server.accept(stream).await.unwrap();
+
+                    // server writes message to client
+                    tls.write_all(MESSAGE).await.unwrap();
+
+                    // server waits for client to initiate shutdown
+                    let read = tls.read(&mut [0]).await.unwrap();
+                    assert_eq!(read, 0);
+
+                    // server completes shutdown
+                    tls.shutdown().await.unwrap();
+
+                    Ok::<(), Box<dyn std::error::Error + Send + Sync>>(())
+                });
+            }
+        });
+
+    let client = tokio::spawn(async move {
+        let mut client_config = s2n_tls::config::Config::builder();
+        client_config.set_security_policy(&security::DEFAULT_TLS13)?;
+        client_config.trust_pem(cert_copy.as_bytes())?;
+
+        // Create the TlsConnector based on the configuration.
+        let client = TlsConnector::new(client_config.build()?);
+
+        // Connect to the server.
+        let stream = TcpStream::connect(addr).await?;
+        let mut tls = client.connect(DEMO_DOMAIN, stream).await?;
+        println!("client successfully connected");
+        println!("{:#?}", tls);
+
+        // client reads expected message from server
+        let mut buffer = [0; MESSAGE.len()];
+        tls.read_exact(&mut buffer).await?;
+        assert_eq!(buffer, MESSAGE);
+
+        // client initiates shutdown
+        tls.shutdown().await?;
+
+        // client waits for server to shutdown
+        let read = tls.read(&mut [0]).await?;
+        assert_eq!(read, 0);
+
+        Ok::<(), Box<dyn std::error::Error + Send + Sync>>(())
+    });
+
+    client.await.unwrap().unwrap();
+    server_loop.abort();
+
+    Ok(())
+}