From a9e0d7b8ea705d6ba0dc90e0ca3b3a3c8652c7f5 Mon Sep 17 00:00:00 2001 From: martintomka Date: Fri, 22 May 2026 17:06:22 +0200 Subject: [PATCH 01/30] feat(fetch_tls): add new TLS abstraction crate Introduce the etch_tls crate providing a unified TLS configuration API with pluggable backends for rustls and native-tls. Includes client identity handling, connection options, and test utilities. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .spelling | 9 + CHANGELOG.md | 1 + Cargo.lock | 37 +++ Cargo.toml | 4 + crates/fetch_tls/CHANGELOG.md | 1 + crates/fetch_tls/Cargo.toml | 52 ++++ crates/fetch_tls/README.md | 52 ++++ crates/fetch_tls/favicon.ico | 3 + crates/fetch_tls/logo.png | 3 + crates/fetch_tls/src/backend.rs | 122 ++++++++ crates/fetch_tls/src/client_identity.rs | 256 +++++++++++++++ crates/fetch_tls/src/lib.rs | 47 +++ crates/fetch_tls/src/native_tls.rs | 177 +++++++++++ crates/fetch_tls/src/options.rs | 334 ++++++++++++++++++++ crates/fetch_tls/src/rustls.rs | 393 ++++++++++++++++++++++++ crates/fetch_tls/src/testing.rs | 80 +++++ 16 files changed, 1571 insertions(+) create mode 100644 crates/fetch_tls/CHANGELOG.md create mode 100644 crates/fetch_tls/Cargo.toml create mode 100644 crates/fetch_tls/README.md create mode 100644 crates/fetch_tls/favicon.ico create mode 100644 crates/fetch_tls/logo.png create mode 100644 crates/fetch_tls/src/backend.rs create mode 100644 crates/fetch_tls/src/client_identity.rs create mode 100644 crates/fetch_tls/src/lib.rs create mode 100644 crates/fetch_tls/src/native_tls.rs create mode 100644 crates/fetch_tls/src/options.rs create mode 100644 crates/fetch_tls/src/rustls.rs create mode 100644 crates/fetch_tls/src/testing.rs diff --git a/.spelling b/.spelling index ea386895d..5945ad724 100644 --- a/.spelling +++ b/.spelling @@ -556,3 +556,12 @@ addressability bumpable dereferenceable NonNull +crypto +DER +macOS +PEM +rustls +TLS +verifier +Verifier +backend's diff --git a/CHANGELOG.md b/CHANGELOG.md index 57c3b358f..08c54da40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Please see each crate's change log below: - [`data_privacy_macros`](./crates/data_privacy_macros/CHANGELOG.md) - [`data_privacy_macros_impl`](./crates/data_privacy_macros_impl/CHANGELOG.md) - [`fetch_hyper`](./crates/fetch_hyper/CHANGELOG.md) +- [`fetch_tls`](./crates/fetch_tls/CHANGELOG.md) - [`fundle`](./crates/fundle/CHANGELOG.md) - [`fundle_macros`](./crates/fundle_macros/CHANGELOG.md) - [`fundle_macros_impl`](./crates/fundle_macros_impl/CHANGELOG.md) diff --git a/Cargo.lock b/Cargo.lock index 438dd32ff..f54f1c578 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -676,6 +676,18 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "convert_case" version = "0.10.0" @@ -1525,6 +1537,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.9.0" @@ -3076,6 +3097,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der 0.7.10", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -3604,6 +3635,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" +[[package]] +name = "typenum" +version = "1.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" + [[package]] name = "unicode-ident" version = "1.0.24" diff --git a/Cargo.toml b/Cargo.toml index 182f1f13a..3a5ccb40b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,7 @@ data_privacy = { path = "crates/data_privacy", default-features = false, version data_privacy_macros = { path = "crates/data_privacy_macros", default-features = false, version = "0.9.0" } data_privacy_macros_impl = { path = "crates/data_privacy_macros_impl", default-features = false, version = "0.9.0" } fetch_hyper = { path = "crates/fetch_hyper", default-features = false, version = "0.1.1" } +fetch_tls = { path = "crates/fetch_tls", default-features = false, version = "0.1.0" } fundle = { path = "crates/fundle", default-features = false, version = "0.3.0" } fundle_macros = { path = "crates/fundle_macros", default-features = false, version = "0.3.0" } fundle_macros_impl = { path = "crates/fundle_macros_impl", default-features = false, version = "0.3.0" } @@ -61,6 +62,7 @@ alloc_tracker = { version = "0.5.9", default-features = false } allocator-api2 = { version = "0.4.0", default-features = false } anyhow = { version = "1.0.100", default-features = false } async-once-cell = { version = "0.5", default-features = false } +base64 = { version = "0.22", default-features = false, features = ["alloc"] } bolero = { version = "0.13.4", default-features = false } bumpalo = { version = "3.20.2", default-features = false } bytemuck = { version = "1.25", default-features = false } @@ -122,6 +124,8 @@ regex = { version = "1.12.2", default-features = false } rstest = { version = "0.26", default-features = false } rustc-hash = { version = "2.1.0", default-features = false } rustls = { version = "0.23.40", default-features = false } +rustls-pki-types = { version = "1.14.1", default-features = false } +rustls-symcrypt = { version = "0.2.3", default-features = false } serde = { version = "1.0.228", default-features = false } serde_core = { version = "1.0.228", default-features = false } serde_json = { version = "1.0.145", default-features = false } diff --git a/crates/fetch_tls/CHANGELOG.md b/crates/fetch_tls/CHANGELOG.md new file mode 100644 index 000000000..825c32f0d --- /dev/null +++ b/crates/fetch_tls/CHANGELOG.md @@ -0,0 +1 @@ +# Changelog diff --git a/crates/fetch_tls/Cargo.toml b/crates/fetch_tls/Cargo.toml new file mode 100644 index 000000000..a244fc0a9 --- /dev/null +++ b/crates/fetch_tls/Cargo.toml @@ -0,0 +1,52 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +[package] +name = "fetch_tls" +description = "TLS configurations and APIs used by 'fetch' crate." +version = "0.1.0" +readme = "README.md" +keywords = ["oxidizer", "tls", "fetch", "http", "client"] +categories = ["network-programming"] + +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +license.workspace = true +homepage.workspace = true +repository = "https://github.com/microsoft/oxidizer/tree/main/crates/fetch_tls" + +[package.metadata.docs.rs] +all-features = true + +[features] +default = [] +rustls = ["dep:rustls-symcrypt", "dep:rustls"] +native-tls = ["dep:native-tls"] + +[dependencies] +# internal +ohno = { workspace = true } + +# external +http = { workspace = true } +base64 = { workspace = true } +# Disabling aws-lc by default; rustls-symcrypt is the intended crypto provider. +rustls = { workspace = true, features = [ + "tls12", + "std", +], default-features = false, optional = true } +rustls-pki-types = { workspace = true, features = ["alloc", "std"] } +rustls-symcrypt = { workspace = true, optional = true } +native-tls = { workspace = true, optional = true, features = ["alpn"] } + +[dev-dependencies] +mutants = { workspace = true } +# Test code unconditionally references both backends; pull them in as +# dev-dependencies so `cargo test` works with no features enabled. +rustls = { workspace = true, features = ["tls12", "std"], default-features = false } +rustls-symcrypt = { workspace = true } +native-tls = { workspace = true, features = ["alpn"] } + +[lints] +workspace = true diff --git a/crates/fetch_tls/README.md b/crates/fetch_tls/README.md new file mode 100644 index 000000000..5c3aecc05 --- /dev/null +++ b/crates/fetch_tls/README.md @@ -0,0 +1,52 @@ +
+ Fetch Tls Logo + +# Fetch Tls + +[![crate.io](https://img.shields.io/crates/v/fetch_tls.svg)](https://crates.io/crates/fetch_tls) +[![docs.rs](https://docs.rs/fetch_tls/badge.svg)](https://docs.rs/fetch_tls) +[![MSRV](https://img.shields.io/crates/msrv/fetch_tls)](https://crates.io/crates/fetch_tls) +[![CI](https://github.com/microsoft/oxidizer/actions/workflows/main.yml/badge.svg?event=push)](https://github.com/microsoft/oxidizer/actions/workflows/main.yml) +[![Coverage](https://codecov.io/gh/microsoft/oxidizer/graph/badge.svg?token=FCUG0EL5TI)](https://codecov.io/gh/microsoft/oxidizer) +[![License](https://img.shields.io/badge/license-MIT-blue.svg)](../../LICENSE) +This crate was developed as part of the Oxidizer project + +
+ +Backend-agnostic TLS configuration for HTTP clients. + +Build a [`TlsOptions`][__link0] with [`TlsOptions::builder_rustls`][__link1] or +[`TlsOptions::builder_native_tls`][__link2], or wrap a pre-built +[`rustls::ClientConfig`][__link3] / +[`native_tls::TlsConnector`][__link4] via `From`/`Into`. +Materialize a [`TlsBackend`][__link5] with [`TlsOptions::build_backend`][__link6]. + +## Features + +* **`rustls`** — pure-Rust [`rustls`][__link7] paired with + [`rustls-symcrypt`][__link8] as the crypto provider. +* **`native-tls`** — platform native TLS (`SChannel` on Windows, + Security Framework on macOS, `OpenSSL` on Linux). + +With neither feature enabled, [`TlsOptions::default`][__link9] still constructs but +[`TlsOptions::build_backend`][__link10] returns a [`BackendError`][__link11]. + + +
+ +This crate was developed as part of The Oxidizer Project. Browse this crate's source code. + + + [__cargo_doc2readme_dependencies_info]: ggGkYW0CYXSEGy4k8ldDFPOhG2VNeXtD5nnKG6EPY6OfW5wBG8g18NOFNdxpYXKEGz0-gwD02BMqGx-aWk5R-ciAG5SPoepFPbthG4u5TGxqSEiPYWSEgmlmZXRjaF90bHNlMC4xLjCCam5hdGl2ZV90bHNmMC4yLjE4gmZydXN0bHNnMC4yMy40MIJvcnVzdGxzX3N5bWNyeXB0ZTAuMi4z + [__link0]: https://docs.rs/fetch_tls/0.1.0/fetch_tls/?search=TlsOptions + [__link1]: https://docs.rs/fetch_tls/0.1.0/fetch_tls/?search=TlsOptions::builder_rustls + [__link10]: https://docs.rs/fetch_tls/0.1.0/fetch_tls/?search=TlsOptions::build_backend + [__link11]: https://docs.rs/fetch_tls/0.1.0/fetch_tls/?search=BackendError + [__link2]: https://docs.rs/fetch_tls/0.1.0/fetch_tls/?search=TlsOptions::builder_native_tls + [__link3]: https://docs.rs/rustls/0.23.40/rustls/?search=ClientConfig + [__link4]: https://docs.rs/native_tls/0.2.18/native_tls/?search=TlsConnector + [__link5]: https://docs.rs/fetch_tls/0.1.0/fetch_tls/?search=TlsBackend + [__link6]: https://docs.rs/fetch_tls/0.1.0/fetch_tls/?search=TlsOptions::build_backend + [__link7]: https://crates.io/crates/rustls/0.23.40 + [__link8]: https://crates.io/crates/rustls_symcrypt/0.2.3 + [__link9]: https://docs.rs/fetch_tls/0.1.0/fetch_tls/?search=TlsOptions::default diff --git a/crates/fetch_tls/favicon.ico b/crates/fetch_tls/favicon.ico new file mode 100644 index 000000000..ccae81142 --- /dev/null +++ b/crates/fetch_tls/favicon.ico @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e48225cb42c8ac02df5dd4a9bdba29c1e8d10436cc27055d6a021bcb951d4145 +size 480683 diff --git a/crates/fetch_tls/logo.png b/crates/fetch_tls/logo.png new file mode 100644 index 000000000..f2bd66915 --- /dev/null +++ b/crates/fetch_tls/logo.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63ad64ebb0185d7cd153acc291989d72e3a0094603d8bfe781a220861de247f2 +size 17876 diff --git a/crates/fetch_tls/src/backend.rs b/crates/fetch_tls/src/backend.rs new file mode 100644 index 000000000..62fc0ec51 --- /dev/null +++ b/crates/fetch_tls/src/backend.rs @@ -0,0 +1,122 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Fully constructed TLS backends ready for use by an HTTP client. + +/// Error returned when materializing a [`TlsBackend`] from +/// [`TlsOptions`](super::TlsOptions) fails. +#[ohno::error] +pub struct BackendError; + +/// A fully constructed TLS backend ready for use by an HTTP client. +/// +/// Unlike [`TlsOptions`](super::TlsOptions), which describes *how* to build +/// a TLS configuration, `TlsBackend` holds the resulting backend-specific +/// state. Which variants are available depends on enabled features: +/// [`TlsBackend::Rustls`] requires `rustls`; [`TlsBackend::NativeTls`] +/// requires `native-tls`. +/// +/// Typically produced from [`TlsOptions`](super::TlsOptions); construct +/// directly only when wrapping a pre-built backend. +#[derive(Debug, Clone)] +#[allow( + clippy::allow_attributes, + clippy::large_enum_variant, + reason = "configuration object; boxing would clutter the public API without performance benefit" +)] +pub enum TlsBackend { + /// rustls backend, carrying a shared [`ClientConfig`](::rustls::ClientConfig). + #[cfg(any(feature = "rustls", test))] + Rustls(std::sync::Arc<::rustls::ClientConfig>), + + /// Platform native TLS backend (`SChannel` on Windows, Security Framework + /// on macOS, `OpenSSL` on Linux). + #[cfg(any(feature = "native-tls", test))] + NativeTls(::native_tls::TlsConnector), +} + +/// Environment-supplied defaults for materializing a [`TlsBackend`]. +/// +/// Lets HTTP client crates own platform / policy choices (such as which +/// crypto provider or root store to use) without baking them into +/// `fetch_tls`. Each backend that needs environment state has its own +/// constructor; native-tls and pre-configured backends ignore defaults. +/// +/// Use [`TlsBackendDefaults::new`] when no backend-specific state is +/// required. Building a rustls backend without +/// [`TlsBackendDefaults::rustls`] returns a [`BackendError`]. +#[derive(Clone, Default)] +pub struct TlsBackendDefaults { + #[cfg(any(feature = "rustls", test))] + pub(crate) rustls: Option, +} + +/// Environment-supplied defaults specific to the rustls backend. +#[cfg(any(feature = "rustls", test))] +#[derive(Clone)] +pub(crate) struct RustlsDefaults { + pub(crate) crypto_provider: std::sync::Arc<::rustls::crypto::CryptoProvider>, + pub(crate) verifier: std::sync::Arc, +} + +impl TlsBackendDefaults { + /// Creates an empty set of defaults. + /// + /// Sufficient for native-tls or pre-configured backends; materializing + /// a rustls backend with these returns a [`BackendError`]. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Supplies the rustls crypto provider and a fallback server certificate verifier. + /// + /// The verifier is used only when the caller did not configure one via + /// [`TlsOptionsBuilder::server_certificate_verifier`](super::TlsOptionsBuilder::server_certificate_verifier). + #[cfg(any(feature = "rustls", test))] + #[cfg_attr(docsrs, doc(cfg(feature = "rustls")))] + #[must_use] + pub fn rustls( + crypto_provider: std::sync::Arc<::rustls::crypto::CryptoProvider>, + verifier: std::sync::Arc, + ) -> Self { + Self { + rustls: Some(RustlsDefaults { crypto_provider, verifier }), + } + } +} + +impl std::fmt::Debug for TlsBackendDefaults { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut s = f.debug_struct("TlsBackendDefaults"); + #[cfg(any(feature = "rustls", test))] + { + s.field( + "rustls", + &self.rustls.as_ref().map(|_| ""), + ); + } + s.finish() + } +} + +#[cfg(any(feature = "rustls", test))] +impl From<::rustls::ClientConfig> for TlsBackend { + fn from(config: ::rustls::ClientConfig) -> Self { + Self::Rustls(std::sync::Arc::new(config)) + } +} + +#[cfg(any(feature = "rustls", test))] +impl From> for TlsBackend { + fn from(config: std::sync::Arc<::rustls::ClientConfig>) -> Self { + Self::Rustls(config) + } +} + +#[cfg(any(feature = "native-tls", test))] +impl From<::native_tls::TlsConnector> for TlsBackend { + fn from(connector: ::native_tls::TlsConnector) -> Self { + Self::NativeTls(connector) + } +} diff --git a/crates/fetch_tls/src/client_identity.rs b/crates/fetch_tls/src/client_identity.rs new file mode 100644 index 000000000..46a2e4ff1 --- /dev/null +++ b/crates/fetch_tls/src/client_identity.rs @@ -0,0 +1,256 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Backend-agnostic `mTLS` client identity. +//! +//! [`ClientIdentity`] holds a DER-encoded certificate chain and private key +//! and is consumed by both backends, so `mTLS` is configured the same way +//! regardless of which backend is selected. + +use std::fmt; + +use rustls_pki_types::pem::PemObject; + +/// Error returned when constructing a [`ClientIdentity`] from key material fails. +#[ohno::error] +pub struct ClientIdentityError; + +/// Client identity for mutual TLS (`mTLS`) authentication. +/// +/// Holds the client certificate chain and private key presented during the +/// TLS handshake. The same value works with either backend; each one +/// converts the contained DER bytes into its own internal form. +/// +/// # Example +/// +/// ```rust,no_run +/// use fetch_tls::ClientIdentity; +/// +/// # fn example() -> Result<(), Box> { +/// let cert_pem = std::fs::read("client.pem")?; +/// let key_pem = std::fs::read("client-key.pem")?; +/// let identity = ClientIdentity::from_pem(&cert_pem, &key_pem)?; +/// # let _ = identity; +/// # Ok(()) +/// # } +/// ``` +pub struct ClientIdentity { + cert_chain: Vec>, + private_key: rustls_pki_types::PrivateKeyDer<'static>, +} + +impl Clone for ClientIdentity { + fn clone(&self) -> Self { + Self { + cert_chain: self.cert_chain.clone(), + // `PrivateKeyDer` does not implement `Clone` because the inner key + // material is sensitive; `clone_key` makes the copy explicit. + private_key: self.private_key.clone_key(), + } + } +} + +impl ClientIdentity { + /// Creates a client identity from PEM-encoded certificate and private key. + /// + /// `cert_pem` may contain one or more certificates (leaf first, then + /// intermediates). `key_pem` must contain exactly one private key + /// (`PKCS#1`, `PKCS#8`, or `SEC1`). + /// + /// The native-tls backend only accepts `PKCS#8` keys; `PKCS#1` and + /// `SEC1` work with rustls but cause native-tls to fail at build time. + /// + /// # Errors + /// + /// Returns an error if either input is not valid PEM. + pub fn from_pem(cert_pem: &[u8], key_pem: &[u8]) -> Result { + let cert_chain: Vec> = rustls_pki_types::CertificateDer::pem_slice_iter(cert_pem) + .collect::>() + .map_err(ClientIdentityError::caused_by)?; + let private_key = rustls_pki_types::PrivateKeyDer::from_pem_slice(key_pem).map_err(ClientIdentityError::caused_by)?; + Ok(Self { cert_chain, private_key }) + } + + /// Creates a client identity from DER-encoded certificate and private key. + /// + /// `cert_chain` is leaf-first; `key_der` must be a `PKCS#8`-encoded + /// private key. + #[must_use] + pub fn from_der(cert_chain: I, key_der: DER) -> Self + where + I: IntoIterator, + DER: AsRef<[u8]>, + { + let cert_chain = cert_chain + .into_iter() + .map(|c| rustls_pki_types::CertificateDer::from(c.as_ref().to_vec())) + .collect(); + let private_key = rustls_pki_types::PrivateKeyDer::from(rustls_pki_types::PrivatePkcs8KeyDer::from(key_der.as_ref().to_vec())); + Self { cert_chain, private_key } + } + + /// Returns the certificate chain as rustls types. + #[cfg(any(feature = "rustls", test))] + #[cfg_attr(test, mutants::skip)] // trivial accessor, tested via `mTLS` integration + pub(crate) fn cert_chain(&self) -> &[rustls_pki_types::CertificateDer<'static>] { + &self.cert_chain + } + + /// Returns the private key as rustls types. + #[cfg(any(feature = "rustls", test))] + pub(crate) fn private_key(&self) -> &rustls_pki_types::PrivateKeyDer<'static> { + &self.private_key + } + + /// Builds a [`native_tls::Identity`] from this client identity. + /// + /// Re-encodes the DER components as PEM and feeds them to + /// [`native_tls::Identity::from_pkcs8`], the format supported across all + /// platform backends. Fails if the private key is not `PKCS#8` or if the + /// platform native TLS implementation rejects the material. + #[cfg(any(feature = "native-tls", test))] + pub(crate) fn build_native_identity(&self) -> Result { + let key_pkcs8_der = match &self.private_key { + rustls_pki_types::PrivateKeyDer::Pkcs8(key) => key.secret_pkcs8_der(), + rustls_pki_types::PrivateKeyDer::Pkcs1(_) | rustls_pki_types::PrivateKeyDer::Sec1(_) => { + return Err(ClientIdentityError::caused_by( + "native-tls backend requires a `PKCS#8` private key (got `PKCS#1` or `SEC1`)", + )); + } + _ => { + return Err(ClientIdentityError::caused_by("native-tls backend requires a `PKCS#8` private key")); + } + }; + + let mut cert_pem = Vec::new(); + for cert in &self.cert_chain { + write_pem_block(&mut cert_pem, "CERTIFICATE", cert.as_ref()); + } + let mut key_pem = Vec::new(); + write_pem_block(&mut key_pem, "PRIVATE KEY", key_pkcs8_der); + + native_tls::Identity::from_pkcs8(&cert_pem, &key_pem).map_err(ClientIdentityError::caused_by) + } +} + +/// Writes a PEM-encoded object to `out`. +/// +/// Format per `RFC 7468`: a textual `-----BEGIN