diff --git a/src/common/websocket.rs b/src/common/websocket.rs index 9e9e2091190d7..583fd9fe04a56 100644 --- a/src/common/websocket.rs +++ b/src/common/websocket.rs @@ -87,7 +87,7 @@ impl WebSocketConnector { } async fn tls_connect(&self) -> Result, WebSocketError> { - let ip = dns::Resolver + let ip = dns::Resolver::default() .lookup_ip(self.host.clone()) .await .context(DnsSnafu)? diff --git a/src/dns.rs b/src/dns.rs index 1329a284caf3a..15c405e71ec88 100644 --- a/src/dns.rs +++ b/src/dns.rs @@ -6,14 +6,42 @@ use std::{ use futures::{FutureExt, future::BoxFuture}; use hyper::client::connect::dns::Name; +use rand::Rng; use snafu::ResultExt; use tokio::task::spawn_blocking; use tower::Service; +use vector_lib::configurable::configurable_component; + +/// Controls how resolved DNS addresses are selected when multiple addresses are returned. +#[configurable_component] +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum DnsAddressSelection { + /// Use the first address returned by the system resolver (default). + #[default] + First, + /// Pick a single random address from the resolved set. + Random, +} pub struct LookupIp(std::vec::IntoIter); +/// The standard Vector DNS resolver. +/// +/// Resolves hostnames via the system's `getaddrinfo` and applies the configured +/// [`DnsAddressSelection`] policy to the returned addresses. #[derive(Debug, Clone, Copy)] -pub(super) struct Resolver; +pub struct Resolver { + pub selection: DnsAddressSelection, +} + +impl Default for Resolver { + fn default() -> Self { + Self { + selection: DnsAddressSelection::First, + } + } +} impl Resolver { pub(crate) async fn lookup_ip(self, name: String) -> Result { @@ -23,6 +51,7 @@ impl Resolver { // Any port will do, but `9` is a well defined port for discarding // packets. let dummy_port = 9; + let selection = self.selection; // https://tools.ietf.org/html/rfc6761#section-6.3 if name == "localhost" { // Not all operating systems support `localhost` as IPv6 `::1`, so @@ -31,7 +60,7 @@ impl Resolver { vec![SocketAddr::new(Ipv4Addr::LOCALHOST.into(), dummy_port)].into_iter(), )) } else { - spawn_blocking(move || { + let addrs = spawn_blocking(move || { // strip IPv6 prefix and suffix let name_str = name.as_str(); let name_ref = name_str @@ -42,8 +71,25 @@ impl Resolver { }) .await .context(JoinSnafu)? - .map(LookupIp) - .context(UnableLookupSnafu) + .context(UnableLookupSnafu)?; + + let mut addrs: Vec = addrs.collect(); + apply_selection(&mut addrs, selection); + Ok(LookupIp(addrs.into_iter())) + } + } +} + +fn apply_selection(addrs: &mut Vec, selection: DnsAddressSelection) { + match selection { + DnsAddressSelection::First => {} + DnsAddressSelection::Random => { + if !addrs.is_empty() { + let idx = rand::rng().random_range(0..addrs.len()); + let chosen = addrs[idx]; + addrs.clear(); + addrs.push(chosen); + } } } } @@ -70,6 +116,68 @@ impl Service for Resolver { } } +/// A resolver compatible with hyper's `HttpConnector` that yields `SocketAddr`. +/// +/// Unlike [`Resolver`] (which yields `IpAddr`), this resolver returns full socket +/// addresses and applies the configured [`DnsAddressSelection`] policy. It is designed +/// to be used with `HttpConnector::new_with_resolver()`. +#[derive(Debug, Clone, Copy)] +pub struct HyperResolver { + pub selection: DnsAddressSelection, +} + +/// Iterator over resolved socket addresses for [`HyperResolver`]. +pub struct SocketAddrIter(std::vec::IntoIter); + +impl Iterator for SocketAddrIter { + type Item = SocketAddr; + + fn next(&mut self) -> Option { + self.0.next() + } +} + +impl Service for HyperResolver { + type Response = SocketAddrIter; + type Error = DnsError; + type Future = BoxFuture<'static, Result>; + + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Ok(()).into() + } + + fn call(&mut self, name: Name) -> Self::Future { + let selection = self.selection; + async move { + let dummy_port = 0; + let name_string = name.as_str().to_owned(); + + if name_string == "localhost" { + return Ok(SocketAddrIter( + vec![SocketAddr::new(Ipv4Addr::LOCALHOST.into(), dummy_port)].into_iter(), + )); + } + + let addrs = spawn_blocking(move || { + let name_str = name_string.as_str(); + let name_ref = name_str + .strip_prefix('[') + .and_then(|s| s.strip_suffix(']')) + .unwrap_or(name_str); + (name_ref, dummy_port).to_socket_addrs() + }) + .await + .context(JoinSnafu)? + .context(UnableLookupSnafu)?; + + let mut addrs: Vec = addrs.collect(); + apply_selection(&mut addrs, selection); + Ok(SocketAddrIter(addrs.into_iter())) + } + .boxed() + } +} + #[derive(Debug, snafu::Snafu)] pub enum DnsError { #[snafu(display("Unable to resolve name: {}", source))] @@ -80,10 +188,12 @@ pub enum DnsError { #[cfg(test)] mod tests { - use super::Resolver; + use std::net::{Ipv4Addr, SocketAddr}; + + use super::{DnsAddressSelection, Resolver, apply_selection}; async fn resolve(name: &str) -> bool { - let resolver = Resolver; + let resolver = Resolver::default(); resolver.lookup_ip(name.to_owned()).await.is_ok() } @@ -106,4 +216,96 @@ mod tests { async fn resolve_ipv6() { assert!(resolve("::1").await); } + + #[test] + fn apply_selection_first_preserves_order() { + let mut addrs = vec![ + SocketAddr::new(Ipv4Addr::new(10, 0, 0, 1).into(), 80), + SocketAddr::new(Ipv4Addr::new(10, 0, 0, 2).into(), 80), + SocketAddr::new(Ipv4Addr::new(10, 0, 0, 3).into(), 80), + ]; + let original = addrs.clone(); + apply_selection(&mut addrs, DnsAddressSelection::First); + assert_eq!(addrs, original); + } + + #[test] + fn apply_selection_random_returns_single_address() { + let mut addrs = vec![ + SocketAddr::new(Ipv4Addr::new(10, 0, 0, 1).into(), 80), + SocketAddr::new(Ipv4Addr::new(10, 0, 0, 2).into(), 80), + SocketAddr::new(Ipv4Addr::new(10, 0, 0, 3).into(), 80), + ]; + let original = addrs.clone(); + apply_selection(&mut addrs, DnsAddressSelection::Random); + assert_eq!(addrs.len(), 1); + assert!(original.contains(&addrs[0])); + } + + #[test] + fn apply_selection_random_with_single_address() { + let mut addrs = vec![SocketAddr::new(Ipv4Addr::new(10, 0, 0, 1).into(), 80)]; + apply_selection(&mut addrs, DnsAddressSelection::Random); + assert_eq!(addrs.len(), 1); + assert_eq!(addrs[0], SocketAddr::new(Ipv4Addr::new(10, 0, 0, 1).into(), 80)); + } + + #[test] + fn apply_selection_random_with_empty_list() { + let mut addrs: Vec = vec![]; + apply_selection(&mut addrs, DnsAddressSelection::Random); + assert!(addrs.is_empty()); + } + + #[test] + fn apply_selection_random_distributes_across_addresses() { + let addrs = vec![ + SocketAddr::new(Ipv4Addr::new(10, 0, 0, 1).into(), 80), + SocketAddr::new(Ipv4Addr::new(10, 0, 0, 2).into(), 80), + SocketAddr::new(Ipv4Addr::new(10, 0, 0, 3).into(), 80), + ]; + + let mut seen = std::collections::HashSet::new(); + for _ in 0..100 { + let mut trial = addrs.clone(); + apply_selection(&mut trial, DnsAddressSelection::Random); + seen.insert(trial[0]); + } + assert!( + seen.len() > 1, + "random selection should pick different addresses across invocations" + ); + } + + #[tokio::test] + async fn resolver_with_random_selection_returns_single_ip() { + let resolver = Resolver { + selection: DnsAddressSelection::Random, + }; + let results: Vec<_> = resolver + .lookup_ip("localhost".to_owned()) + .await + .unwrap() + .collect(); + assert_eq!(results.len(), 1); + } + + #[test] + fn dns_address_selection_deserializes() { + #[derive(serde::Deserialize)] + struct Config { + selection: DnsAddressSelection, + } + + let config: Config = toml::from_str(r#"selection = "first""#).unwrap(); + assert_eq!(config.selection, DnsAddressSelection::First); + + let config: Config = toml::from_str(r#"selection = "random""#).unwrap(); + assert_eq!(config.selection, DnsAddressSelection::Random); + } + + #[test] + fn dns_address_selection_default_is_first() { + assert_eq!(DnsAddressSelection::default(), DnsAddressSelection::First); + } } diff --git a/src/sinks/util/service/net/tcp.rs b/src/sinks/util/service/net/tcp.rs index ac714cfd970bd..68ac8772272e5 100644 --- a/src/sinks/util/service/net/tcp.rs +++ b/src/sinks/util/service/net/tcp.rs @@ -67,7 +67,7 @@ impl TcpConnector { pub(super) async fn connect( &self, ) -> Result<(SocketAddr, MaybeTlsStream), NetError> { - let ip = dns::Resolver + let ip = dns::Resolver::default() .lookup_ip(self.address.host.clone()) .await .context(FailedToResolve)? diff --git a/src/sinks/util/service/net/udp.rs b/src/sinks/util/service/net/udp.rs index ef6cb4da4673d..1ce24e5a578c1 100644 --- a/src/sinks/util/service/net/udp.rs +++ b/src/sinks/util/service/net/udp.rs @@ -49,7 +49,7 @@ pub(super) struct UdpConnector { impl UdpConnector { pub(super) async fn connect(&self) -> Result { - let ip = dns::Resolver + let ip = dns::Resolver::default() .lookup_ip(self.address.host.clone()) .await .context(FailedToResolve)? diff --git a/src/sinks/util/tcp.rs b/src/sinks/util/tcp.rs index 3f4051a18c5f1..4473fd6c3af4d 100644 --- a/src/sinks/util/tcp.rs +++ b/src/sinks/util/tcp.rs @@ -162,7 +162,7 @@ impl TcpConnector { } async fn connect(&self) -> Result, TcpError> { - let ip = dns::Resolver + let ip = dns::Resolver::default() .lookup_ip(self.host.clone()) .await .context(DnsSnafu)? diff --git a/src/sinks/util/udp.rs b/src/sinks/util/udp.rs index 0b9441e682cde..aaa5cf4c41001 100644 --- a/src/sinks/util/udp.rs +++ b/src/sinks/util/udp.rs @@ -117,7 +117,7 @@ impl UdpConnector { } async fn connect(&self) -> Result { - let ip = dns::Resolver + let ip = dns::Resolver::default() .lookup_ip(self.host.clone()) .await .context(DnsSnafu)? diff --git a/src/sinks/vector/config.rs b/src/sinks/vector/config.rs index 9c0cf47e84a95..621e7b42e1f37 100644 --- a/src/sinks/vector/config.rs +++ b/src/sinks/vector/config.rs @@ -18,10 +18,12 @@ use tonic::body::BoxBody; use tower::{Service, ServiceBuilder}; use vector_lib::configurable::configurable_component; +use crate::dns::{DnsAddressSelection, HyperResolver}; + use super::{ VectorSinkError, compression::VectorCompression, - service::{VectorRequest, VectorResponse, VectorService}, + service::{VectorProxyConnector, VectorRequest, VectorResponse, VectorService}, sink::VectorSink, }; use crate::{ @@ -29,7 +31,6 @@ use crate::{ AcknowledgementsConfig, GenerateConfig, Input, ProxyConfig, SinkConfig, SinkContext, SinkHealthcheckOptions, }, - http::build_proxy_connector, proto::vector as proto, sinks::{ Healthcheck, VectorSink as VectorSinkType, @@ -116,6 +117,15 @@ pub struct VectorConfig { #[serde(default, skip_serializing_if = "Option::is_none")] keepalive: Option, + /// Controls how resolved DNS addresses are selected when a hostname resolves + /// to multiple addresses. + /// + /// - `first`: use the first address returned by the system resolver (default) + /// - `random`: pick a single random address from the resolved set + #[configurable(derived)] + #[serde(default)] + pub(in crate::sinks::vector) dns_address_selection: DnsAddressSelection, + #[configurable(derived)] #[serde( default, @@ -218,6 +228,7 @@ fn default_config(address: &str) -> VectorConfig { request: TowerRequestConfig::default(), tls: None, keepalive: None, + dns_address_selection: DnsAddressSelection::default(), acknowledgements: Default::default(), } } @@ -233,7 +244,7 @@ impl SinkConfig for VectorConfig { .as_ref() .map_or_else(EndpointStrategy::default, |routing| routing.strategy); - let client = new_client(&tls, cx.proxy(), self.keepalive)?; + let client = new_client(&tls, cx.proxy(), self.keepalive, self.dns_address_selection)?; let healthcheck = healthchecks(client.clone(), &uris, cx.healthcheck, endpoint_strategy); let request_settings = self.request.into_settings(); @@ -725,7 +736,7 @@ async fn healthcheck( } fn healthchecks( - client: hyper::Client>, BoxBody>, + client: hyper::Client, uris: &[Uri], options: SinkHealthcheckOptions, endpoint_strategy: EndpointStrategy, @@ -826,8 +837,9 @@ fn new_client( tls_settings: &MaybeTlsSettings, proxy_config: &ProxyConfig, keepalive: Option, -) -> crate::Result>, BoxBody>> { - let proxy = build_proxy_connector(tls_settings.clone(), proxy_config)?; + dns_address_selection: DnsAddressSelection, +) -> crate::Result> { + let proxy = build_vector_proxy_connector(tls_settings.clone(), proxy_config, dns_address_selection)?; let mut builder = hyper::Client::builder(); builder.http2_only(true); @@ -846,6 +858,38 @@ fn new_client( Ok(builder.build(proxy)) } +fn build_vector_proxy_connector( + tls_settings: MaybeTlsSettings, + proxy_config: &ProxyConfig, + dns_address_selection: DnsAddressSelection, +) -> crate::Result { + use crate::tls::tls_connector_builder; + + let resolver = HyperResolver { + selection: dns_address_selection, + }; + let mut http = HttpConnector::new_with_resolver(resolver); + http.enforce_http(false); + + let tls = tls_connector_builder(&tls_settings)?; + let mut https = HttpsConnector::with_connector(http, tls)?; + + let settings = tls_settings.tls().cloned(); + https.set_callback(move |c, _uri| { + if let Some(settings) = &settings { + settings.apply_connect_configuration(c) + } else { + Ok(()) + } + }); + + let tls_for_proxy = tls_connector_builder(&tls_settings)?.build(); + let mut proxy = ProxyConnector::new(https).unwrap(); + proxy.set_tls(Some(tls_for_proxy)); + proxy_config.configure(&mut proxy)?; + Ok(proxy) +} + #[derive(Debug, Clone)] struct VectorGrpcRetryLogic; @@ -1434,4 +1478,31 @@ mod tests { vec![endpoints[0].clone()] ); } + + #[test] + fn parse_dns_address_selection_random() { + let config: VectorConfig = toml::from_str( + r#" + address = "http://127.0.0.1:6000" + dns_address_selection = "random" + "#, + ) + .unwrap(); + + assert_eq!( + config.dns_address_selection, + crate::dns::DnsAddressSelection::Random + ); + } + + #[test] + fn dns_address_selection_defaults_to_first() { + let config: VectorConfig = + toml::from_str(r#"address = "http://127.0.0.1:6000""#).unwrap(); + + assert_eq!( + config.dns_address_selection, + crate::dns::DnsAddressSelection::First + ); + } } diff --git a/src/sinks/vector/service.rs b/src/sinks/vector/service.rs index 9c3d09fb0e541..758fa83fe5934 100644 --- a/src/sinks/vector/service.rs +++ b/src/sinks/vector/service.rs @@ -16,12 +16,15 @@ use vector_lib::{ use super::{VectorSinkError, compression::VectorCompression}; use crate::{ Error, + dns::HyperResolver, event::{EventFinalizers, EventStatus, Finalizable}, internal_events::EndpointBytesSent, proto::vector as proto_vector, sinks::util::uri, }; +pub type VectorHttpConnector = HttpConnector; + #[derive(Clone, Debug)] pub struct VectorService { pub client: proto_vector::Client, @@ -29,6 +32,8 @@ pub struct VectorService { pub endpoint: String, } +pub type VectorProxyConnector = ProxyConnector>; + pub struct VectorResponse { events_byte_size: GroupedCountByteSize, } @@ -68,7 +73,7 @@ impl MetaDescriptive for VectorRequest { impl VectorService { pub fn new( - hyper_client: hyper::Client>, BoxBody>, + hyper_client: hyper::Client, uri: Uri, compression: VectorCompression, ) -> Self { @@ -136,7 +141,7 @@ impl Service for VectorService { #[derive(Clone, Debug)] pub struct HyperSvc { uri: Uri, - client: hyper::Client>, BoxBody>, + client: hyper::Client, } impl Service> for HyperSvc {