Improve recursive DNS tunnel stability

crates/slipstream-client/src/dns/path.rs

@@ -11,11 +11,24 @@ use super::resolver::{reset_resolver_path, ResolverState};
 
 const PATH_PROBE_INITIAL_DELAY_US: u64 = 250_000;
 const PATH_PROBE_MAX_DELAY_US: u64 = 10_000_000;
+const PATH_PROBE_DISABLE_AFTER_ATTEMPTS: u32 = 5;
+const PATH_PROBE_DISABLE_US: u64 = 300_000_000;
 
 pub(crate) fn refresh_resolver_path(
     cnx: *mut picoquic_cnx_t,
     resolver: &mut ResolverState,
 ) -> bool {
+    let now = unsafe { picoquic_current_time() };
+    if resolver.disabled_until > now {
+        resolver.added = false;
+        resolver.path_id = -1;
+        resolver.unique_path_id = None;
+        resolver.local_addr_storage = None;
+        resolver.pending_polls = 0;
+        resolver.inflight_poll_ids.clear();
+        resolver.last_pacing_snapshot = None;
+        return false;
+    }
     if let Some(unique_path_id) = resolver.unique_path_id {
         let path_id = unsafe { slipstream_get_path_id_from_unique(cnx, unique_path_id) };
         if path_id >= 0 {
@@ -64,6 +77,9 @@ pub(crate) fn add_paths(
         if resolver.added {
             continue;
         }
+        if resolver.disabled_until > now {
+            continue;
+        }
         if resolver.next_probe_at > now {
             continue;
         }
@@ -90,12 +106,18 @@ pub(crate) fn add_paths(
             continue;
         }
         resolver.probe_attempts = resolver.probe_attempts.saturating_add(1);
-        let delay = path_probe_backoff(resolver.probe_attempts);
+        let attempt = resolver.probe_attempts;
+        let mut delay = path_probe_backoff(resolver.probe_attempts);
+        if resolver.probe_attempts >= PATH_PROBE_DISABLE_AFTER_ATTEMPTS {
+            resolver.disabled_until = now.saturating_add(PATH_PROBE_DISABLE_US);
+            resolver.probe_attempts = 0;
+            delay = PATH_PROBE_DISABLE_US;
+        }
         resolver.next_probe_at = now.saturating_add(delay);
         warn!(
             "Failed adding path {} (attempt {}), retrying in {}ms",
             resolver.addr,
-            resolver.probe_attempts,
+            attempt,
             delay / 1000
         );
     }

crates/slipstream-client/src/dns/poll.rs

@@ -1,6 +1,6 @@
 use crate::error::ClientError;
 use slipstream_core::net::is_transient_udp_error;
-use slipstream_dns::{build_qname, encode_query, QueryParams, CLASS_IN, RR_TXT};
+use slipstream_dns::{build_qname_with_nonce, encode_query, QueryParams, CLASS_IN, RR_TXT};
 use slipstream_ffi::picoquic::{
     picoquic_cnx_t, picoquic_current_time, picoquic_prepare_packet_ex, slipstream_request_poll,
 };
@@ -10,6 +10,7 @@ use tokio::net::UdpSocket as TokioUdpSocket;
 
 use super::path::refresh_resolver_path;
 use super::resolver::{sockaddr_storage_to_socket_addr, PeerAddrMode, ResolverState};
+use tracing::warn;
 
 const AUTHORITATIVE_POLL_TIMEOUT_US: u64 = 5_000_000;
 
@@ -87,8 +88,18 @@ pub(crate) async fn send_poll_queries(
         resolver.debug.polls_sent = resolver.debug.polls_sent.saturating_add(1);
 
         let poll_id = *dns_id;
-        let qname = build_qname(&send_buf[..send_length], config.domain)
-            .map_err(|err| ClientError::new(err.to_string()))?;
+        let qname = match build_qname_with_nonce(&send_buf[..send_length], config.domain, poll_id) {
+            Ok(qname) => qname,
+            Err(err) if err.to_string().contains("payload too large") => {
+                warn!(
+                    "Dropping oversized poll packet for DNS query transport: packet_len={} domain={}",
+                    send_length,
+                    config.domain
+                );
+                continue;
+            }
+            Err(err) => return Err(ClientError::new(err.to_string())),
+        };
         let params = QueryParams {
             id: poll_id,
             qname: &qname,

crates/slipstream-client/src/dns/resolver.rs

@@ -41,11 +41,19 @@ pub(crate) struct ResolverState {
     pub(crate) unique_path_id: Option<u64>,
     pub(crate) probe_attempts: u32,
     pub(crate) next_probe_at: u64,
+    pub(crate) disabled_until: u64,
+    pub(crate) last_health_check_at: u64,
+    pub(crate) last_health_send_packets: u64,
+    pub(crate) last_health_dns_responses: u64,
+    pub(crate) last_active_poll_kick_at: u64,
     pub(crate) pending_polls: usize,
     pub(crate) inflight_poll_ids: HashMap<u16, u64>,
     pub(crate) pacing_budget: Option<PacingPollBudget>,
     pub(crate) last_pacing_snapshot: Option<PacingBudgetSnapshot>,
     pub(crate) debug: DebugMetrics,
+    pub(crate) is_primary: bool,
+    pub(crate) path_loss_count: u32,
+    pub(crate) last_path_loss_at: u64,
 }
 
 impl ResolverState {
@@ -87,6 +95,11 @@ pub(crate) fn resolve_resolvers(
             unique_path_id: if is_primary { Some(0) } else { None },
             probe_attempts: 0,
             next_probe_at: 0,
+            disabled_until: 0,
+            last_health_check_at: 0,
+            last_health_send_packets: 0,
+            last_health_dns_responses: 0,
+            last_active_poll_kick_at: 0,
             pending_polls: 0,
             inflight_poll_ids: HashMap::new(),
             pacing_budget: match resolver.mode {
@@ -95,16 +108,43 @@ pub(crate) fn resolve_resolvers(
             },
             last_pacing_snapshot: None,
             debug: DebugMetrics::new(debug_poll),
+            is_primary,
+            path_loss_count: 0,
+            last_path_loss_at: 0,
         });
     }
     Ok(resolved)
 }
 
+const PATH_LOSS_WINDOW_US: u64 = 10_000_000;
+const PATH_LOSS_DISABLE_AFTER: u32 = 3;
+const PATH_LOSS_DISABLE_US: u64 = 300_000_000;
+
 pub(crate) fn reset_resolver_path(resolver: &mut ResolverState) {
     warn!(
         "Path for resolver {} became unavailable; resetting state",
         resolver.addr
     );
+    let now = unsafe { slipstream_ffi::picoquic::picoquic_current_time() };
+    if !resolver.is_primary {
+        if resolver.last_path_loss_at == 0
+            || now.saturating_sub(resolver.last_path_loss_at) > PATH_LOSS_WINDOW_US
+        {
+            resolver.path_loss_count = 1;
+        } else {
+            resolver.path_loss_count = resolver.path_loss_count.saturating_add(1);
+        }
+        resolver.last_path_loss_at = now;
+        if resolver.path_loss_count >= PATH_LOSS_DISABLE_AFTER {
+            resolver.disabled_until = now.saturating_add(PATH_LOSS_DISABLE_US);
+            resolver.path_loss_count = 0;
+            warn!(
+                "Path for resolver {} is flapping; cooling down for {}ms",
+                resolver.addr,
+                PATH_LOSS_DISABLE_US / 1000
+            );
+        }
+    }
     resolver.added = false;
     resolver.path_id = -1;
     resolver.unique_path_id = None;
@@ -114,6 +154,10 @@ pub(crate) fn reset_resolver_path(resolver: &mut ResolverState) {
     resolver.last_pacing_snapshot = None;
     resolver.probe_attempts = 0;
     resolver.next_probe_at = 0;
+    resolver.last_health_check_at = 0;
+    resolver.last_health_send_packets = 0;
+    resolver.last_health_dns_responses = 0;
+    resolver.last_active_poll_kick_at = 0;
 }
 
 pub(crate) fn sockaddr_storage_to_socket_addr(

crates/slipstream-client/src/main.rs

@@ -52,8 +52,15 @@ struct Args {
     domain: Option<String>,
     #[arg(long = "cert", value_name = "PATH")]
     cert: Option<String>,
-    #[arg(long = "keep-alive-interval", short = 't', default_value_t = 400)]
+    #[arg(
+        long = "keep-alive-interval",
+        short = 't',
+        default_value_t = 400,
+        help = "Send keep alive pings at this interval in milliseconds (disabled: 0)"
+    )]
     keep_alive_interval: u16,
+    #[arg(long = "quic-idle-timeout-seconds", default_value_t = 120)]
+    quic_idle_timeout_seconds: u64,
     #[arg(long = "debug-poll")]
     debug_poll: bool,
     #[arg(long = "debug-streams")]
@@ -170,6 +177,19 @@ fn main() {
         );
         keep_alive_override.unwrap_or(args.keep_alive_interval)
     };
+    let quic_idle_timeout_seconds = if cli_provided(&matches, "quic_idle_timeout_seconds") {
+        args.quic_idle_timeout_seconds
+    } else {
+        sip003::last_option_value(&sip003_env.plugin_options, "quic-idle-timeout-seconds")
+            .map(|value| {
+                unwrap_or_exit(
+                    parse_quic_idle_timeout_seconds(&value),
+                    "SIP003 env error",
+                    2,
+                )
+            })
+            .unwrap_or(args.quic_idle_timeout_seconds)
+    };
 
     let config = ClientConfig {
         tcp_listen_host: &tcp_listen_host,
@@ -180,6 +200,7 @@ fn main() {
         domain: &domain,
         cert: cert.as_deref(),
         keep_alive_interval: keep_alive_interval as usize,
+        quic_idle_timeout_seconds,
         debug_poll: args.debug_poll,
         debug_streams: args.debug_streams,
     };
@@ -344,6 +365,13 @@ fn parse_keep_alive_interval(options: &[sip003::Sip003Option]) -> Result<Option<
     Ok(last)
 }
 
+fn parse_quic_idle_timeout_seconds(value: &str) -> Result<u64, String> {
+    let trimmed = value.trim();
+    trimmed
+        .parse::<u64>()
+        .map_err(|_| format!("Invalid quic-idle-timeout-seconds value: {}", trimmed))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;