From bdc1f19d1cea2bb1a1cad8eaa840c4eac656c3d3 Mon Sep 17 00:00:00 2001 From: Nic Borensztein Date: Mon, 16 Mar 2026 10:41:13 -0700 Subject: [PATCH] fix(sandbox): add DNS forwarder for sandbox network namespace The sandbox runs in an isolated network namespace where the only route is to the host side of a veth pair (10.200.0.1). The pod resolv.conf points to CoreDNS (10.43.0.10), which is unreachable from the sandbox netns because kube-proxy iptables rules only exist in the pod original netns. Applications that honor HTTP_PROXY route through the CONNECT proxy (which resolves DNS on the host side), but WebSocket clients and other tools that resolve DNS directly fail with getaddrinfo EAI_AGAIN. This adds a lightweight UDP DNS forwarder that: - Listens on the host veth IP port 53 (reachable from sandbox) - Forwards queries to the pod original upstream DNS server - Overwrites /etc/resolv.conf to point at the forwarder The proxy still enforces network policy on TCP connections. DNS resolution alone does not grant network access. Tested with OpenClaw sandbox: Discord WebSocket gateway now connects successfully where it previously failed with EAI_AGAIN. Signed-off-by: Nic Borensztein --- crates/openshell-sandbox/src/dns_forwarder.rs | 191 ++++++++++++++++++ crates/openshell-sandbox/src/lib.rs | 32 +++ 2 files changed, 223 insertions(+) create mode 100644 crates/openshell-sandbox/src/dns_forwarder.rs diff --git a/crates/openshell-sandbox/src/dns_forwarder.rs b/crates/openshell-sandbox/src/dns_forwarder.rs new file mode 100644 index 00000000..d85e7009 --- /dev/null +++ b/crates/openshell-sandbox/src/dns_forwarder.rs @@ -0,0 +1,191 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Lightweight DNS forwarder for sandbox network namespaces. +//! +//! The sandbox runs in an isolated network namespace where the only route is +//! to the host side of a veth pair (e.g. `10.200.0.1`). The pod's +//! `/etc/resolv.conf` points to CoreDNS (`10.43.0.10`), which is unreachable +//! from the sandbox netns. +//! +//! This module provides a UDP DNS forwarder that: +//! 1. Listens on `:53` (reachable from the sandbox) +//! 2. Forwards queries to the pod's original upstream DNS server +//! 3. Relays responses back to the client +//! +//! After starting, the sandbox's `/etc/resolv.conf` is overwritten to point +//! at the forwarder. This allows applications that resolve DNS directly +//! (bypassing `HTTP_PROXY`) to function — the proxy still enforces network +//! policy on the subsequent TCP connections. + +use miette::{IntoDiagnostic, Result}; +use std::net::{IpAddr, SocketAddr}; +use std::time::Duration; +use tokio::net::UdpSocket; +use tokio::task::JoinHandle; +use tracing::{debug, info, warn}; + +/// Handle to a running DNS forwarder. The forwarder runs until dropped. +pub struct DnsForwarderHandle { + _task: JoinHandle<()>, +} + +/// Maximum DNS UDP message size (EDNS0 allows up to 4096). +const MAX_DNS_PACKET: usize = 4096; + +/// Timeout for upstream DNS responses. +const UPSTREAM_TIMEOUT: Duration = Duration::from_secs(5); + +impl DnsForwarderHandle { + /// Start a DNS forwarder on `listen_ip:53` that forwards to `upstream`. + /// + /// # Errors + /// + /// Returns an error if the UDP socket cannot be bound (e.g. port 53 is + /// already in use or the process lacks `CAP_NET_BIND_SERVICE`). + pub async fn start(listen_ip: IpAddr, upstream: SocketAddr) -> Result { + let listen_addr = SocketAddr::new(listen_ip, 53); + let socket = UdpSocket::bind(listen_addr).await.into_diagnostic()?; + info!( + listen = %listen_addr, + upstream = %upstream, + "DNS forwarder started" + ); + + let task = tokio::spawn(async move { + run_forwarder(socket, upstream).await; + }); + + Ok(Self { _task: task }) + } +} + +/// Main forwarder loop: receive a query, spawn a task to forward it upstream +/// and relay the response. +async fn run_forwarder(socket: UdpSocket, upstream: SocketAddr) { + let socket = std::sync::Arc::new(socket); + let mut buf = [0u8; MAX_DNS_PACKET]; + + loop { + let (len, client_addr) = match socket.recv_from(&mut buf).await { + Ok(result) => result, + Err(e) => { + warn!(error = %e, "DNS forwarder recv_from failed"); + continue; + } + }; + + let query = buf[..len].to_vec(); + let socket = socket.clone(); + + // Spawn a task per query so concurrent lookups don't block each other. + tokio::spawn(async move { + match forward_query(&query, upstream).await { + Ok(response) => { + if let Err(e) = socket.send_to(&response, client_addr).await { + debug!(error = %e, client = %client_addr, "Failed to send DNS response"); + } + } + Err(e) => { + debug!(error = %e, client = %client_addr, "DNS upstream query failed"); + } + } + }); + } +} + +/// Forward a single DNS query to the upstream server and return the response. +async fn forward_query( + query: &[u8], + upstream: SocketAddr, +) -> std::result::Result, String> { + let sock = UdpSocket::bind("0.0.0.0:0") + .await + .map_err(|e| format!("bind ephemeral socket: {e}"))?; + + sock.send_to(query, upstream) + .await + .map_err(|e| format!("send to upstream: {e}"))?; + + let mut buf = [0u8; MAX_DNS_PACKET]; + let len = tokio::time::timeout(UPSTREAM_TIMEOUT, sock.recv(&mut buf)) + .await + .map_err(|_| "upstream DNS timeout".to_string())? + .map_err(|e| format!("recv from upstream: {e}"))?; + + Ok(buf[..len].to_vec()) +} + +/// Read the first `nameserver` entry from `/etc/resolv.conf`. +/// +/// Must be called **before** [`override_resolv_conf`] overwrites the file. +pub fn read_upstream_nameserver() -> Result { + let contents = std::fs::read_to_string("/etc/resolv.conf").into_diagnostic()?; + parse_first_nameserver(&contents) + .ok_or_else(|| miette::miette!("No nameserver found in /etc/resolv.conf")) +} + +/// Parse the first `nameserver` IP from resolv.conf content. +fn parse_first_nameserver(contents: &str) -> Option { + for line in contents.lines() { + let line = line.trim(); + if let Some(rest) = line.strip_prefix("nameserver") { + if let Ok(ip) = rest.trim().parse::() { + return Some(SocketAddr::new(ip, 53)); + } + } + } + None +} + +/// Overwrite `/etc/resolv.conf` to point at the DNS forwarder. +/// +/// Preserves `search` and `options` directives from the original file, +/// replacing only the `nameserver` lines. +pub fn override_resolv_conf(forwarder_ip: IpAddr) -> Result<()> { + let original = std::fs::read_to_string("/etc/resolv.conf").unwrap_or_default(); + + let mut lines: Vec = original + .lines() + .filter(|l| !l.trim_start().starts_with("nameserver")) + .map(String::from) + .collect(); + + lines.push(format!("nameserver {forwarder_ip}")); + + std::fs::write("/etc/resolv.conf", lines.join("\n") + "\n").into_diagnostic()?; + info!(forwarder = %forwarder_ip, "Overwrote /etc/resolv.conf"); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_nameserver_standard() { + let content = "search cluster.local\nnameserver 10.43.0.10\noptions ndots:5\n"; + let addr = parse_first_nameserver(content).unwrap(); + assert_eq!(addr, SocketAddr::new("10.43.0.10".parse().unwrap(), 53)); + } + + #[test] + fn parse_nameserver_multiple() { + let content = "nameserver 10.43.0.10\nnameserver 8.8.8.8\n"; + let addr = parse_first_nameserver(content).unwrap(); + assert_eq!(addr, SocketAddr::new("10.43.0.10".parse().unwrap(), 53)); + } + + #[test] + fn parse_nameserver_missing() { + let content = "search cluster.local\noptions ndots:5\n"; + assert!(parse_first_nameserver(content).is_none()); + } + + #[test] + fn parse_nameserver_with_comments() { + let content = "# comment\nnameserver 127.0.0.53\n"; + let addr = parse_first_nameserver(content).unwrap(); + assert_eq!(addr, SocketAddr::new("127.0.0.53".parse().unwrap(), 53)); + } +} diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 754c3be0..1441d105 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -8,6 +8,7 @@ pub mod bypass_monitor; mod child_env; pub mod denial_aggregator; +mod dns_forwarder; mod grpc_client; mod identity; pub mod l7; @@ -293,6 +294,37 @@ pub async fn run_sandbox( #[allow(clippy::no_effect_underscore_binding)] let _netns: Option<()> = None; + // Start DNS forwarder on the host side of the veth so sandbox processes + // can resolve DNS. The sandbox netns inherits /etc/resolv.conf from the + // pod (pointing to CoreDNS on the K3s service network), but that address + // is unreachable from the isolated netns. The forwarder listens on the + // host veth IP and relays queries to the original upstream DNS server. + #[cfg(target_os = "linux")] + let _dns_forwarder = if let Some(ref ns) = netns { + match dns_forwarder::read_upstream_nameserver() { + Ok(upstream) => match dns_forwarder::DnsForwarderHandle::start(ns.host_ip(), upstream) + .await + { + Ok(handle) => { + if let Err(e) = dns_forwarder::override_resolv_conf(ns.host_ip()) { + warn!(error = %e, "Failed to override /etc/resolv.conf, DNS may not work in sandbox"); + } + Some(handle) + } + Err(e) => { + warn!(error = %e, "Failed to start DNS forwarder, DNS may not work in sandbox"); + None + } + }, + Err(e) => { + warn!(error = %e, "Failed to read upstream nameserver, DNS forwarder not started"); + None + } + } + } else { + None + }; + // Shared PID: set after process spawn so the proxy can look up // the entrypoint process's /proc/net/tcp for identity binding. let entrypoint_pid = Arc::new(AtomicU32::new(0));