Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions architecture/gateway-single-node.md
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,7 @@ For the target daemon (local or remote):
- Volume bind mount: `openshell-cluster-{name}:/var/lib/rancher/k3s`.
- Network: `openshell-cluster-{name}` (per-gateway bridge network).
- Extra host: `host.docker.internal:host-gateway`.
- The cluster entrypoint prefers the resolved IPv4 for `host.docker.internal` when populating sandbox pod `hostAliases`, then falls back to the container default gateway. This keeps sandbox host aliases working on Docker Desktop, where the host-reachable IP differs from the bridge gateway.
- Port mappings:

| Container Port | Host Port | Purpose |
Expand Down
15 changes: 15 additions & 0 deletions crates/openshell-core/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,13 @@ pub struct Config {
/// the server over mTLS.
#[serde(default)]
pub client_tls_secret_name: String,

/// Host gateway IP for sandbox pod hostAliases.
/// When set, sandbox pods get hostAliases entries mapping
/// `host.docker.internal` and `host.openshell.internal` to this IP,
/// allowing them to reach services running on the Docker host.
#[serde(default)]
pub host_gateway_ip: String,
}

/// TLS configuration.
Expand Down Expand Up @@ -125,6 +132,7 @@ impl Config {
ssh_handshake_skew_secs: default_ssh_handshake_skew_secs(),
ssh_session_ttl_secs: default_ssh_session_ttl_secs(),
client_tls_secret_name: String::new(),
host_gateway_ip: String::new(),
}
}

Expand Down Expand Up @@ -232,6 +240,13 @@ impl Config {
self.client_tls_secret_name = name.into();
self
}

/// Set the host gateway IP for sandbox pod hostAliases.
#[must_use]
pub fn with_host_gateway_ip(mut self, ip: impl Into<String>) -> Self {
self.host_gateway_ip = ip.into();
self
}
}

fn default_bind_address() -> SocketAddr {
Expand Down
1 change: 1 addition & 0 deletions crates/openshell-server/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ pub async fn run_server(config: Config, tracing_log_bus: TracingLogBus) -> Resul
config.ssh_handshake_secret.clone(),
config.ssh_handshake_skew_secs,
config.client_tls_secret_name.clone(),
config.host_gateway_ip.clone(),
)
.await
.map_err(|e| Error::execution(format!("failed to create kubernetes client: {e}")))?;
Expand Down
10 changes: 10 additions & 0 deletions crates/openshell-server/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,12 @@ struct Args {
#[arg(long, env = "OPENSHELL_CLIENT_TLS_SECRET_NAME")]
client_tls_secret_name: Option<String>,

/// Host gateway IP for sandbox pod hostAliases.
/// When set, sandbox pods get hostAliases entries mapping
/// host.docker.internal and host.openshell.internal to this IP.
#[arg(long, env = "OPENSHELL_HOST_GATEWAY_IP")]
host_gateway_ip: Option<String>,

/// Disable TLS entirely — listen on plaintext HTTP.
/// Use this when the gateway sits behind a reverse proxy or tunnel
/// (e.g. Cloudflare Tunnel) that terminates TLS at the edge.
Expand Down Expand Up @@ -178,6 +184,10 @@ async fn main() -> Result<()> {
config = config.with_client_tls_secret_name(name);
}

if let Some(ip) = args.host_gateway_ip {
config = config.with_host_gateway_ip(ip);
}

if args.disable_tls {
info!("TLS disabled — listening on plaintext HTTP");
} else if args.disable_gateway_auth {
Expand Down
149 changes: 149 additions & 0 deletions crates/openshell-server/src/sandbox/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,9 @@ pub struct SandboxClient {
ssh_handshake_skew_secs: u64,
/// When non-empty, sandbox pods get this K8s secret mounted for mTLS to the server.
client_tls_secret_name: String,
/// When non-empty, sandbox pods get `hostAliases` entries mapping
/// `host.docker.internal` and `host.openshell.internal` to this IP.
host_gateway_ip: String,
}

impl std::fmt::Debug for SandboxClient {
Expand All @@ -71,6 +74,7 @@ impl SandboxClient {
ssh_handshake_secret: String,
ssh_handshake_skew_secs: u64,
client_tls_secret_name: String,
host_gateway_ip: String,
) -> Result<Self, KubeError> {
let mut config = match kube::Config::incluster() {
Ok(c) => c,
Expand All @@ -92,6 +96,7 @@ impl SandboxClient {
ssh_handshake_secret,
ssh_handshake_skew_secs,
client_tls_secret_name,
host_gateway_ip,
})
}

Expand Down Expand Up @@ -206,6 +211,7 @@ impl SandboxClient {
self.ssh_handshake_secret(),
self.ssh_handshake_skew_secs(),
&self.client_tls_secret_name,
&self.host_gateway_ip,
);
let api = self.api();

Expand Down Expand Up @@ -759,6 +765,7 @@ fn sandbox_to_k8s_spec(
ssh_handshake_secret: &str,
ssh_handshake_skew_secs: u64,
client_tls_secret_name: &str,
host_gateway_ip: &str,
) -> serde_json::Value {
let mut root = serde_json::Map::new();
if let Some(spec) = spec {
Expand Down Expand Up @@ -787,6 +794,7 @@ fn sandbox_to_k8s_spec(
ssh_handshake_skew_secs,
&spec.environment,
client_tls_secret_name,
host_gateway_ip,
),
);
if !template.agent_socket.is_empty() {
Expand Down Expand Up @@ -820,6 +828,7 @@ fn sandbox_to_k8s_spec(
ssh_handshake_skew_secs,
spec_env,
client_tls_secret_name,
host_gateway_ip,
),
);
}
Expand All @@ -843,6 +852,7 @@ fn sandbox_template_to_k8s(
ssh_handshake_skew_secs: u64,
spec_environment: &std::collections::HashMap<String, String>,
client_tls_secret_name: &str,
host_gateway_ip: &str,
) -> serde_json::Value {
if let Some(pod_template) = struct_to_json(&template.pod_template) {
return inject_pod_template(
Expand All @@ -859,6 +869,7 @@ fn sandbox_template_to_k8s(
ssh_handshake_skew_secs,
spec_environment,
client_tls_secret_name,
host_gateway_ip,
);
}

Expand Down Expand Up @@ -968,6 +979,17 @@ fn sandbox_template_to_k8s(
);
}

// Add hostAliases so sandbox pods can reach the Docker host.
if !host_gateway_ip.is_empty() {
spec.insert(
"hostAliases".to_string(),
serde_json::json!([{
"ip": host_gateway_ip,
"hostnames": ["host.docker.internal", "host.openshell.internal"]
}]),
);
}

let mut template_value = serde_json::Map::new();
if !metadata.is_empty() {
template_value.insert("metadata".to_string(), serde_json::Value::Object(metadata));
Expand Down Expand Up @@ -997,6 +1019,7 @@ fn inject_pod_template(
ssh_handshake_skew_secs: u64,
spec_environment: &std::collections::HashMap<String, String>,
client_tls_secret_name: &str,
host_gateway_ip: &str,
) -> serde_json::Value {
let Some(spec) = pod_template
.get_mut("spec")
Expand All @@ -1012,6 +1035,17 @@ fn inject_pod_template(
);
}

// Add hostAliases so sandbox pods can reach the Docker host.
if !host_gateway_ip.is_empty() {
spec.insert(
"hostAliases".to_string(),
serde_json::json!([{
"ip": host_gateway_ip,
"hostnames": ["host.docker.internal", "host.openshell.internal"]
}]),
);
}

// Inject TLS volume at the pod spec level.
if !client_tls_secret_name.is_empty() {
let volumes = spec
Expand Down Expand Up @@ -1806,6 +1840,7 @@ mod tests {
300,
&std::collections::HashMap::new(),
"",
"",
);

assert_eq!(
Expand Down Expand Up @@ -1851,6 +1886,7 @@ mod tests {
300,
&std::collections::HashMap::new(),
"",
"",
);

let limits = &pod_template["spec"]["containers"][0]["resources"]["limits"];
Expand Down Expand Up @@ -1910,6 +1946,7 @@ mod tests {
300,
&std::collections::HashMap::new(),
"",
"",
);

assert_eq!(
Expand All @@ -1921,4 +1958,116 @@ mod tests {
serde_json::json!(GPU_RESOURCE_QUANTITY)
);
}

#[test]
fn host_aliases_injected_when_gateway_ip_set() {
let pod_template = sandbox_template_to_k8s(
&SandboxTemplate::default(),
false,
"openshell/sandbox:latest",
"",
"sandbox-id",
"sandbox-name",
"https://gateway.example.com",
"0.0.0.0:2222",
"secret",
300,
&std::collections::HashMap::new(),
"",
"172.17.0.1",
);

let host_aliases = pod_template["spec"]["hostAliases"]
.as_array()
.expect("hostAliases should exist");
assert_eq!(host_aliases.len(), 1);
assert_eq!(host_aliases[0]["ip"], "172.17.0.1");
let hostnames = host_aliases[0]["hostnames"]
.as_array()
.expect("hostnames should exist");
assert!(hostnames.contains(&serde_json::json!("host.docker.internal")));
assert!(hostnames.contains(&serde_json::json!("host.openshell.internal")));
}

#[test]
fn host_aliases_not_injected_when_gateway_ip_empty() {
let pod_template = sandbox_template_to_k8s(
&SandboxTemplate::default(),
false,
"openshell/sandbox:latest",
"",
"sandbox-id",
"sandbox-name",
"https://gateway.example.com",
"0.0.0.0:2222",
"secret",
300,
&std::collections::HashMap::new(),
"",
"",
);

assert!(
pod_template["spec"]["hostAliases"].is_null(),
"hostAliases should not be present when host_gateway_ip is empty"
);
}

#[test]
fn host_aliases_injected_in_custom_pod_template() {
let template = SandboxTemplate {
pod_template: Some(Struct {
fields: [(
"spec".to_string(),
Value {
kind: Some(Kind::StructValue(Struct {
fields: [(
"containers".to_string(),
Value {
kind: Some(Kind::ListValue(prost_types::ListValue {
values: vec![Value {
kind: Some(Kind::StructValue(Struct {
fields: [(
"name".to_string(),
string_value("agent"),
)]
.into_iter()
.collect(),
})),
}],
})),
},
)]
.into_iter()
.collect(),
})),
},
)]
.into_iter()
.collect(),
}),
..SandboxTemplate::default()
};

let pod_template = sandbox_template_to_k8s(
&template,
false,
"openshell/sandbox:latest",
"",
"sandbox-id",
"sandbox-name",
"https://gateway.example.com",
"0.0.0.0:2222",
"secret",
300,
&std::collections::HashMap::new(),
"",
"192.168.65.2",
);

let host_aliases = pod_template["spec"]["hostAliases"]
.as_array()
.expect("hostAliases should exist in custom pod template");
assert_eq!(host_aliases[0]["ip"], "192.168.65.2");
}
}
31 changes: 31 additions & 0 deletions deploy/docker/cluster-entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -330,6 +330,27 @@ if [ "${GPU_ENABLED:-}" = "true" ]; then
fi
fi

# ---------------------------------------------------------------------------
# Detect host gateway IP for sandbox pod hostAliases
# ---------------------------------------------------------------------------
# Sandbox pods need to reach services running on the Docker host (e.g.
# provider endpoints during local development). On Docker Desktop,
# host.docker.internal resolves to a special host-reachable IP that is NOT the
# bridge default gateway, so prefer Docker's own resolution when available.
# Fall back to the container default gateway on Linux engines where
# host.docker.internal commonly maps to the bridge gateway anyway.
HOST_GATEWAY_IP=$(getent ahostsv4 host.docker.internal 2>/dev/null | awk 'NR == 1 { print $1; exit }')
if [ -n "$HOST_GATEWAY_IP" ]; then
echo "Detected host gateway IP from host.docker.internal: $HOST_GATEWAY_IP"
else
HOST_GATEWAY_IP=$(ip -4 route | awk '/default/ { print $3; exit }')
if [ -n "$HOST_GATEWAY_IP" ]; then
echo "Detected host gateway IP from default route: $HOST_GATEWAY_IP"
else
echo "Warning: Could not detect host gateway IP from host.docker.internal or default route"
fi
fi

# ---------------------------------------------------------------------------
# Override image tag and pull policy for local development
# ---------------------------------------------------------------------------
Expand Down Expand Up @@ -428,6 +449,16 @@ if [ -f "$HELMCHART" ]; then
fi
fi

# Inject host gateway IP into the HelmChart manifest so sandbox pods can
# reach services on the Docker host via host.docker.internal / host.openshell.internal.
if [ -n "$HOST_GATEWAY_IP" ] && [ -f "$HELMCHART" ]; then
echo "Setting host gateway IP: $HOST_GATEWAY_IP"
sed -i "s|__HOST_GATEWAY_IP__|${HOST_GATEWAY_IP}|g" "$HELMCHART"
else
# Clear the placeholder so the server gets an empty string (feature disabled)
sed -i "s|hostGatewayIP: __HOST_GATEWAY_IP__|hostGatewayIP: \"\"|g" "$HELMCHART"
fi

# Inject chart checksum into the HelmChart manifest so that a changed chart
# tarball causes the HelmChart CR spec to differ, forcing the k3s Helm
# controller to upgrade the release.
Expand Down
4 changes: 4 additions & 0 deletions deploy/helm/openshell/templates/statefulset.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,10 @@ spec:
- name: OPENSHELL_SSH_GATEWAY_PORT
value: {{ .Values.server.sshGatewayPort | quote }}
{{- end }}
{{- if .Values.server.hostGatewayIP }}
- name: OPENSHELL_HOST_GATEWAY_IP
value: {{ .Values.server.hostGatewayIP | quote }}
{{- end }}
- name: OPENSHELL_SSH_HANDSHAKE_SECRET
value: {{ required "server.sshHandshakeSecret is required" .Values.server.sshHandshakeSecret | quote }}
{{- if .Values.server.disableTls }}
Expand Down
Loading
Loading