Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 10 additions & 11 deletions blackbox/harness/cloud.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,8 @@ type CloudEnv struct {

// NewCloudEnv builds a full cloud+POP environment for testing the global router.
// It builds cloud/POP binaries, starts cloud, registers a POP and cluster,
// starts POP, and restarts the miren server with --labs globalrouter.
// The environment is torn down via t.Cleanup.
// starts POP, and restarts the miren server so it picks up the new
// registration.json. The environment is torn down via t.Cleanup.
func NewCloudEnv(t *testing.T, m *Miren) *CloudEnv {
t.Helper()

Expand Down Expand Up @@ -98,8 +98,8 @@ func NewCloudEnv(t *testing.T, m *Miren) *CloudEnv {
// Start POP
env.startPOP(t)

// Restart miren server with globalrouter enabled
env.restartServerWithGlobalRouter(t)
// Restart miren server so it picks up the registration
env.restartServerWithRegistration(t)

// Wait for global router to connect
env.waitForConnection(t)
Expand Down Expand Up @@ -442,9 +442,9 @@ func (env *CloudEnv) startPOP(t *testing.T) {
t.Log("POP server ready")
}

func (env *CloudEnv) restartServerWithGlobalRouter(t *testing.T) {
func (env *CloudEnv) restartServerWithRegistration(t *testing.T) {
t.Helper()
t.Log("restarting miren server with --labs globalrouter...")
t.Log("restarting miren server to pick up registration...")

// Stop current server
env.m.RunCmdAsRoot("bash", "-c", "hack/dev-server stop")
Expand All @@ -453,13 +453,12 @@ func (env *CloudEnv) restartServerWithGlobalRouter(t *testing.T) {
// old "Miren server started" line from the previous startup.
env.m.RunCmdAsRoot("bash", "-c", ": > /tmp/miren-server.log")

// Start with globalrouter lab flag
env.m.RunCmdAsRoot("bash", "-c", "DEV_SERVER_FLAGS='--labs globalrouter' hack/dev-server start")
env.m.RunCmdAsRoot("bash", "-c", "hack/dev-server start")

// Wait for server to be ready
r := env.m.RunCmdAsRoot("hack/dev-server", "wait-ready", "60")
if !r.Success() {
t.Fatalf("server failed to start with globalrouter: %s", r.Stderr)
t.Fatalf("server failed to start: %s", r.Stderr)
}

// Also wait for buildkit's hosts file to be updated with the registry IP,
Expand All @@ -474,14 +473,14 @@ func (env *CloudEnv) restartServerWithGlobalRouter(t *testing.T) {
})

t.Cleanup(func() {
// Restart server without globalrouter to restore original state
// Restart server after registration files are removed to restore original state
env.m.RunCmdAsRoot("bash", "-c", "hack/dev-server stop")
env.m.RunCmdAsRoot("bash", "-c", ": > /tmp/miren-server.log")
env.m.RunCmdAsRoot("bash", "-c", "hack/dev-server start")
env.m.RunCmdAsRoot("hack/dev-server", "wait-ready", "30")
})

t.Log("miren server restarted with globalrouter enabled")
t.Log("miren server restarted with registration loaded")
}

func (env *CloudEnv) waitForConnection(t *testing.T) {
Expand Down
4 changes: 2 additions & 2 deletions components/coordinate/coordinate.go
Original file line number Diff line number Diff line change
Expand Up @@ -1176,8 +1176,8 @@ func (c *Coordinator) Start(ctx context.Context) error {
go c.reportStatusPeriodically(ctx)
}

// Start global router for NAT traversal when enabled
if labs.GlobalRouter() && c.CloudAuth.Enabled && c.authClient != nil {
// Start global router for NAT traversal when cloud auth is configured
if c.CloudAuth.Enabled && c.authClient != nil {
cloudURL := c.CloudAuth.CloudURL
if cloudURL == "" {
cloudURL = DefaultCloudURL
Expand Down
60 changes: 60 additions & 0 deletions docs/docs/miren-cloud/global-router.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@

# Global Router

The global router is the connection between your cluster and Miren Cloud. It's how requests for your [subdomain](/miren-cloud/subdomains) reach your cluster, even when the cluster is behind NAT or a firewall.

## How It Works

When your cluster is registered with Miren Cloud, the miren server opens a long-lived connection to miren.cloud itself. When a request arrives at a Miren Cloud Point of Presence (POP), the POP determines which cluster the request is for and asks miren.cloud to have that cluster's server connect directly to the POP. Subsequent requests for your hostnames terminate at the POP, which forwards them to your cluster over the new connection.

The result: you don't need a public IP, port forwarding, or DNS gymnastics on your cluster — Miren Cloud handles ingress and uses the global router as the path back to you.

## When It Runs

The global router starts automatically whenever the miren server has cloud auth configured — i.e. after you've run `miren server install` (without `--without-cloud`) or `miren server register`. There's no flag to enable, no separate process to run.

If you run a fully standalone cluster (`miren server install --without-cloud`), the global router doesn't start; your cluster reaches the network however you've wired it.

## How Requests Use the Global Router

To use the global router, requests need to arrive at the Miren Cloud POP network. There are two ways that can happen:

_These examples use `cluster-xyz` as a stand-in for cluster IDs. You can find cluster IDs in miren.cloud._

1. **A request to `cluster-xyz.global.miren.systems`.** `*.global.miren.systems` points at `pop-global.miren.cloud`, so any request to that hostname is routed explicitly to the cluster named in the hostname.
2. **A cluster whose port 443 is not publicly reachable.** When a cluster connects to miren.cloud it checks whether its own port 443 is publicly accessible. If it isn't, miren.cloud automatically points the cluster's default DNS record (`cluster-xyz.miren.systems`) at `pop-global.miren.cloud` so traffic flows through the POP network instead.

### Miren Cloud Subdomains

A subdomain assigned by miren.cloud automatically routes traffic via the global router when the cluster has no publicly reachable port. It does this by pointing the subdomain at the cluster's auto-assigned DNS record — which, as described above, already points at `pop-global.miren.cloud`.

### User-Provided Domains

In the future, we'll allow users to bring their own domain names and route them through the global router. Today, you can point any DNS record at a cluster's public IP, which works fine for clusters that are publicly reachable.

## Verifying the Connection

Once the server is running, look for the global router connecting to cloud in the server logs:

```
component=globalrouter ... connected to cloud
```
Comment on lines +39 to +41

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

Add language identifier to the code block.

The fenced code block should specify a language identifier for proper syntax highlighting and linting compliance.

📝 Proposed fix
-```
+```text
 component=globalrouter ... connected to cloud

</details>

As per coding guidelines: markdownlint flagged this as MD040 (fenced-code-language).

<!-- suggestion_start -->

<details>
<summary>📝 Committable suggestion</summary>

> ‼️ **IMPORTANT**
> Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

```suggestion

🧰 Tools
🪛 markdownlint-cli2 (0.22.1)

[warning] 39-39: Fenced code blocks should have a language specified

(MD040, fenced-code-language)

🤖 Prompt for AI Agents
Verify each finding against the current code and only fix it if needed.

In `@docs/docs/miren-cloud/global-router.md` around lines 39 - 41, The fenced code
block in global-router.md lacks a language identifier causing MD040; update the
block fence from ``` to include a language (e.g., use ```text or another
appropriate language) so it becomes ```text and ensure the inner line
"component=globalrouter ... connected to cloud" remains unchanged to satisfy
linting and enable proper highlighting.


You can also confirm from the cluster side:

```bash
miren cluster list
```

A registered cluster that has connected through the global router will show up here.

## Troubleshooting

**Cluster won't connect.** The global router requires outbound connectivity to miren.cloud (TLS over TCP on port 443) and to every host under `pop-global.miren.cloud` (HTTP/3 over UDP on port 8443). If your network blocks outbound traffic, check that the POP host (visible in server logs under `component=globalrouter`) is reachable.

**Hostnames return cloud errors.** If a request to `mycluster.run.garden` lands at the POP but fails to forward, the global router connection may be down. Restart the miren server and watch the logs for the `connected to cloud` marker.

## Next Steps

- [Subdomains](/miren-cloud/subdomains) — Claim a hostname that uses the global router
- [Miren Cloud Overview](/miren-cloud/overview) — Cluster registration and login
5 changes: 5 additions & 0 deletions docs/sidebars.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,11 @@ const sidebars: SidebarsConfig = {
label: 'Overview',
},
'miren-cloud/subdomains',
{
type: 'doc',
id: 'miren-cloud/global-router',
label: 'Global Router',
},
{
type: 'doc',
id: 'miren-cloud/cloud-updates',
Expand Down
5 changes: 0 additions & 5 deletions pkg/labs/features.yaml
Original file line number Diff line number Diff line change
@@ -1,9 +1,4 @@
features:
- name: globalrouter
predicate: GlobalRouter
description: Use global NAT traversal router for connectivity
default: false

- name: distributedrunners
predicate: DistributedRunners
description: Schedule jobs across multiple runner nodes
Expand Down
10 changes: 0 additions & 10 deletions pkg/labs/labs.gen.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

20 changes: 10 additions & 10 deletions pkg/labs/labs_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,20 +11,20 @@ func TestDisableFeatureWithPrefix(t *testing.T) {
Reset()

// Enable first, then disable
Init(nil, []string{"globalrouter", "-globalrouter"})
Init(nil, []string{"routeoidc", "-routeoidc"})

if GlobalRouter() {
t.Error("GlobalRouter should be disabled after '-globalrouter'")
if RouteOIDC() {
t.Error("RouteOIDC should be disabled after '-routeoidc'")
}
}

func TestCaseInsensitiveFeatureNames(t *testing.T) {
Reset()

Init(nil, []string{"GlobalRouter", "ADMINAPI"})
Init(nil, []string{"RouteOIDC", "ADMINAPI"})

if !GlobalRouter() {
t.Error("GlobalRouter should be enabled (case-insensitive)")
if !RouteOIDC() {
t.Error("RouteOIDC should be enabled (case-insensitive)")
}
if !AdminAPI() {
t.Error("AdminAPI should be enabled (case-insensitive)")
Expand All @@ -51,10 +51,10 @@ func TestUnknownFeatureLogsWarning(t *testing.T) {
func TestEmptyAndWhitespaceFlags(t *testing.T) {
Reset()

Init(nil, []string{"", " ", "globalrouter", " ", ""})
Init(nil, []string{"", " ", "routeoidc", " ", ""})

if !GlobalRouter() {
t.Error("GlobalRouter should be enabled despite empty/whitespace flags")
if !RouteOIDC() {
t.Error("RouteOIDC should be enabled despite empty/whitespace flags")
}
}

Expand Down Expand Up @@ -91,7 +91,7 @@ func TestAllKeywordWithExclusion(t *testing.T) {
func TestNegativeAllDisablesAll(t *testing.T) {
Reset()

Init(nil, []string{"globalrouter", "adminapi", "-all"})
Init(nil, []string{"routeoidc", "adminapi", "-all"})

for _, name := range AllFeatures() {
if IsEnabled(name) {
Expand Down
Loading