From 7867a28922e2bbc5afbfa830068cf581e43603b2 Mon Sep 17 00:00:00 2001 From: Aleks Clark Date: Mon, 22 Jun 2026 17:55:50 -0500 Subject: [PATCH] feat(nomad): support HA mode with multiple servers Add sticky failover across multiple Nomad server addresses so the integration keeps working when the leader or an individual server is unreachable. Accepts a new addresses credential (comma/newline list, NOMAD_ADDRS) alongside the existing address; requests fail over on transport errors and retryable 5xx/429 while definitive 4xx responses return immediately. Co-Authored-By: Claude Opus --- config/config.go | 7 +- config/config_test.go | 3 +- integrations/nomad/nomad.go | 134 +++++++++++++++++-- integrations/nomad/nomad_test.go | 212 ++++++++++++++++++++++++++++--- 4 files changed, 321 insertions(+), 35 deletions(-) diff --git a/config/config.go b/config/config.go index 8269d7d..0545265 100644 --- a/config/config.go +++ b/config/config.go @@ -137,8 +137,9 @@ var envMapping = map[string]map[string]string{ "base_url": "SIGNOZ_BASE_URL", }, "nomad": { - "address": "NOMAD_ADDR", - "token": "NOMAD_TOKEN", + "address": "NOMAD_ADDR", + "addresses": "NOMAD_ADDRS", + "token": "NOMAD_TOKEN", }, "agents": { "base_url": "AGENTS_BASE_URL", @@ -368,7 +369,7 @@ func defaultConfig() *mcp.Config { }, "nomad": { Enabled: false, - Credentials: mcp.Credentials{"address": "", "token": ""}, + Credentials: mcp.Credentials{"address": "", "addresses": "", "token": ""}, }, "agents": { Enabled: false, diff --git a/config/config_test.go b/config/config_test.go index 1ca95dc..4ceb676 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -282,7 +282,7 @@ func TestDefaultConfig(t *testing.T) { "vercel": {"api_token", "team_id", "team_slug", "base_url"}, "web": {}, "signoz": {"api_key", "base_url", "skip_verify"}, - "nomad": {"address", "token"}, + "nomad": {"address", "addresses", "token"}, } for name, keys := range expected { @@ -511,6 +511,7 @@ func TestEnvMapping_ReturnsMapping(t *testing.T) { assert.Equal(t, "X_BEARER_TOKEN", m["x"]["bearer_token"]) assert.Equal(t, "SIGNOZ_API_KEY", m["signoz"]["api_key"]) assert.Equal(t, "NOMAD_ADDR", m["nomad"]["address"]) + assert.Equal(t, "NOMAD_ADDRS", m["nomad"]["addresses"]) assert.Equal(t, "NOMAD_TOKEN", m["nomad"]["token"]) assert.Equal(t, "STRIPE_API_KEY", m["stripe"]["api_key"]) assert.Equal(t, "STRIPE_ACCOUNT", m["stripe"]["account"]) diff --git a/integrations/nomad/nomad.go b/integrations/nomad/nomad.go index 4032f79..7447783 100644 --- a/integrations/nomad/nomad.go +++ b/integrations/nomad/nomad.go @@ -5,12 +5,15 @@ import ( "context" _ "embed" "encoding/json" + "errors" "fmt" "io" "net/http" "net/url" "strings" + "sync/atomic" "time" + "unicode" mcp "github.com/daltoniam/switchboard" "github.com/daltoniam/switchboard/compact" @@ -28,12 +31,19 @@ var ( _ mcp.Integration = (*nomad)(nil) _ mcp.FieldCompactionIntegration = (*nomad)(nil) _ mcp.ToolMaxBytesIntegration = (*nomad)(nil) + _ mcp.PlaceholderHints = (*nomad)(nil) + _ mcp.OptionalCredentials = (*nomad)(nil) ) type nomad struct { - address string - token string - client *http.Client + addresses []string + token string + client *http.Client + + // preferred is the index of the server address to try first. It advances to + // the last server that answered so a downed leader is not retried on every + // call (sticky failover). Safe for concurrent use. + preferred atomic.Uint32 } func New() mcp.Integration { @@ -45,15 +55,31 @@ func New() mcp.Integration { func (n *nomad) Name() string { return "nomad" } func (n *nomad) Configure(_ context.Context, creds mcp.Credentials) error { - n.address = creds["address"] - if n.address == "" { + n.addresses = parseAddresses(creds["address"], creds["addresses"]) + if len(n.addresses) == 0 { return fmt.Errorf("nomad: address is required") } - n.address = strings.TrimRight(n.address, "/") n.token = creds["token"] + n.preferred.Store(0) return nil } +// Placeholders provides web-UI hints for the credential fields. +func (n *nomad) Placeholders() map[string]string { + return map[string]string{ + "address": "http://localhost:4646", + "addresses": "HA: comma- or newline-separated, e.g. http://server1:4646,http://server2:4646", + "token": "Nomad ACL token (optional)", + } +} + +// OptionalKeys marks credentials that are not strictly required. Either address +// or addresses must be set; addresses (the HA server list) and the ACL token are +// optional on their own. +func (n *nomad) OptionalKeys() []string { + return []string{"addresses", "token"} +} + func (n *nomad) Healthy(ctx context.Context) bool { if n.client == nil { return false @@ -86,30 +112,64 @@ func (n *nomad) Execute(ctx context.Context, toolName mcp.ToolName, args map[str // --- HTTP helpers --- +// doRequest issues an HTTP request to the configured Nomad server(s). In HA +// deployments multiple server addresses may be configured; doRequest performs +// sticky failover: it starts at the last server that answered and walks the +// remaining servers in order. Definitive (non-retryable) responses are returned +// immediately, while transport failures and retryable statuses (429/5xx) move on +// to the next server. func (n *nomad) doRequest(ctx context.Context, method, path string, body any) (json.RawMessage, error) { - var bodyReader io.Reader + var bodyData []byte if body != nil { data, err := json.Marshal(body) if err != nil { return nil, err } - bodyReader = bytes.NewReader(data) + bodyData = data + } + + if len(n.addresses) == 0 { + return nil, fmt.Errorf("nomad: no server address configured") + } + + start := int(n.preferred.Load()) % len(n.addresses) + var lastErr error + for i := range n.addresses { + idx := (start + i) % len(n.addresses) + data, err := n.doRequestTo(ctx, n.addresses[idx], method, path, bodyData, body != nil) + if err == nil { + n.preferred.Store(uint32(idx)) + return data, nil + } + lastErr = err + if ctx.Err() != nil || !shouldFailover(err) { + return nil, err + } } + return nil, lastErr +} - req, err := http.NewRequestWithContext(ctx, method, n.address+path, bodyReader) +// doRequestTo issues a single request against one server address. +func (n *nomad) doRequestTo(ctx context.Context, address, method, path string, bodyData []byte, hasBody bool) (json.RawMessage, error) { + var bodyReader io.Reader + if hasBody { + bodyReader = bytes.NewReader(bodyData) + } + + req, err := http.NewRequestWithContext(ctx, method, address+path, bodyReader) if err != nil { return nil, err } if n.token != "" { req.Header.Set("X-Nomad-Token", n.token) } - if body != nil { + if hasBody { req.Header.Set("Content-Type", "application/json") } resp, err := n.client.Do(req) if err != nil { - return nil, err + return nil, fmt.Errorf("nomad: request to %s failed: %w", address, err) } defer func() { _ = resp.Body.Close() }() @@ -124,7 +184,7 @@ func (n *nomad) doRequest(ctx context.Context, method, path string, body any) (j return nil, re } if resp.StatusCode >= 400 { - return nil, fmt.Errorf("nomad API error (%d): %s", resp.StatusCode, string(data)) + return nil, &apiError{StatusCode: resp.StatusCode, Body: string(data)} } if resp.StatusCode == 204 || len(data) == 0 { return json.RawMessage(`{"status":"success"}`), nil @@ -132,6 +192,56 @@ func (n *nomad) doRequest(ctx context.Context, method, path string, body any) (j return json.RawMessage(data), nil } +// apiError is a definitive (non-retryable) HTTP error from a reachable Nomad +// server. Failover treats these as authoritative and does not retry the request +// against another server. +type apiError struct { + StatusCode int + Body string +} + +func (e *apiError) Error() string { + return fmt.Sprintf("nomad API error (%d): %s", e.StatusCode, e.Body) +} + +// shouldFailover reports whether a failed request should be retried against the +// next server. Definitive 4xx responses (apiError) are authoritative and are not +// retried; transport failures and retryable statuses (429/5xx) are. +func shouldFailover(err error) bool { + if err == nil { + return false + } + var ae *apiError + return !errors.As(err, &ae) +} + +// parseAddresses splits one or more raw credential values into a deduplicated, +// ordered list of Nomad server base URLs. Values may be separated by commas or +// any whitespace (spaces, tabs, newlines), so a single multi-line credential +// field or a comma-separated NOMAD_ADDR both work for HA deployments. Trailing +// slashes are trimmed and duplicates are dropped while preserving order. +func parseAddresses(values ...string) []string { + var out []string + seen := make(map[string]struct{}) + for _, v := range values { + fields := strings.FieldsFunc(v, func(r rune) bool { + return r == ',' || unicode.IsSpace(r) + }) + for _, field := range fields { + addr := strings.TrimRight(field, "/") + if addr == "" { + continue + } + if _, dup := seen[addr]; dup { + continue + } + seen[addr] = struct{}{} + out = append(out, addr) + } + } + return out +} + func (n *nomad) get(ctx context.Context, pathFmt string, args ...any) (json.RawMessage, error) { return n.doRequest(ctx, "GET", fmt.Sprintf(pathFmt, args...), nil) } diff --git a/integrations/nomad/nomad_test.go b/integrations/nomad/nomad_test.go index 3abc4c6..5af451a 100644 --- a/integrations/nomad/nomad_test.go +++ b/integrations/nomad/nomad_test.go @@ -43,7 +43,7 @@ func TestConfigure_TrimsTrailingSlash(t *testing.T) { n := &nomad{client: &http.Client{}} err := n.Configure(context.Background(), mcp.Credentials{"address": "http://localhost:4646/"}) assert.NoError(t, err) - assert.Equal(t, "http://localhost:4646", n.address) + assert.Equal(t, "http://localhost:4646", n.addresses[0]) } func TestTools(t *testing.T) { @@ -74,7 +74,7 @@ func TestTools_NoDuplicateNames(t *testing.T) { } func TestExecute_UnknownTool(t *testing.T) { - n := &nomad{address: "http://localhost:4646", client: &http.Client{}} + n := &nomad{addresses: []string{"http://localhost:4646"}, client: &http.Client{}} result, err := n.Execute(context.Background(), "nomad_nonexistent", nil) require.NoError(t, err) assert.True(t, result.IsError) @@ -109,7 +109,7 @@ func TestDoRequest_Success(t *testing.T) { })) defer ts.Close() - n := &nomad{address: ts.URL, client: ts.Client()} + n := &nomad{addresses: []string{ts.URL}, client: ts.Client()} data, err := n.get(context.Background(), "/v1/jobs") require.NoError(t, err) assert.Contains(t, string(data), "test-job") @@ -122,7 +122,7 @@ func TestDoRequest_WithToken(t *testing.T) { })) defer ts.Close() - n := &nomad{address: ts.URL, token: "my-token", client: ts.Client()} + n := &nomad{addresses: []string{ts.URL}, token: "my-token", client: ts.Client()} _, err := n.get(context.Background(), "/v1/agent/self") require.NoError(t, err) } @@ -134,7 +134,7 @@ func TestDoRequest_NoTokenHeader_WhenEmpty(t *testing.T) { })) defer ts.Close() - n := &nomad{address: ts.URL, client: ts.Client()} + n := &nomad{addresses: []string{ts.URL}, client: ts.Client()} _, err := n.get(context.Background(), "/v1/agent/self") require.NoError(t, err) } @@ -146,7 +146,7 @@ func TestDoRequest_APIError(t *testing.T) { })) defer ts.Close() - n := &nomad{address: ts.URL, client: ts.Client()} + n := &nomad{addresses: []string{ts.URL}, client: ts.Client()} _, err := n.get(context.Background(), "/v1/jobs") assert.Error(t, err) assert.Contains(t, err.Error(), "nomad API error (403)") @@ -158,7 +158,7 @@ func TestDoRequest_204NoContent(t *testing.T) { })) defer ts.Close() - n := &nomad{address: ts.URL, client: ts.Client()} + n := &nomad{addresses: []string{ts.URL}, client: ts.Client()} data, err := n.doRequest(context.Background(), "PUT", "/v1/system/gc", nil) require.NoError(t, err) assert.Contains(t, string(data), "success") @@ -175,7 +175,7 @@ func TestPost(t *testing.T) { })) defer ts.Close() - n := &nomad{address: ts.URL, client: ts.Client()} + n := &nomad{addresses: []string{ts.URL}, client: ts.Client()} data, err := n.post(context.Background(), "/v1/jobs", map[string]any{"Job": map[string]string{"ID": "test"}}) require.NoError(t, err) assert.Contains(t, string(data), "eval-123") @@ -189,7 +189,7 @@ func TestDoRequest_RetryableOn429(t *testing.T) { })) defer ts.Close() - n := &nomad{address: ts.URL, client: ts.Client()} + n := &nomad{addresses: []string{ts.URL}, client: ts.Client()} _, err := n.get(context.Background(), "/v1/jobs") require.Error(t, err) assert.True(t, mcp.IsRetryable(err), "429 should produce RetryableError") @@ -207,7 +207,7 @@ func TestDoRequest_RetryableOn5xx(t *testing.T) { })) defer ts.Close() - n := &nomad{address: ts.URL, client: ts.Client()} + n := &nomad{addresses: []string{ts.URL}, client: ts.Client()} _, err := n.get(context.Background(), "/v1/jobs") require.Error(t, err) assert.True(t, mcp.IsRetryable(err), "503 should produce RetryableError") @@ -220,7 +220,7 @@ func TestDoRequest_NonRetryableOn4xx(t *testing.T) { })) defer ts.Close() - n := &nomad{address: ts.URL, client: ts.Client()} + n := &nomad{addresses: []string{ts.URL}, client: ts.Client()} _, err := n.get(context.Background(), "/v1/jobs") require.Error(t, err) assert.False(t, mcp.IsRetryable(err), "404 should NOT be retryable") @@ -270,7 +270,7 @@ func TestListJobs(t *testing.T) { })) defer ts.Close() - n := &nomad{address: ts.URL, client: ts.Client()} + n := &nomad{addresses: []string{ts.URL}, client: ts.Client()} result, err := n.Execute(context.Background(), "nomad_list_jobs", map[string]any{}) require.NoError(t, err) assert.False(t, result.IsError) @@ -284,7 +284,7 @@ func TestGetJob(t *testing.T) { })) defer ts.Close() - n := &nomad{address: ts.URL, client: ts.Client()} + n := &nomad{addresses: []string{ts.URL}, client: ts.Client()} result, err := n.Execute(context.Background(), "nomad_get_job", map[string]any{"job_id": "web"}) require.NoError(t, err) assert.False(t, result.IsError) @@ -298,7 +298,7 @@ func TestListNodes(t *testing.T) { })) defer ts.Close() - n := &nomad{address: ts.URL, client: ts.Client()} + n := &nomad{addresses: []string{ts.URL}, client: ts.Client()} result, err := n.Execute(context.Background(), "nomad_list_nodes", map[string]any{}) require.NoError(t, err) assert.False(t, result.IsError) @@ -320,7 +320,7 @@ func TestGetClusterStatus(t *testing.T) { })) defer ts.Close() - n := &nomad{address: ts.URL, client: ts.Client()} + n := &nomad{addresses: []string{ts.URL}, client: ts.Client()} result, err := n.Execute(context.Background(), "nomad_get_cluster_status", map[string]any{}) require.NoError(t, err) assert.False(t, result.IsError) @@ -340,7 +340,7 @@ func TestDrainNode(t *testing.T) { })) defer ts.Close() - n := &nomad{address: ts.URL, client: ts.Client()} + n := &nomad{addresses: []string{ts.URL}, client: ts.Client()} result, err := n.Execute(context.Background(), "nomad_drain_node", map[string]any{"node_id": "node-1", "enable": true, "deadline": "1h"}) require.NoError(t, err) assert.False(t, result.IsError) @@ -356,7 +356,7 @@ func TestNodeEligibility(t *testing.T) { })) defer ts.Close() - n := &nomad{address: ts.URL, client: ts.Client()} + n := &nomad{addresses: []string{ts.URL}, client: ts.Client()} result, err := n.Execute(context.Background(), "nomad_node_eligibility", map[string]any{"node_id": "node-1", "eligible": false}) require.NoError(t, err) assert.False(t, result.IsError) @@ -377,7 +377,7 @@ func TestHealthy(t *testing.T) { })) defer ts.Close() - n := &nomad{address: ts.URL, client: ts.Client()} + n := &nomad{addresses: []string{ts.URL}, client: ts.Client()} assert.True(t, n.Healthy(context.Background())) }) @@ -388,7 +388,7 @@ func TestHealthy(t *testing.T) { })) defer ts.Close() - n := &nomad{address: ts.URL, client: ts.Client()} + n := &nomad{addresses: []string{ts.URL}, client: ts.Client()} assert.False(t, n.Healthy(context.Background())) }) @@ -397,3 +397,177 @@ func TestHealthy(t *testing.T) { assert.False(t, n.Healthy(context.Background())) }) } + +// --- HA (multiple servers) tests --- + +func TestParseAddresses(t *testing.T) { + tests := []struct { + name string + in []string + want []string + }{ + {"empty", []string{""}, nil}, + {"single", []string{"http://a:4646"}, []string{"http://a:4646"}}, + {"trims trailing slash", []string{"http://a:4646/"}, []string{"http://a:4646"}}, + {"comma separated", []string{"http://a:4646,http://b:4646"}, []string{"http://a:4646", "http://b:4646"}}, + {"comma with spaces", []string{"http://a:4646, http://b:4646"}, []string{"http://a:4646", "http://b:4646"}}, + {"newline separated", []string{"http://a:4646\nhttp://b:4646"}, []string{"http://a:4646", "http://b:4646"}}, + {"dedups", []string{"http://a:4646,http://a:4646/"}, []string{"http://a:4646"}}, + {"merges multiple values", []string{"http://a:4646", "http://b:4646"}, []string{"http://a:4646", "http://b:4646"}}, + {"merges and dedups across values", []string{"http://a:4646", "http://a:4646,http://b:4646"}, []string{"http://a:4646", "http://b:4646"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, parseAddresses(tt.in...)) + }) + } +} + +func TestConfigure_MultipleAddresses(t *testing.T) { + t.Run("comma-separated in address", func(t *testing.T) { + n := &nomad{client: &http.Client{}} + require.NoError(t, n.Configure(context.Background(), mcp.Credentials{"address": "http://a:4646, http://b:4646/"})) + assert.Equal(t, []string{"http://a:4646", "http://b:4646"}, n.addresses) + }) + t.Run("addresses plural key", func(t *testing.T) { + n := &nomad{client: &http.Client{}} + require.NoError(t, n.Configure(context.Background(), mcp.Credentials{"addresses": "http://a:4646\nhttp://b:4646"})) + assert.Equal(t, []string{"http://a:4646", "http://b:4646"}, n.addresses) + }) + t.Run("merges and dedups address and addresses", func(t *testing.T) { + n := &nomad{client: &http.Client{}} + require.NoError(t, n.Configure(context.Background(), mcp.Credentials{"address": "http://a:4646", "addresses": "http://a:4646,http://b:4646"})) + assert.Equal(t, []string{"http://a:4646", "http://b:4646"}, n.addresses) + }) + t.Run("missing both is error", func(t *testing.T) { + n := &nomad{client: &http.Client{}} + err := n.Configure(context.Background(), mcp.Credentials{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "address is required") + }) +} + +func TestDoRequest_FailoverWhenFirstServerDown(t *testing.T) { + down := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + downURL := down.URL + down.Close() // connection refused from now on + + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Write([]byte(`[{"ID":"web"}]`)) + })) + defer up.Close() + + n := &nomad{addresses: []string{downURL, up.URL}, client: &http.Client{Timeout: 5 * time.Second}} + data, err := n.get(context.Background(), "/v1/jobs") + require.NoError(t, err) + assert.Contains(t, string(data), "web") + assert.Equal(t, uint32(1), n.preferred.Load(), "preferred should advance to the healthy server") +} + +func TestDoRequest_FailoverOnRetryableStatus(t *testing.T) { + s1 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(503) + })) + defer s1.Close() + s2Hits := 0 + s2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + s2Hits++ + w.Write([]byte(`{"ok":true}`)) + })) + defer s2.Close() + + n := &nomad{addresses: []string{s1.URL, s2.URL}, client: &http.Client{}} + data, err := n.get(context.Background(), "/v1/jobs") + require.NoError(t, err) + assert.Contains(t, string(data), "ok") + assert.Equal(t, 1, s2Hits) +} + +func TestDoRequest_FailoverExhaustedReturnsRetryable(t *testing.T) { + mk := func() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(503) + })) + } + s1, s2 := mk(), mk() + defer s1.Close() + defer s2.Close() + + n := &nomad{addresses: []string{s1.URL, s2.URL}, client: &http.Client{}} + _, err := n.get(context.Background(), "/v1/jobs") + require.Error(t, err) + assert.True(t, mcp.IsRetryable(err), "all servers retryable should surface a retryable error") +} + +func TestDoRequest_NoFailoverOnDefinitiveError(t *testing.T) { + s1Hits, s2Hits := 0, 0 + s1 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + s1Hits++ + w.WriteHeader(404) + w.Write([]byte("not found")) + })) + defer s1.Close() + s2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + s2Hits++ + w.Write([]byte(`{"ok":true}`)) + })) + defer s2.Close() + + n := &nomad{addresses: []string{s1.URL, s2.URL}, client: &http.Client{}} + _, err := n.get(context.Background(), "/v1/jobs") + require.Error(t, err) + assert.Contains(t, err.Error(), "nomad API error (404)") + assert.Equal(t, 1, s1Hits) + assert.Equal(t, 0, s2Hits, "definitive 4xx must not fail over to the next server") +} + +func TestDoRequest_StickyPreferredAfterFailover(t *testing.T) { + s1Hits, s2Hits := 0, 0 + s1 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + s1Hits++ + w.WriteHeader(503) + })) + defer s1.Close() + s2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + s2Hits++ + w.Write([]byte(`{"ok":true}`)) + })) + defer s2.Close() + + n := &nomad{addresses: []string{s1.URL, s2.URL}, client: &http.Client{}} + for range 3 { + _, err := n.get(context.Background(), "/v1/jobs") + require.NoError(t, err) + } + assert.Equal(t, 1, s1Hits, "unhealthy server should only be tried on the first call") + assert.Equal(t, 3, s2Hits) +} + +func TestDoRequest_FailoverReplaysBody(t *testing.T) { + down := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + downURL := down.URL + down.Close() + + gotJob := false + up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body map[string]any + _ = json.NewDecoder(r.Body).Decode(&body) + gotJob = body["Job"] != nil + w.Write([]byte(`{"EvalID":"e1"}`)) + })) + defer up.Close() + + n := &nomad{addresses: []string{downURL, up.URL}, client: &http.Client{Timeout: 5 * time.Second}} + _, err := n.post(context.Background(), "/v1/jobs", map[string]any{"Job": map[string]string{"ID": "x"}}) + require.NoError(t, err) + assert.True(t, gotJob, "request body must be replayed to the failover server") +} + +func TestPlaceholdersAndOptionalKeys(t *testing.T) { + n := &nomad{} + ph := n.Placeholders() + assert.Contains(t, ph, "address") + assert.Contains(t, ph, "addresses") + assert.Contains(t, ph, "token") + assert.Equal(t, []string{"addresses", "token"}, n.OptionalKeys()) +}