diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cc9842..a6c1436 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.5.6] + +### Fixed + +- Improve error message when the OpenAI API returns an `incorrect_hostname` + error due to regional endpoint requirements ([#70]). The error now tells + the user to set `OPENAI_BASE_URL` or `--base-url` with the correct + regional host (e.g. `https://us.api.openai.com/v1`). + ## [1.5.5] ### Fixed @@ -202,6 +211,7 @@ First stable release. Includes the complete CLI and importable library packages. - `types` — shared data types (`Report`, `Result`, `Level`, etc.) - `judge.LLMClient` interface for custom LLM providers +[1.5.6]: https://github.com/agent-ecosystem/skill-validator/compare/v1.5.5...v1.5.6 [1.5.5]: https://github.com/agent-ecosystem/skill-validator/compare/v1.5.4...v1.5.5 [1.5.4]: https://github.com/agent-ecosystem/skill-validator/compare/v1.5.3...v1.5.4 [1.5.3]: https://github.com/agent-ecosystem/skill-validator/compare/v1.5.2...v1.5.3 diff --git a/README.md b/README.md index 63171a8..a6f31b4 100644 --- a/README.md +++ b/README.md @@ -341,6 +341,12 @@ skill-validator score evaluate --provider claude-cli \*\* The `openai` provider also reads these optional environment variables: `OPENAI_BASE_URL` (API base URL, overridden by `--base-url`), `OPENAI_ORG_ID` (sent as `OpenAI-Organization` header), and `OPENAI_PROJECT_ID` (sent as `OpenAI-Project` header). The org/project headers are only sent when targeting an OpenAI endpoint (e.g. `api.openai.com`, `us.api.openai.com`), not when using third-party compatible APIs. +If your OpenAI organization is bound to a regional endpoint (e.g. `us.api.openai.com` or `eu.api.openai.com`), set `OPENAI_BASE_URL` to the full regional URL including the `/v1` path: + +```bash +export OPENAI_BASE_URL=https://us.api.openai.com/v1 +``` + Use `--model` to override the default model and `--base-url` to point at any OpenAI-compatible endpoint (e.g. `http://localhost:11434/v1` for Ollama). If the endpoint requires a specific token limit parameter, use `--max-tokens-style` to override auto-detection: | Value | Behavior | diff --git a/cmd/root.go b/cmd/root.go index 79bdd75..c9e662a 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -11,7 +11,7 @@ import ( "github.com/agent-ecosystem/skill-validator/types" ) -const version = "v1.5.5" +const version = "v1.5.6" var ( outputFormat string diff --git a/judge/client.go b/judge/client.go index b86203e..3664685 100644 --- a/judge/client.go +++ b/judge/client.go @@ -306,6 +306,16 @@ func (c *openaiClient) Complete(ctx context.Context, systemPrompt, userContent s } if resp.StatusCode != http.StatusOK { + var errResp struct { + Error struct { + Code string `json:"code"` + Message string `json:"message"` + } `json:"error"` + } + if json.Unmarshal(respBody, &errResp) == nil && errResp.Error.Code == "incorrect_hostname" { + return "", fmt.Errorf("your OpenAI organization requires a regional endpoint: %s\nSet OPENAI_BASE_URL or use --base-url to specify the correct host (e.g. https://us.api.openai.com/v1)", + errResp.Error.Message) + } return "", fmt.Errorf("API returned status %d: %s", resp.StatusCode, string(respBody)) } diff --git a/judge/client_test.go b/judge/client_test.go index f124c58..112ad67 100644 --- a/judge/client_test.go +++ b/judge/client_test.go @@ -269,6 +269,37 @@ func TestOpenAIClient_OrgProjectHeaders(t *testing.T) { }) } +func TestOpenAIClient_RegionalHostnameError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusUnauthorized) + _, _ = fmt.Fprint(w, `{"error":{"message":"Attempted to access resource with incorrect regional hostname. Please make your request to us.api.openai.com","type":"invalid_request_error","code":"incorrect_hostname","param":null},"status":401}`) + })) + defer server.Close() + + client, err := NewClient(ClientOptions{ + Provider: "openai", + APIKey: "test-key", + BaseURL: server.URL, + Model: "gpt-4o", + }) + if err != nil { + t.Fatalf("NewClient: %v", err) + } + + _, err = client.Complete(t.Context(), "system", "user") + if err == nil { + t.Fatal("expected error, got nil") + } + + if !strings.Contains(err.Error(), "regional endpoint") { + t.Errorf("expected error to mention regional endpoint, got: %v", err) + } + if !strings.Contains(err.Error(), "OPENAI_BASE_URL") { + t.Errorf("expected error to mention OPENAI_BASE_URL, got: %v", err) + } +} + // rewriteTransport rewrites requests to a different target URL while // preserving the original Host header for testing. type rewriteTransport struct {