Skip to content

Ability to easily provide custom error messages/suggestions/docs for errors. #6700

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wbreza wants to merge 3 commits into
base: main
Choose a base branch
from
+1,278 −75
Open

Ability to easily provide custom error messages/suggestions/docs for errors. #6700

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8406090
Custom error patterns
wbreza Feb 3, 2026
b9e24b3
Update cli/azd/resources/error_suggestions.yaml
kristenwomack Feb 4, 2026
f10b9db
Update cli/azd/docs/error-suggestions.md
kristenwomack Feb 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions cli/azd/cmd/container.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import (
"github.com/azure/azure-dev/cli/azd/pkg/entraid"
"github.com/azure/azure-dev/cli/azd/pkg/environment"
"github.com/azure/azure-dev/cli/azd/pkg/environment/azdcontext"
"github.com/azure/azure-dev/cli/azd/pkg/errorhandler"
"github.com/azure/azure-dev/cli/azd/pkg/exec"
"github.com/azure/azure-dev/cli/azd/pkg/extensions"
"github.com/azure/azure-dev/cli/azd/pkg/helm"
Expand Down Expand Up @@ -650,6 +651,7 @@ func registerCommonDependencies(container *ioc.NestedContainer) {
container.MustRegisterSingleton(containerregistry.NewRemoteBuildManager)
container.MustRegisterSingleton(keyvault.NewKeyVaultService)
container.MustRegisterSingleton(storage.NewFileShareService)
container.MustRegisterSingleton(errorhandler.NewErrorSuggestionService)

container.MustRegisterScoped(project.NewContainerHelper)
container.MustRegisterScoped(func(serviceLocator ioc.ServiceLocator) *lazy.Lazy[*project.ContainerHelper] {
Expand Down
59 changes: 37 additions & 22 deletions cli/azd/cmd/middleware/error.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"github.com/azure/azure-dev/cli/azd/internal/tracing/resource"
"github.com/azure/azure-dev/cli/azd/pkg/alpha"
"github.com/azure/azure-dev/cli/azd/pkg/config"
"github.com/azure/azure-dev/cli/azd/pkg/errorhandler"
"github.com/azure/azure-dev/cli/azd/pkg/input"
"github.com/azure/azure-dev/cli/azd/pkg/llm"
"github.com/azure/azure-dev/cli/azd/pkg/output"
Expand All @@ -28,12 +29,13 @@ import (
)

type ErrorMiddleware struct {
options *Options
console input.Console
agentFactory *agent.AgentFactory
global *internal.GlobalCommandOptions
featuresManager *alpha.FeatureManager
userConfigManager config.UserConfigManager
options *Options
console input.Console
agentFactory *agent.AgentFactory
global *internal.GlobalCommandOptions
featuresManager *alpha.FeatureManager
userConfigManager config.UserConfigManager
errorSuggestionService *errorhandler.ErrorSuggestionService
}

func NewErrorMiddleware(
Expand All @@ -42,14 +44,16 @@ func NewErrorMiddleware(
global *internal.GlobalCommandOptions,
featuresManager *alpha.FeatureManager,
userConfigManager config.UserConfigManager,
errorSuggestionService *errorhandler.ErrorSuggestionService,
) Middleware {
return &ErrorMiddleware{
options: options,
console: console,
agentFactory: agentFactory,
global: global,
featuresManager: featuresManager,
userConfigManager: userConfigManager,
options: options,
console: console,
agentFactory: agentFactory,
global: global,
featuresManager: featuresManager,
userConfigManager: userConfigManager,
errorSuggestionService: errorSuggestionService,
}
}

Expand All @@ -64,14 +68,6 @@ func (e *ErrorMiddleware) displayAgentResponse(ctx context.Context, response str
}

func (e *ErrorMiddleware) Run(ctx context.Context, next NextFn) (*actions.ActionResult, error) {
// Short-circuit agentic error handling in non-interactive scenarios:
// - LLM feature is disabled
// - User specified --no-prompt (non-interactive mode)
// - Running in CI/CD environment where user interaction is not possible
if !e.featuresManager.IsEnabled(llm.FeatureLlm) || e.global.NoPrompt || resource.IsRunningOnCI() {
return next(ctx)
}

actionResult, err := next(ctx)

// Stop the spinner always to un-hide cursor
Expand All @@ -81,10 +77,29 @@ func (e *ErrorMiddleware) Run(ctx context.Context, next NextFn) (*actions.Action
return actionResult, err
}

// Error already has a suggestion, no need for AI
// Check if error already has a suggestion
var suggestionErr *internal.ErrorWithSuggestion
if errors.As(err, &suggestionErr) {
e.console.Message(ctx, suggestionErr.Suggestion)
// Already has a suggestion, return as-is
return actionResult, err
}

// Try to match error against known patterns and wrap with suggestion
if suggestion := e.errorSuggestionService.FindSuggestion(err.Error()); suggestion != nil {
wrappedErr := &internal.ErrorWithSuggestion{
Err: err,
Message: suggestion.Message,
Suggestion: suggestion.Suggestion,
DocUrl: suggestion.DocUrl,
}
return actionResult, wrappedErr
}

// Short-circuit agentic error handling in non-interactive scenarios:
// - LLM feature is disabled
// - User specified --no-prompt (non-interactive mode)
// - Running in CI/CD environment where user interaction is not possible
if !e.featuresManager.IsEnabled(llm.FeatureLlm) || e.global.NoPrompt || resource.IsRunningOnCI() {
return actionResult, err
}

Expand Down
119 changes: 81 additions & 38 deletions cli/azd/cmd/middleware/error_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"github.com/azure/azure-dev/cli/azd/internal"
"github.com/azure/azure-dev/cli/azd/pkg/alpha"
"github.com/azure/azure-dev/cli/azd/pkg/config"
"github.com/azure/azure-dev/cli/azd/pkg/errorhandler"
"github.com/azure/azure-dev/cli/azd/pkg/llm"
"github.com/azure/azure-dev/cli/azd/test/mocks"
"github.com/stretchr/testify/require"
Expand All @@ -30,13 +31,15 @@ func Test_ErrorMiddleware_SuccessNoError(t *testing.T) {
NoPrompt: false,
}
userConfigManager := config.NewUserConfigManager(mockContext.ConfigManager)
errorSuggestionService := errorhandler.NewErrorSuggestionService()
middleware := NewErrorMiddleware(
&Options{Name: "test"},
mockContext.Console,
nil, // agentFactory not needed for success case
global,
featureManager,
userConfigManager,
errorSuggestionService,
)
nextFn := func(ctx context.Context) (*actions.ActionResult, error) {
return &actions.ActionResult{
Expand All @@ -61,13 +64,15 @@ func Test_ErrorMiddleware_LLMAlphaFeatureDisabled(t *testing.T) {
NoPrompt: false,
}
userConfigManager := config.NewUserConfigManager(mockContext.ConfigManager)
errorSuggestionService := errorhandler.NewErrorSuggestionService()
middleware := NewErrorMiddleware(
&Options{Name: "test"},
mockContext.Console,
nil,
global,
featureManager,
userConfigManager,
errorSuggestionService,
)

testError := errors.New("test error")
Expand All @@ -77,12 +82,13 @@ func Test_ErrorMiddleware_LLMAlphaFeatureDisabled(t *testing.T) {

result, err := middleware.Run(*mockContext.Context, nextFn)

// Should return error without AI intervention in no-prompt mode
Copy link

Copilot AI Feb 3, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Misleading comment: The comment states "Should return error without AI intervention in no-prompt mode" but this test is actually checking the scenario where the LLM alpha feature is disabled (NoPrompt is false in line 64). The comment should clarify that this tests the case where the LLM feature is disabled, not no-prompt mode.

Suggested change
// Should return error without AI intervention in no-prompt mode
// Should return error without AI intervention when the LLM alpha feature is disabled

Copilot uses AI. Check for mistakes.
Copy link
Collaborator

@kristenwomack kristenwomack Feb 4, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think this change copilot is suggesting is correct. I think it's conflating no-prompt mode for agent interaction with the alpha agentic mode. @wbreza pls confirm

require.Error(t, err)
require.Nil(t, result)
require.Equal(t, testError, err)
}

func Test_ErrorMiddleware_NoPromptMode(t *testing.T) {
func Test_ErrorMiddleware_ChildAction(t *testing.T) {
mockContext := mocks.NewMockContext(context.Background())
cfg := config.NewConfig(map[string]any{
"alpha": map[string]any{
Expand All @@ -91,32 +97,38 @@ func Test_ErrorMiddleware_NoPromptMode(t *testing.T) {
})
featureManager := alpha.NewFeaturesManagerWithConfig(cfg)
global := &internal.GlobalCommandOptions{
NoPrompt: true, // Non-interactive mode
NoPrompt: false,
}
userConfigManager := config.NewUserConfigManager(mockContext.ConfigManager)
errorSuggestionService := errorhandler.NewErrorSuggestionService()
middleware := NewErrorMiddleware(
&Options{Name: "test"},
mockContext.Console,
nil,
global,
featureManager,
userConfigManager,
errorSuggestionService,
)

testError := errors.New("test error")
nextFn := func(ctx context.Context) (*actions.ActionResult, error) {
return nil, testError
}

result, err := middleware.Run(*mockContext.Context, nextFn)
// Mark context as child action
ctx := WithChildAction(*mockContext.Context)
result, err := middleware.Run(ctx, nextFn)

// Should return error without AI intervention in no-prompt mode
require.Error(t, err)
require.Nil(t, result)
require.Equal(t, testError, err)
}

func Test_ErrorMiddleware_ChildAction(t *testing.T) {
func Test_ErrorMiddleware_ErrorWithSuggestion(t *testing.T) {
if os.Getenv("TF_BUILD") != "" || os.Getenv("GITHUB_ACTIONS") != "" || os.Getenv("CI") != "" {
t.Skip("Skipping test in CI/CD environment")
}

mockContext := mocks.NewMockContext(context.Background())
cfg := config.NewConfig(map[string]any{
"alpha": map[string]any{
Expand All @@ -128,78 +140,109 @@ func Test_ErrorMiddleware_ChildAction(t *testing.T) {
NoPrompt: false,
}
userConfigManager := config.NewUserConfigManager(mockContext.ConfigManager)
errorSuggestionService := errorhandler.NewErrorSuggestionService()
middleware := NewErrorMiddleware(
&Options{Name: "test"},
mockContext.Console,
nil,
global,
featureManager,
userConfigManager,
errorSuggestionService,
)
testError := errors.New("test error")

// Create error with suggestion
testErr := errors.New("test error")
suggestionErr := &internal.ErrorWithSuggestion{
Err: testErr,
Suggestion: "Suggested fix",
}
nextFn := func(ctx context.Context) (*actions.ActionResult, error) {
return nil, testError
return nil, suggestionErr
}

// Mark context as child action
ctx := WithChildAction(*mockContext.Context)
result, err := middleware.Run(ctx, nextFn)
result, err := middleware.Run(*mockContext.Context, nextFn)

require.Error(t, err)
require.Nil(t, result)
require.Equal(t, testError, err)
}

func Test_ErrorMiddleware_ErrorWithSuggestion(t *testing.T) {
if os.Getenv("TF_BUILD") != "" || os.Getenv("GITHUB_ACTIONS") != "" || os.Getenv("CI") != "" {
t.Skip("Skipping test in CI/CD environment")
}
// Verify the error with suggestion is returned as-is (not modified)
var returnedSuggestionErr *internal.ErrorWithSuggestion
require.True(t, errors.As(err, &returnedSuggestionErr), "Expected ErrorWithSuggestion to be returned")
require.Equal(t, "Suggested fix", returnedSuggestionErr.Suggestion)
}

func Test_ErrorMiddleware_PatternMatchingSuggestion(t *testing.T) {
mockContext := mocks.NewMockContext(context.Background())
cfg := config.NewConfig(map[string]any{
"alpha": map[string]any{
string(llm.FeatureLlm): "on",
},
})
cfg := config.NewEmptyConfig()
featureManager := alpha.NewFeaturesManagerWithConfig(cfg)
global := &internal.GlobalCommandOptions{
NoPrompt: false,
}
userConfigManager := config.NewUserConfigManager(mockContext.ConfigManager)
errorSuggestionService := errorhandler.NewErrorSuggestionService()
middleware := NewErrorMiddleware(
&Options{Name: "test"},
mockContext.Console,
nil,
global,
featureManager,
userConfigManager,
errorSuggestionService,
)

// Create error with suggestion
testErr := errors.New("test error")
suggestionErr := &internal.ErrorWithSuggestion{
Err: testErr,
Suggestion: "Suggested fix",
}
// Create an error that matches a known pattern (quota error)
quotaError := errors.New("Deployment failed: QuotaExceeded for resource")
nextFn := func(ctx context.Context) (*actions.ActionResult, error) {
return nil, suggestionErr
return nil, quotaError
}

result, err := middleware.Run(*mockContext.Context, nextFn)

require.Error(t, err)
require.Nil(t, result)

// Check that suggestion was displayed
consoleOutput := mockContext.Console.Output()
foundSuggestion := false
for _, message := range consoleOutput {
if message == "Suggested fix" {
foundSuggestion = true
break
}
// Verify the error was wrapped with a suggestion
var suggestionErr *internal.ErrorWithSuggestion
require.True(t, errors.As(err, &suggestionErr), "Expected error to be wrapped with suggestion")
require.Contains(t, suggestionErr.Suggestion, "quota")
require.NotEmpty(t, suggestionErr.DocUrl, "Expected a documentation URL")
}

func Test_ErrorMiddleware_NoPatternMatch(t *testing.T) {
mockContext := mocks.NewMockContext(context.Background())
cfg := config.NewEmptyConfig()
featureManager := alpha.NewFeaturesManagerWithConfig(cfg)
global := &internal.GlobalCommandOptions{
NoPrompt: true, // Use no-prompt mode to avoid AI processing
}
userConfigManager := config.NewUserConfigManager(mockContext.ConfigManager)
errorSuggestionService := errorhandler.NewErrorSuggestionService()
middleware := NewErrorMiddleware(
&Options{Name: "test"},
mockContext.Console,
nil,
global,
featureManager,
userConfigManager,
errorSuggestionService,
)

// Create an error that doesn't match any pattern
unknownError := errors.New("some completely unique error xyz123abc")
nextFn := func(ctx context.Context) (*actions.ActionResult, error) {
return nil, unknownError
}
require.True(t, foundSuggestion, "No suggestion displayed for ErrorWithSuggestion")

result, err := middleware.Run(*mockContext.Context, nextFn)

require.Error(t, err)
require.Nil(t, result)

// Verify the error was NOT wrapped with a suggestion
var suggestionErr *internal.ErrorWithSuggestion
require.False(t, errors.As(err, &suggestionErr), "Expected error NOT to be wrapped with suggestion")
require.Equal(t, unknownError, err)
}

func Test_ExtractSuggestedSolutions(t *testing.T) {
Expand Down
Loading
Loading