diff --git a/apps/workspace-engine/pkg/workspace/releasemanager/policy/evaluator/environmentprogression/environment_progression_action.go b/apps/workspace-engine/pkg/workspace/releasemanager/policy/evaluator/environmentprogression/environment_progression_action.go index 0db7298ad..593c9da5b 100644 --- a/apps/workspace-engine/pkg/workspace/releasemanager/policy/evaluator/environmentprogression/environment_progression_action.go +++ b/apps/workspace-engine/pkg/workspace/releasemanager/policy/evaluator/environmentprogression/environment_progression_action.go @@ -221,7 +221,22 @@ func (a *EnvironmentProgressionAction) didThresholdJustCross( return false } - return satisfiedAt.Equal(*job.CompletedAt) + successCompletedAt := job.CompletedAt + verificationStatus := a.store.JobVerifications.GetJobVerificationStatus(job.Id) + if verificationStatus != "" && verificationStatus != oapi.JobVerificationStatusPassed { + return false + } + if verificationStatus == oapi.JobVerificationStatusPassed { + if verificationCompletedAt := a.getLatestVerificationCompletedAt(job.Id); verificationCompletedAt != nil { + successCompletedAt = verificationCompletedAt + } + } + + if successCompletedAt == nil { + return false + } + + return satisfiedAt.Equal(*successCompletedAt) } func (a *EnvironmentProgressionAction) getThresholdSatisfiedAt( @@ -233,3 +248,18 @@ func (a *EnvironmentProgressionAction) getThresholdSatisfiedAt( } return tracker.GetSuccessPercentageSatisfiedAt(minPercentage) } + +func (a *EnvironmentProgressionAction) getLatestVerificationCompletedAt(jobId string) *time.Time { + verifications := a.store.JobVerifications.GetByJobId(jobId) + var latest *time.Time + for _, verification := range verifications { + completedAt := verification.CompletedAt() + if completedAt == nil { + continue + } + if latest == nil || completedAt.After(*latest) { + latest = completedAt + } + } + return latest +} diff --git a/apps/workspace-engine/pkg/workspace/releasemanager/policy/evaluator/environmentprogression/jobtracker.go b/apps/workspace-engine/pkg/workspace/releasemanager/policy/evaluator/environmentprogression/jobtracker.go index beefda4d4..c5549c394 100644 --- a/apps/workspace-engine/pkg/workspace/releasemanager/policy/evaluator/environmentprogression/jobtracker.go +++ b/apps/workspace-engine/pkg/workspace/releasemanager/policy/evaluator/environmentprogression/jobtracker.go @@ -118,14 +118,14 @@ func (t *ReleaseTargetJobTracker) compute(ctx context.Context) []*oapi.Job { continue } - if t.SuccessStatuses[job.Status] && job.CompletedAt != nil { + if completedAt, ok := t.successCompletionTime(job); ok { targetKey := rt.Key() // Store the oldest successful completion time for this release target - if existingTime, exists := t.successfulReleaseTargets[targetKey]; !exists || job.CompletedAt.Before(existingTime) { - t.successfulReleaseTargets[targetKey] = *job.CompletedAt + if existingTime, exists := t.successfulReleaseTargets[targetKey]; !exists || completedAt.Before(existingTime) { + t.successfulReleaseTargets[targetKey] = *completedAt } - if t.mostRecentSuccess.Before(*job.CompletedAt) { - t.mostRecentSuccess = *job.CompletedAt + if t.mostRecentSuccess.Before(*completedAt) { + t.mostRecentSuccess = *completedAt } } @@ -142,6 +142,40 @@ func (t *ReleaseTargetJobTracker) compute(ctx context.Context) []*oapi.Job { return t.jobs } +func (t *ReleaseTargetJobTracker) successCompletionTime(job *oapi.Job) (*time.Time, bool) { + if !t.SuccessStatuses[job.Status] || job.CompletedAt == nil { + return nil, false + } + + verificationStatus := t.store.JobVerifications.GetJobVerificationStatus(job.Id) + if verificationStatus != "" && verificationStatus != oapi.JobVerificationStatusPassed { + return nil, false + } + + if verificationStatus == oapi.JobVerificationStatusPassed { + if completedAt := t.getLatestVerificationCompletedAt(job.Id); completedAt != nil { + return completedAt, true + } + } + + return job.CompletedAt, true +} + +func (t *ReleaseTargetJobTracker) getLatestVerificationCompletedAt(jobId string) *time.Time { + verifications := t.store.JobVerifications.GetByJobId(jobId) + var latest *time.Time + for _, verification := range verifications { + completedAt := verification.CompletedAt() + if completedAt == nil { + continue + } + if latest == nil || completedAt.After(*latest) { + latest = completedAt + } + } + return latest +} + // GetSuccessPercentage returns the percentage of release targets that have at least one successful job (0-100) func (t *ReleaseTargetJobTracker) GetSuccessPercentage() float32 { _, span := jobTrackerTracer.Start(context.Background(), "GetSuccessPercentage") diff --git a/apps/workspace-engine/pkg/workspace/releasemanager/policy/evaluator/environmentprogression/jobtracker_test.go b/apps/workspace-engine/pkg/workspace/releasemanager/policy/evaluator/environmentprogression/jobtracker_test.go index 7d7ae36fc..de8b72829 100644 --- a/apps/workspace-engine/pkg/workspace/releasemanager/policy/evaluator/environmentprogression/jobtracker_test.go +++ b/apps/workspace-engine/pkg/workspace/releasemanager/policy/evaluator/environmentprogression/jobtracker_test.go @@ -67,6 +67,71 @@ func setupTestStoreForJobTracker() *store.Store { return st } +func upsertVerificationWithStatus( + st *store.Store, + ctx context.Context, + jobId string, + status oapi.JobVerificationStatus, + createdAt time.Time, +) *oapi.JobVerification { + successCondition := "result.statusCode == 200" + metrics := []oapi.VerificationMetricStatus{} + + switch status { + case oapi.JobVerificationStatusPassed: + metrics = []oapi.VerificationMetricStatus{ + { + Name: "health-check", + IntervalSeconds: 30, + Count: 2, + SuccessCondition: successCondition, + Provider: oapi.MetricProvider{}, + Measurements: []oapi.VerificationMeasurement{ + {Status: oapi.Passed, MeasuredAt: createdAt}, + {Status: oapi.Passed, MeasuredAt: createdAt.Add(30 * time.Second)}, + }, + }, + } + case oapi.JobVerificationStatusFailed: + metrics = []oapi.VerificationMetricStatus{ + { + Name: "health-check", + IntervalSeconds: 30, + Count: 2, + SuccessCondition: successCondition, + Provider: oapi.MetricProvider{}, + Measurements: []oapi.VerificationMeasurement{ + {Status: oapi.Failed, MeasuredAt: createdAt}, + {Status: oapi.Failed, MeasuredAt: createdAt.Add(30 * time.Second)}, + }, + }, + } + default: + metrics = []oapi.VerificationMetricStatus{ + { + Name: "health-check", + IntervalSeconds: 30, + Count: 2, + SuccessCondition: successCondition, + Provider: oapi.MetricProvider{}, + Measurements: []oapi.VerificationMeasurement{ + {Status: oapi.Passed, MeasuredAt: createdAt}, + }, + }, + } + } + + verification := &oapi.JobVerification{ + Id: "verification-" + jobId, + JobId: jobId, + CreatedAt: createdAt, + Metrics: metrics, + } + + st.JobVerifications.Upsert(ctx, verification) + return verification +} + func TestGetReleaseTargets(t *testing.T) { st := setupTestStoreForJobTracker() ctx := context.Background() @@ -216,6 +281,73 @@ func TestReleaseTargetJobTracker_GetSuccessPercentage_WithSuccesses(t *testing.T assert.InDelta(t, expected, percentage, 0.1, "expected ~33.33%% success") } +func TestReleaseTargetJobTracker_VerificationStatus(t *testing.T) { + st := setupTestStoreForJobTracker() + ctx := context.Background() + + env, _ := st.Environments.Get("env-1") + version, _ := st.DeploymentVersions.Get("version-1") + + rt := &oapi.ReleaseTarget{ + ResourceId: "resource-1", + EnvironmentId: "env-1", + DeploymentId: "deploy-1", + } + _ = st.ReleaseTargets.Upsert(ctx, rt) + + release := &oapi.Release{ + ReleaseTarget: *rt, + Version: *version, + Variables: map[string]oapi.LiteralValue{}, + CreatedAt: time.Now().Format(time.RFC3339), + } + _ = st.Releases.Upsert(ctx, release) + + completedAt := time.Now().Add(-10 * time.Minute) + job1 := &oapi.Job{ + Id: "job-1", + ReleaseId: release.ID(), + JobAgentId: "agent-1", + Status: oapi.JobStatusSuccessful, + CreatedAt: completedAt.Add(-1 * time.Minute), + UpdatedAt: completedAt, + CompletedAt: &completedAt, + JobAgentConfig: oapi.JobAgentConfig{}, + } + st.Jobs.Upsert(ctx, job1) + + tracker := NewReleaseTargetJobTracker(ctx, st, env, version, nil) + assert.Equal(t, float32(100.0), tracker.GetSuccessPercentage(), "expected 100%% with no verification") + + upsertVerificationWithStatus(st, ctx, job1.Id, oapi.JobVerificationStatusFailed, completedAt.Add(1*time.Minute)) + + tracker2 := NewReleaseTargetJobTracker(ctx, st, env, version, nil) + assert.Equal(t, float32(0.0), tracker2.GetSuccessPercentage(), "expected 0%% when verification failed") + + completedAt2 := time.Now().Add(-5 * time.Minute) + job2 := &oapi.Job{ + Id: "job-2", + ReleaseId: release.ID(), + JobAgentId: "agent-1", + Status: oapi.JobStatusSuccessful, + CreatedAt: completedAt2.Add(-1 * time.Minute), + UpdatedAt: completedAt2, + CompletedAt: &completedAt2, + JobAgentConfig: oapi.JobAgentConfig{}, + } + st.Jobs.Upsert(ctx, job2) + + verification := upsertVerificationWithStatus(st, ctx, job2.Id, oapi.JobVerificationStatusPassed, completedAt2.Add(1*time.Minute)) + + tracker3 := NewReleaseTargetJobTracker(ctx, st, env, version, nil) + assert.Equal(t, float32(100.0), tracker3.GetSuccessPercentage(), "expected 100%% with passed verification") + + expectedCompletedAt := verification.CompletedAt() + if assert.NotNil(t, expectedCompletedAt) { + assert.True(t, tracker3.GetEarliestSuccess().Equal(*expectedCompletedAt)) + } +} + func TestReleaseTargetJobTracker_GetSuccessPercentage_AllSuccessful(t *testing.T) { st := setupTestStoreForJobTracker() ctx := context.Background()