From f24a7aeda591671a73c0c7da0e49bcfba9ce20c3 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 7 Jan 2026 14:09:22 +0100 Subject: [PATCH 1/2] update vendored code from the latest dd-trace-go version --- civisibility/constants/ci.go | 3 + civisibility/constants/env.go | 6 + civisibility/constants/git.go | 29 +- .../integrations/civisibility_features.go | 74 +++-- civisibility/integrations/manual_api.go | 300 ------------------ .../integrations/manual_api_common.go | 131 -------- .../integrations/manual_api_ddtestmodule.go | 131 -------- .../integrations/manual_api_ddtestsession.go | 185 ----------- civisibility/utils/environmentTags.go | 19 +- .../utils/filebitmap/filebitmap_test.go | 284 ----------------- civisibility/utils/git.go | 284 +++++++++++++---- civisibility/utils/net/client.go | 24 +- civisibility/utils/net/http.go | 72 +++-- civisibility/utils/net/known_tests_api.go | 4 +- civisibility/utils/net/logs_api.go | 2 +- civisibility/utils/net/searchcommits_api.go | 1 - civisibility/utils/net/sendpackfiles_api.go | 4 +- civisibility/utils/net/settings_api.go | 7 +- civisibility/utils/net/skippable.go | 4 +- .../utils/net/test_management_tests_api.go | 34 +- 20 files changed, 413 insertions(+), 1185 deletions(-) delete mode 100644 civisibility/integrations/manual_api.go delete mode 100644 civisibility/integrations/manual_api_common.go delete mode 100644 civisibility/integrations/manual_api_ddtestmodule.go delete mode 100644 civisibility/integrations/manual_api_ddtestsession.go delete mode 100644 civisibility/utils/filebitmap/filebitmap_test.go diff --git a/civisibility/constants/ci.go b/civisibility/constants/ci.go index d1ca2e8..4d98c37 100644 --- a/civisibility/constants/ci.go +++ b/civisibility/constants/ci.go @@ -6,6 +6,9 @@ package constants const ( + // CIJobID indicates the id of the CI job. + CIJobID = "ci.job.id" + // CIJobName indicates the name of the CI job. CIJobName = "ci.job.name" diff --git a/civisibility/constants/env.go b/civisibility/constants/env.go index ad6e485..fb46466 100644 --- a/civisibility/constants/env.go +++ b/civisibility/constants/env.go @@ -55,4 +55,10 @@ const ( // CIVisibilityInternalParallelEarlyFlakeDetectionEnabled indicates if the internal parallel early flake detection feature is enabled. CIVisibilityInternalParallelEarlyFlakeDetectionEnabled = "DD_CIVISIBILITY_INTERNAL_PARALLEL_EARLY_FLAKE_DETECTION_ENABLED" + + // CIVisibilitySubtestFeaturesEnabled indicates if subtest-specific management and retry features are enabled. + CIVisibilitySubtestFeaturesEnabled = "DD_CIVISIBILITY_SUBTEST_FEATURES_ENABLED" + + // CIVisibilityUseNoopTracer indicates if the ci visibility mode must set a noop tracer (avoid change current test behaviors over the noop tracer implementation) + CIVisibilityUseNoopTracer = "DD_CIVISIBILITY_USE_NOOP_TRACER" ) diff --git a/civisibility/constants/git.go b/civisibility/constants/git.go index a373132..f23209f 100644 --- a/civisibility/constants/git.go +++ b/civisibility/constants/git.go @@ -51,11 +51,38 @@ const ( GitTag = "git.tag" // GitHeadCommit indicates the GIT head commit hash. - GitHeadCommit = "git.commit.head_sha" + GitHeadCommit = "git.commit.head.sha" + + // GitHeadMessage indicates the GIT head commit message. + GitHeadMessage = "git.commit.head.message" + + // GitHeadAuthorDate indicates the GIT head commit author date. + GitHeadAuthorDate = "git.commit.head.author.date" + + // GitHeadAuthorEmail indicates the GIT head commit author email. + GitHeadAuthorEmail = "git.commit.head.author.email" + + // GitHeadAuthorName indicates the GIT head commit author name. + GitHeadAuthorName = "git.commit.head.author.name" + + // GitHeadCommitterDate indicates the GIT head commit committer date. + GitHeadCommitterDate = "git.commit.head.committer.date" + + // GitHeadCommitterEmail indicates the GIT head commit committer email. + GitHeadCommitterEmail = "git.commit.head.committer.email" + + // GitHeadCommitterName indicates the GIT head commit committer name. + GitHeadCommitterName = "git.commit.head.committer.name" // GitPrBaseCommit indicates the GIT PR base commit hash. GitPrBaseCommit = "git.pull_request.base_branch_sha" + // GitPrBaseHeadCommit indicates the GIT PR base branch head commit hash. + GitPrBaseHeadCommit = "git.pull_request.base_branch_head_sha" + // GitPrBaseBranch indicates the GIT PR base branch name. GitPrBaseBranch = "git.pull_request.base_branch" + + // PrNumber indicates the pull request number. + PrNumber = "pr.number" ) diff --git a/civisibility/integrations/civisibility_features.go b/civisibility/integrations/civisibility_features.go index 874efc8..c223428 100644 --- a/civisibility/integrations/civisibility_features.go +++ b/civisibility/integrations/civisibility_features.go @@ -11,6 +11,7 @@ import ( "os" "slices" "sync" + "time" "github.com/DataDog/ddtest/civisibility" "github.com/DataDog/ddtest/civisibility/constants" @@ -83,45 +84,56 @@ func ensureSettingsInitialization(serviceName string) { // upload the repository changes var uploadChannel = make(chan struct{}) go func() { + defer func() { + close(uploadChannel) + }() bytes, err := uploadRepositoryChanges() if err != nil { slog.Error("civisibility: error uploading repository changes:", "error", err.Error()) } else { slog.Debug("civisibility: uploaded bytes in pack files", "count", bytes) } - uploadChannel <- struct{}{} }() + //Wait for the upload with timeout func + waitUpload := func(timeout time.Duration) bool { + select { + case <-uploadChannel: + // All ok, upload succeeded + return true + case <-time.After(timeout): + slog.Warn("civisibility: timeout waiting for upload repository changes") + return false + } + } + // returns a closure suitable for PushCiVisibilityCloseAction that will wait + // for the upload to complete (or time out) using the given timeout. + waitUploadFactory := func(timeout time.Duration) func() { + return func() { waitUpload(timeout) } + } + // Get the CI Visibility settings payload for this test session ciSettings, err := ciVisibilityClient.GetSettings() if err != nil || ciSettings == nil { slog.Error("civisibility: error getting CI visibility settings", "error", err.Error()) slog.Debug("civisibility: no need to wait for the git upload to finish") // Enqueue a close action to wait for the upload to finish before finishing the process - PushCiVisibilityCloseAction(func() { - <-uploadChannel - }) + PushCiVisibilityCloseAction(waitUploadFactory(time.Minute)) return } // check if we need to wait for the upload to finish and repeat the settings request or we can just continue if ciSettings.RequireGit { slog.Debug("civisibility: waiting for the git upload to finish and repeating the settings request") - <-uploadChannel + if !waitUpload(1 * time.Minute) { + slog.Error("civisibility: error getting CI visibility settings due to timeout") + return + } ciSettings, err = ciVisibilityClient.GetSettings() if err != nil { slog.Error("civisibility: error getting CI visibility settings", "error", err.Error()) return } - } else if ciSettings.ImpactedTestsEnabled { - slog.Debug("civisibility: impacted tests is enabled we need to wait for the upload to finish (for the unshallow process)") - <-uploadChannel - } else { - slog.Debug("civisibility: no need to wait for the git upload to finish") - // Enqueue a close action to wait for the upload to finish before finishing the process - PushCiVisibilityCloseAction(func() { - <-uploadChannel - }) } // check if we need to disable EFD because known tests is not enabled @@ -138,6 +150,12 @@ func ensureSettingsInitialization(serviceName string) { ciSettings.FlakyTestRetriesEnabled = false } + // check if impacted tests is disabled by env-vars + if ciSettings.ImpactedTestsEnabled && !civisibility.BoolEnv(constants.CIVisibilityImpactedTestsDetectionEnabled, true) { + slog.Warn("civisibility: impacted tests was disabled by the environment variable") + ciSettings.ImpactedTestsEnabled = false + } + // check if test management is disabled by env-vars if ciSettings.TestManagement.Enabled && !civisibility.BoolEnv(constants.CIVisibilityTestManagementEnabledEnvironmentVariable, true) { slog.Warn("civisibility: test management was disabled by the environment variable") @@ -150,6 +168,23 @@ func ensureSettingsInitialization(serviceName string) { ciSettings.TestManagement.AttemptToFixRetries = testManagementAttemptToFixRetriesEnv } + // determine if subtest-specific features are enabled via environment variables + subtestFeaturesEnabled := civisibility.BoolEnv(constants.CIVisibilitySubtestFeaturesEnabled, true) + if !subtestFeaturesEnabled { + slog.Debug("civisibility: subtest test management features disabled by environment variable") + } + ciSettings.SubtestFeaturesEnabled = subtestFeaturesEnabled + + // check if we need to wait for the upload to finish before continuing + if ciSettings.ImpactedTestsEnabled { + slog.Debug("civisibility: impacted tests is enabled we need to wait for the upload to finish (for the unshallow process)") + waitUpload(30 * time.Second) + } else { + slog.Debug("civisibility: no need to wait for the git upload to finish") + // Enqueue a close action to wait for the upload to finish before finishing the process + PushCiVisibilityCloseAction(waitUploadFactory(time.Minute)) + } + // set the ciVisibilitySettings with the settings from the backend ciVisibilitySettings = *ciSettings }) @@ -184,7 +219,7 @@ func ensureAdditionalFeaturesInitialization(_ string) { additionalTags[constants.LibraryCapabilitiesTestImpactAnalysis] = "1" additionalTags[constants.LibraryCapabilitiesTestManagementQuarantine] = "1" additionalTags[constants.LibraryCapabilitiesTestManagementDisable] = "1" - additionalTags[constants.LibraryCapabilitiesTestManagementAttemptToFix] = "2" + additionalTags[constants.LibraryCapabilitiesTestManagementAttemptToFix] = "5" // mutex to protect the additional tags map var aTagsMutex sync.Mutex @@ -258,8 +293,7 @@ func ensureAdditionalFeaturesInitialization(_ string) { } // if wheter the settings response or the env var is true we load the impacted tests analyzer - if currentSettings.ImpactedTestsEnabled || - civisibility.BoolEnv(constants.CIVisibilityImpactedTestsDetectionEnabled, false) { + if currentSettings.ImpactedTestsEnabled { wg.Add(1) go func() { defer wg.Done() @@ -324,7 +358,7 @@ func uploadRepositoryChanges() (bytes int64, err error) { // get the search commits response initialCommitData, err := getSearchCommits() if err != nil { - return 0, fmt.Errorf("civisibility: error getting the search commits response: %s", err.Error()) + return 0, fmt.Errorf("civisibility: error getting the search commits response: %s", err) } // let's check if we could retrieve commit data @@ -363,11 +397,11 @@ func uploadRepositoryChanges() (bytes int64, err error) { // after unshallowing the repository we need to get the search commits to calculate the missing commits again commitsData, err := getSearchCommits() if err != nil { - return 0, fmt.Errorf("civisibility: error getting the search commits response: %s", err.Error()) + return 0, fmt.Errorf("civisibility: error getting the search commits response: %s", err) } // let's check if we could retrieve commit data - if !initialCommitData.IsOk { + if !commitsData.IsOk { return 0, nil } diff --git a/civisibility/integrations/manual_api.go b/civisibility/integrations/manual_api.go deleted file mode 100644 index f0222c2..0000000 --- a/civisibility/integrations/manual_api.go +++ /dev/null @@ -1,300 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2024 Datadog, Inc. - -package integrations - -import ( - "context" - "runtime" - "time" -) - -// TestResultStatus represents the result status of a test. -type TestResultStatus int - -const ( - // ResultStatusPass indicates that the test has passed. - ResultStatusPass TestResultStatus = 0 - - // ResultStatusFail indicates that the test has failed. - ResultStatusFail TestResultStatus = 1 - - // ResultStatusSkip indicates that the test has been skipped. - ResultStatusSkip TestResultStatus = 2 -) - -// ErrorOption is a function that sets an option for creating an error. -type ErrorOption func(*tslvErrorOptions) - -// tslvErrorOptions is a struct that holds options for creating an error. -type tslvErrorOptions struct { - err error - errType string - message string - callstack string -} - -// WithError sets the error on the options. -func WithError(err error) ErrorOption { - return func(o *tslvErrorOptions) { o.err = err } -} - -// WithErrorInfo sets detailed error information on the options. -func WithErrorInfo(errType string, message string, callstack string) ErrorOption { - return func(o *tslvErrorOptions) { - o.errType = errType - o.message = message - o.callstack = callstack - } -} - -// ddTslvEvent is an interface that provides common methods for CI visibility events. -type ddTslvEvent interface { - // Context returns the context of the event. - Context() context.Context - - // StartTime returns the start time of the event. - StartTime() time.Time - - // SetError sets an error on the event. - SetError(options ...ErrorOption) - - // SetTag sets a tag on the event. - SetTag(key string, value interface{}) - - // GetTag retrieves a tag from the event. - GetTag(key string) (interface{}, bool) -} - -// TestSessionStartOption represents an option that can be passed to CreateTestSession. -type TestSessionStartOption func(*tslvTestSessionStartOptions) - -// tslvTestSessionStartOptions contains the options for creating a new test session. -type tslvTestSessionStartOptions struct { - command string - workingDirectory string - framework string - frameworkVersion string - startTime time.Time -} - -// WithTestSessionCommand sets the command used to run the test session. -func WithTestSessionCommand(command string) TestSessionStartOption { - return func(o *tslvTestSessionStartOptions) { o.command = command } -} - -// WithTestSessionWorkingDirectory sets the working directory of the test session. -func WithTestSessionWorkingDirectory(workingDirectory string) TestSessionStartOption { - return func(o *tslvTestSessionStartOptions) { o.workingDirectory = workingDirectory } -} - -// WithTestSessionFramework sets the testing framework used in the test session. -func WithTestSessionFramework(framework, frameworkVersion string) TestSessionStartOption { - return func(o *tslvTestSessionStartOptions) { - o.framework = framework - o.frameworkVersion = frameworkVersion - } -} - -// WithTestSessionStartTime sets the start time of the test session. -func WithTestSessionStartTime(startTime time.Time) TestSessionStartOption { - return func(o *tslvTestSessionStartOptions) { o.startTime = startTime } -} - -// TestSessionCloseOption represents an option that can be passed to Close. -type TestSessionCloseOption func(*tslvTestSessionCloseOptions) - -// tslvTestSessionCloseOptions contains the options for closing a test session. -type tslvTestSessionCloseOptions struct { - finishTime time.Time -} - -// WithTestSessionFinishTime sets the finish time of the test session. -func WithTestSessionFinishTime(finishTime time.Time) TestSessionCloseOption { - return func(o *tslvTestSessionCloseOptions) { o.finishTime = finishTime } -} - -// TestModuleStartOption represents an option that can be passed to GetOrCreateModule. -type TestModuleStartOption func(*tslvTestModuleStartOptions) - -// tslvTestModuleOptions contains the options for creating a new test module. -type tslvTestModuleStartOptions struct { - framework string - frameworkVersion string - startTime time.Time -} - -// WithTestModuleFramework sets the testing framework used by the test module. -func WithTestModuleFramework(framework, frameworkVersion string) TestModuleStartOption { - return func(o *tslvTestModuleStartOptions) { - o.framework = framework - o.frameworkVersion = frameworkVersion - } -} - -// WithTestModuleStartTime sets the start time of the test module. -func WithTestModuleStartTime(startTime time.Time) TestModuleStartOption { - return func(o *tslvTestModuleStartOptions) { o.startTime = startTime } -} - -// TestSession represents a session for a set of tests. -type TestSession interface { - ddTslvEvent - - // SessionID returns the ID of the session. - SessionID() uint64 - - // Command returns the command used to run the session. - Command() string - - // Framework returns the testing framework used. - Framework() string - - // WorkingDirectory returns the working directory of the session. - WorkingDirectory() string - - // Close closes the test session with the given exit code. - Close(exitCode int, options ...TestSessionCloseOption) - - // GetOrCreateModule returns an existing module or creates a new one with the given name. - GetOrCreateModule(name string, options ...TestModuleStartOption) TestModule -} - -// TestModuleCloseOption represents an option for closing a test module. -type TestModuleCloseOption func(*tslvTestModuleCloseOptions) - -// tslvTestModuleCloseOptions represents the options for closing a test module. -type tslvTestModuleCloseOptions struct { - finishTime time.Time -} - -// WithTestModuleFinishTime sets the finish time for closing the test module. -func WithTestModuleFinishTime(finishTime time.Time) TestModuleCloseOption { - return func(o *tslvTestModuleCloseOptions) { o.finishTime = finishTime } -} - -// TestSuiteStartOption represents an option for starting a test suite. -type TestSuiteStartOption func(*tslvTestSuiteStartOptions) - -// tslvTestSuiteStartOptions represents the options for starting a test suite. -type tslvTestSuiteStartOptions struct { - startTime time.Time -} - -// WithTestSuiteStartTime sets the start time for starting a test suite. -func WithTestSuiteStartTime(startTime time.Time) TestSuiteStartOption { - return func(o *tslvTestSuiteStartOptions) { o.startTime = startTime } -} - -// TestModule represents a module within a test session. -type TestModule interface { - ddTslvEvent - - // ModuleID returns the ID of the module. - ModuleID() uint64 - - // Session returns the test session to which the module belongs. - Session() TestSession - - // Framework returns the testing framework used by the module. - Framework() string - - // Name returns the name of the module. - Name() string - - // Close closes the test module. - Close(options ...TestModuleCloseOption) -} - -// TestSuiteCloseOption represents an option for closing a test suite. -type TestSuiteCloseOption func(*tslvTestSuiteCloseOptions) - -// tslvTestSuiteCloseOptions represents the options for closing a test suite. -type tslvTestSuiteCloseOptions struct { - finishTime time.Time -} - -// WithTestSuiteFinishTime sets the finish time for closing the test suite. -func WithTestSuiteFinishTime(finishTime time.Time) TestSuiteCloseOption { - return func(o *tslvTestSuiteCloseOptions) { o.finishTime = finishTime } -} - -// TestStartOption represents an option for starting a test. -type TestStartOption func(*tslvTestStartOptions) - -// tslvTestStartOptions represents the options for starting a test. -type tslvTestStartOptions struct { - startTime time.Time -} - -// WithTestStartTime sets the start time for starting a test. -func WithTestStartTime(startTime time.Time) TestStartOption { - return func(o *tslvTestStartOptions) { o.startTime = startTime } -} - -// TestSuite represents a suite of tests within a module. -type TestSuite interface { - ddTslvEvent - - // SuiteID returns the ID of the suite. - SuiteID() uint64 - - // Module returns the module to which the suite belongs. - Module() TestModule - - // Name returns the name of the suite. - Name() string - - // Close closes the test suite. - Close(options ...TestSuiteCloseOption) - - // CreateTest creates a new test with the given name and options. - CreateTest(name string, options ...TestStartOption) Test -} - -// TestCloseOption represents an option for closing a test. -type TestCloseOption func(*tslvTestCloseOptions) - -// tslvTestCloseOptions represents the options for closing a test. -type tslvTestCloseOptions struct { - finishTime time.Time - skipReason string -} - -// WithTestFinishTime sets the finish time of the test. -func WithTestFinishTime(finishTime time.Time) TestCloseOption { - return func(o *tslvTestCloseOptions) { o.finishTime = finishTime } -} - -// WithTestSkipReason sets the skip reason of the test. -func WithTestSkipReason(skipReason string) TestCloseOption { - return func(o *tslvTestCloseOptions) { o.skipReason = skipReason } -} - -// Test represents an individual test within a suite. -type Test interface { - ddTslvEvent - - // TestID returns the ID of the test. - TestID() uint64 - - // Name returns the name of the test. - Name() string - - // Suite returns the suite to which the test belongs. - Suite() TestSuite - - // Close closes the test with the given status. - Close(status TestResultStatus, options ...TestCloseOption) - - // SetTestFunc sets the function to be tested. (Sets the test.source tags and test.codeowners) - SetTestFunc(fn *runtime.Func) - - // SetBenchmarkData sets benchmark data for the test. - SetBenchmarkData(measureType string, data map[string]any) - - // Log logs a message with the given tags. - Log(message string, tags string) -} diff --git a/civisibility/integrations/manual_api_common.go b/civisibility/integrations/manual_api_common.go deleted file mode 100644 index 6fa0900..0000000 --- a/civisibility/integrations/manual_api_common.go +++ /dev/null @@ -1,131 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2024 Datadog, Inc. - -package integrations - -import ( - "context" - "sync" - "time" - _ "unsafe" // for go:linkname - - "github.com/DataDog/dd-trace-go/v2/ddtrace/ext" - "github.com/DataDog/dd-trace-go/v2/ddtrace/tracer" - "github.com/DataDog/ddtest/civisibility/constants" - "github.com/DataDog/ddtest/civisibility/utils" -) - -// Go linknames - -//go:linkname getMeta github.com/DataDog/dd-trace-go/v2/ddtrace/tracer.getMeta -func getMeta(s *tracer.Span, key string) (string, bool) - -//go:linkname getMetric github.com/DataDog/dd-trace-go/v2/ddtrace/tracer.getMetric -func getMetric(s *tracer.Span, key string) (float64, bool) - -// common -var _ ddTslvEvent = (*ciVisibilityCommon)(nil) - -// ciVisibilityCommon is a struct that implements the ddTslvEvent interface and provides common functionality for CI visibility. -type ciVisibilityCommon struct { - mutex sync.Mutex - startTime time.Time - - tags []tracer.StartSpanOption - span *tracer.Span - closed bool - - ctxMutex sync.Mutex - ctx context.Context -} - -// Context returns the context of the event. -func (c *ciVisibilityCommon) Context() context.Context { - c.ctxMutex.Lock() - defer c.ctxMutex.Unlock() - return c.ctx -} - -// StartTime returns the start time of the event. -func (c *ciVisibilityCommon) StartTime() time.Time { return c.startTime } - -// SetError sets an error on the event. -func (c *ciVisibilityCommon) SetError(options ...ErrorOption) { - defaults := &tslvErrorOptions{} - for _, o := range options { - o(defaults) - } - - // if there is an error, set the span with the error - if defaults.err != nil { - c.span.SetTag(ext.Error, defaults.err) - return - } - - // if there is no error, set the span with error the error info - - // set the span with error:1 - c.span.SetTag(ext.Error, true) - - // set the error type - if defaults.errType != "" { - c.span.SetTag(ext.ErrorType, defaults.errType) - } - - // set the error message - if defaults.message != "" { - c.span.SetTag(ext.ErrorMsg, defaults.message) - } - - // set the error stacktrace - if defaults.callstack != "" { - c.span.SetTag(ext.ErrorStack, defaults.callstack) - } -} - -// SetTag sets a tag on the event. -func (c *ciVisibilityCommon) SetTag(key string, value interface{}) { c.span.SetTag(key, value) } - -// GetTag retrieves a tag from the event. -func (c *ciVisibilityCommon) GetTag(key string) (interface{}, bool) { - // Check if the span is nil - if c.span == nil { - return nil, false - } - - // Check if the key is a meta key - metaVal, ok := getMeta(c.span, key) - if ok { - return metaVal, true - } - - // Check if the key is a metric key - metricVal, ok := getMetric(c.span, key) - return metricVal, ok -} - -// fillCommonTags adds common tags to the span options for CI visibility. -func fillCommonTags(opts []tracer.StartSpanOption) []tracer.StartSpanOption { - opts = append(opts, []tracer.StartSpanOption{ - tracer.Tag(constants.Origin, constants.CIAppTestOrigin), - tracer.Tag(ext.ManualKeep, true), - }...) - - // Apply CI tags - for k, v := range utils.GetCITags() { - // Ignore the test session name (sent at the payload metadata level, see `civisibility_payload.go`) - if k == constants.TestSessionName { - continue - } - opts = append(opts, tracer.Tag(k, v)) - } - - // Apply CI metrics - for k, v := range utils.GetCIMetrics() { - opts = append(opts, tracer.Tag(k, v)) - } - - return opts -} diff --git a/civisibility/integrations/manual_api_ddtestmodule.go b/civisibility/integrations/manual_api_ddtestmodule.go deleted file mode 100644 index 81c8eeb..0000000 --- a/civisibility/integrations/manual_api_ddtestmodule.go +++ /dev/null @@ -1,131 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2024 Datadog, Inc. - -package integrations - -import ( - "context" - "fmt" - "slices" - "strings" - "time" - - "github.com/DataDog/dd-trace-go/v2/ddtrace/tracer" - "github.com/DataDog/ddtest/civisibility/constants" -) - -// Test Module - -// Ensures that tslvTestModule implements the TestModule interface. -var _ TestModule = (*tslvTestModule)(nil) - -// tslvTestModule implements the DdTestModule interface and represents a module within a test session. -type tslvTestModule struct { - ciVisibilityCommon - session *tslvTestSession - moduleID uint64 - name string - framework string - - suites map[string]TestSuite -} - -// createTestModule initializes a new test module within a given session. -func createTestModule(session *tslvTestSession, name string, framework string, frameworkVersion string, startTime time.Time) TestModule { - // Ensure CI visibility is properly configured. - EnsureCiVisibilityInitialization() - - operationName := "test_module" - if framework != "" { - operationName = fmt.Sprintf("%s.%s", strings.ToLower(framework), operationName) - } - - resourceName := name - - var sessionTags []tracer.StartSpanOption - if session != nil { - sessionTags = slices.Clone(session.tags) - } - - // Module tags should include session tags so the backend can calculate the session fingerprint from the module. - moduleTags := append(sessionTags, []tracer.StartSpanOption{ - tracer.Tag(constants.TestType, constants.TestTypeTest), - tracer.Tag(constants.TestModule, name), - tracer.Tag(constants.TestFramework, framework), - tracer.Tag(constants.TestFrameworkVersion, frameworkVersion), - }...) - - testOpts := append(fillCommonTags([]tracer.StartSpanOption{ - tracer.ResourceName(resourceName), - tracer.SpanType(constants.SpanTypeTestModule), - tracer.StartTime(startTime), - }), moduleTags...) - - span, ctx := tracer.StartSpanFromContext(context.Background(), operationName, testOpts...) - moduleID := span.Context().SpanID() - if session != nil { - span.SetTag(constants.TestSessionIDTag, fmt.Sprint(session.sessionID)) - } - span.SetTag(constants.TestModuleIDTag, fmt.Sprint(moduleID)) - - module := &tslvTestModule{ - session: session, - moduleID: moduleID, - name: name, - framework: framework, - suites: map[string]TestSuite{}, - ciVisibilityCommon: ciVisibilityCommon{ - startTime: startTime, - tags: moduleTags, - span: span, - ctx: ctx, - }, - } - - // Ensure to close everything before CI visibility exits. In CI visibility mode, we try to never lose data. - PushCiVisibilityCloseAction(func() { module.Close() }) - - return module -} - -// ModuleID returns the ID of the module. -func (t *tslvTestModule) ModuleID() uint64 { - return t.moduleID -} - -// Name returns the name of the test module. -func (t *tslvTestModule) Name() string { return t.name } - -// Framework returns the testing framework used by the test module. -func (t *tslvTestModule) Framework() string { return t.framework } - -// Session returns the test session to which the test module belongs. -func (t *tslvTestModule) Session() TestSession { return t.session } - -// Close closes the test module. -func (t *tslvTestModule) Close(options ...TestModuleCloseOption) { - t.mutex.Lock() - defer t.mutex.Unlock() - if t.closed { - return - } - - defaults := &tslvTestModuleCloseOptions{} - for _, o := range options { - o(defaults) - } - - if defaults.finishTime.IsZero() { - defaults.finishTime = time.Now() - } - - for _, suite := range t.suites { - suite.Close() - } - t.suites = map[string]TestSuite{} - - t.span.Finish(tracer.FinishTime(defaults.finishTime)) - t.closed = true -} diff --git a/civisibility/integrations/manual_api_ddtestsession.go b/civisibility/integrations/manual_api_ddtestsession.go deleted file mode 100644 index df75fb1..0000000 --- a/civisibility/integrations/manual_api_ddtestsession.go +++ /dev/null @@ -1,185 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2024 Datadog, Inc. - -package integrations - -import ( - "context" - "fmt" - "os" - "strings" - "time" - - "github.com/DataDog/dd-trace-go/v2/ddtrace/tracer" - "github.com/DataDog/ddtest/civisibility/constants" - "github.com/DataDog/ddtest/civisibility/utils" -) - -// Test Session - -// Ensures that tslvTestSession implements the TestSession interface. -var _ TestSession = (*tslvTestSession)(nil) - -// tslvTestSession implements the DdTestSession interface and represents a session for a set of tests. -type tslvTestSession struct { - ciVisibilityCommon - sessionID uint64 - command string - workingDirectory string - framework string - frameworkVersion string - - modules map[string]TestModule -} - -// CreateTestSession initializes a new test session with the given command and working directory. -func CreateTestSession(options ...TestSessionStartOption) TestSession { - defaults := &tslvTestSessionStartOptions{} - for _, f := range options { - f(defaults) - } - - if defaults.command == "" { - defaults.command = utils.GetCITags()[constants.TestCommand] - } - if defaults.workingDirectory == "" { - wd, err := os.Getwd() - if err == nil { - wd = utils.GetRelativePathFromCITagsSourceRoot(wd) - } - defaults.workingDirectory = wd - } - if defaults.startTime.IsZero() { - defaults.startTime = time.Now() - } - - // Ensure CI visibility is properly configured. - EnsureCiVisibilityInitialization() - - sessionTags := []tracer.StartSpanOption{ - tracer.Tag(constants.TestType, constants.TestTypeTest), - tracer.Tag(constants.TestCommand, defaults.command), - tracer.Tag(constants.TestCommandWorkingDirectory, defaults.workingDirectory), - } - - operationName := "test_session" - if defaults.framework != "" { - operationName = fmt.Sprintf("%s.%s", strings.ToLower(defaults.framework), operationName) - sessionTags = append(sessionTags, - tracer.Tag(constants.TestFramework, defaults.framework), - tracer.Tag(constants.TestFrameworkVersion, defaults.frameworkVersion)) - } - - resourceName := fmt.Sprintf("%s.%s", operationName, defaults.command) - - testOpts := append(fillCommonTags([]tracer.StartSpanOption{ - tracer.ResourceName(resourceName), - tracer.SpanType(constants.SpanTypeTestSession), - tracer.StartTime(defaults.startTime), - }), sessionTags...) - - span, ctx := tracer.StartSpanFromContext(context.Background(), operationName, testOpts...) - sessionID := span.Context().SpanID() - span.SetTag(constants.TestSessionIDTag, fmt.Sprint(sessionID)) - - s := &tslvTestSession{ - sessionID: sessionID, - command: defaults.command, - workingDirectory: defaults.workingDirectory, - framework: defaults.framework, - frameworkVersion: defaults.frameworkVersion, - modules: map[string]TestModule{}, - ciVisibilityCommon: ciVisibilityCommon{ - startTime: defaults.startTime, - tags: sessionTags, - span: span, - ctx: ctx, - }, - } - - // Ensure to close everything before CI visibility exits. In CI visibility mode, we try to never lose data. - PushCiVisibilityCloseAction(func() { s.Close(1) }) - - return s -} - -// SessionID returns the ID of the test session. -func (t *tslvTestSession) SessionID() uint64 { - return t.sessionID -} - -// Command returns the command used to run the test session. -func (t *tslvTestSession) Command() string { return t.command } - -// Framework returns the testing framework used in the test session. -func (t *tslvTestSession) Framework() string { return t.framework } - -// WorkingDirectory returns the working directory of the test session. -func (t *tslvTestSession) WorkingDirectory() string { return t.workingDirectory } - -// Close closes the test session with the given exit code. -func (t *tslvTestSession) Close(exitCode int, options ...TestSessionCloseOption) { - t.mutex.Lock() - defer t.mutex.Unlock() - if t.closed { - return - } - - defaults := &tslvTestSessionCloseOptions{} - for _, f := range options { - f(defaults) - } - - if defaults.finishTime.IsZero() { - defaults.finishTime = time.Now() - } - - for _, m := range t.modules { - m.Close() - } - t.modules = map[string]TestModule{} - - t.span.SetTag(constants.TestCommandExitCode, exitCode) - if exitCode == 0 { - t.span.SetTag(constants.TestStatus, constants.TestStatusPass) - } else { - t.SetError(WithErrorInfo("ExitCode", "exit code is not zero.", "")) - t.span.SetTag(constants.TestStatus, constants.TestStatusFail) - } - - t.span.Finish(tracer.FinishTime(defaults.finishTime)) - t.closed = true - - tracer.Flush() -} - -// GetOrCreateModule returns an existing module or creates a new one with the given name, framework, framework version, and start time. -func (t *tslvTestSession) GetOrCreateModule(name string, options ...TestModuleStartOption) TestModule { - t.mutex.Lock() - defer t.mutex.Unlock() - - defaults := &tslvTestModuleStartOptions{} - for _, f := range options { - f(defaults) - } - - if defaults.framework == "" { - defaults.framework = t.framework - defaults.frameworkVersion = t.frameworkVersion - } - if defaults.startTime.IsZero() { - defaults.startTime = time.Now() - } - - var mod TestModule - if v, ok := t.modules[name]; ok { - mod = v - } else { - mod = createTestModule(t, name, defaults.framework, defaults.frameworkVersion, defaults.startTime) - t.modules[name] = mod - } - - return mod -} diff --git a/civisibility/utils/environmentTags.go b/civisibility/utils/environmentTags.go index 402225c..c4a8b25 100644 --- a/civisibility/utils/environmentTags.go +++ b/civisibility/utils/environmentTags.go @@ -306,6 +306,23 @@ func createCITagsMap() map[string]string { } } + // If the head commit SHA is available, populate additional Git head metadata + if headCommitSha, ok := localTags[constants.GitHeadCommit]; ok { + if headCommitData, err := fetchCommitData(headCommitSha); err != nil { + slog.Warn("civisibility: failed to fetch head commit data for", "commitSha", headCommitSha, "error", err.Error()) + } else if headCommitSha == headCommitData.CommitSha { + localTags[constants.GitHeadAuthorDate] = headCommitData.AuthorDate.String() + localTags[constants.GitHeadAuthorName] = headCommitData.AuthorName + localTags[constants.GitHeadAuthorEmail] = headCommitData.AuthorEmail + localTags[constants.GitHeadCommitterDate] = headCommitData.CommitterDate.String() + localTags[constants.GitHeadCommitterName] = headCommitData.CommitterName + localTags[constants.GitHeadCommitterEmail] = headCommitData.CommitterEmail + localTags[constants.GitHeadMessage] = headCommitData.CommitMessage + } else { + slog.Warn("civisibility: head commit SHA does not match the fetched commit SHA", "commitSha", headCommitSha, "fetchedCommitSha", headCommitData.CommitSha) + } + } + // Apply environmental data if is available applyEnvironmentalDataIfRequired(localTags) @@ -323,6 +340,6 @@ func createCIMetricsMap() map[string]float64 { localMetrics := make(map[string]float64) localMetrics[constants.LogicalCPUCores] = float64(runtime.NumCPU()) - slog.Debug("civisibility: common metrics created", "count", len(localMetrics)) + slog.Debug("civisibility: common metrics created with", "items", len(localMetrics)) return localMetrics } diff --git a/civisibility/utils/filebitmap/filebitmap_test.go b/civisibility/utils/filebitmap/filebitmap_test.go deleted file mode 100644 index d8872f2..0000000 --- a/civisibility/utils/filebitmap/filebitmap_test.go +++ /dev/null @@ -1,284 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2025 Datadog, Inc. - -package filebitmap - -import ( - "testing" -) - -// TestConstructorWithSizeCreatesEmptyBitmap verifies that a new bitmap (created from a byte slice) -// has the expected size and that all bits are initially false. -func TestConstructorWithSizeCreatesEmptyBitmap(t *testing.T) { - lines := 135 - size := getSize(lines) - bitmap := NewFileBitmapFromBytes(make([]byte, size)) - - if bitmap.Size() != size { - t.Errorf("expected size %d, got %d", size, bitmap.Size()) - } - - // Check each bit (1-indexed) is false. - for i := 0; i < lines; i++ { - if bitmap.Get(i + 1) { - t.Errorf("expected bit %d to be false", i+1) - } - } -} - -// TestSetSingleBitSetsBitCorrectly verifies that setting a single bit works. -func TestSetSingleBitSetsBitCorrectly(t *testing.T) { - bitmap := NewFileBitmapFromBytes(make([]byte, 1)) - bitmap.Set(1) // Set the first bit - if !bitmap.Get(1) { - t.Errorf("expected bit 1 to be set") - } -} - -// TestCountActiveBitsNoBitsSetReturnsZero verifies that a bitmap with no bits set returns zero. -func TestCountActiveBitsNoBitsSetReturnsZero(t *testing.T) { - bitmap := NewFileBitmapFromBytes(make([]byte, 1)) - if count := bitmap.CountActiveBits(); count != 0 { - t.Errorf("expected 0 active bits, got %d", count) - } -} - -// TestCountActiveBitsOneBitSetReturnsOne verifies that a bitmap with one bit set returns one. -func TestCountActiveBitsOneBitSetReturnsOne(t *testing.T) { - bitmap := NewFileBitmapFromBytes(make([]byte, 1)) - bitmap.Set(1) - if count := bitmap.CountActiveBits(); count != 1 { - t.Errorf("expected 1 active bit, got %d", count) - } -} - -// TestBitwiseOrTwoBitmapsCombinesCorrectly verifies the OR operation on two bitmaps. -func TestBitwiseOrTwoBitmapsCombinesCorrectly(t *testing.T) { - bitmapA := NewFileBitmapFromBytes([]byte{0b00000001}) - bitmapB := NewFileBitmapFromBytes([]byte{0b00000010}) - resultBitmap := Or(bitmapA, bitmapB, false) - result := resultBitmap.ToArray() - if result[0] != 0b00000011 { - t.Errorf("expected 0b00000011, got %08b", result[0]) - } -} - -// TestBitwiseAndTwoBitmapsIntersectsCorrectly verifies the AND operation on two bitmaps. -func TestBitwiseAndTwoBitmapsIntersectsCorrectly(t *testing.T) { - bitmapA := NewFileBitmapFromBytes([]byte{0b00000011}) - bitmapB := NewFileBitmapFromBytes([]byte{0b00000010}) - resultBitmap := And(bitmapA, bitmapB, false) - result := resultBitmap.ToArray() - if result[0] != 0b00000010 { - t.Errorf("expected 0b00000010, got %08b", result[0]) - } -} - -// TestBitwiseNotSingleBitmapInvertsCorrectly verifies that the NOT operation correctly inverts the bits. -func TestBitwiseNotSingleBitmapInvertsCorrectly(t *testing.T) { - bitmap := NewFileBitmapFromBytes([]byte{0b11111110}) - resultBitmap := Not(bitmap, false) - result := resultBitmap.ToArray() - if result[0] != 0b00000001 { - t.Errorf("expected 0b00000001, got %08b", result[0]) - } -} - -// TestLargeBitmapBitwiseOperationsHandleCorrectly verifies bitwise operations on larger bitmaps. -func TestLargeBitmapBitwiseOperationsHandleCorrectly(t *testing.T) { - size := 1024 // 1 KB in bytes - bitmapA := NewFileBitmapFromBytes(make([]byte, size)) - bitmapB := NewFileBitmapFromBytes(make([]byte, size)) - totalBits := size * 8 - - // Set alternating bits in bitmapA and bitmapB with shifted positions. - for i := 1; i <= totalBits; i += 2 { - bitmapA.Set(i) - if i+1 <= totalBits { - bitmapB.Set(i + 1) - } - } - - resultOr := Or(bitmapA, bitmapB, false) - resultAnd := And(bitmapA, bitmapB, false) - - if count := resultOr.CountActiveBits(); count != totalBits { - t.Errorf("expected all %d bits set in OR result, got %d", totalBits, count) - } - if count := resultAnd.CountActiveBits(); count != 0 { - t.Errorf("expected 0 bits set in AND result, got %d", count) - } -} - -// TestBitwiseNotComplexPatternInvertsCorrectly verifies that a complex pattern is inverted properly. -func TestBitwiseNotComplexPatternInvertsCorrectly(t *testing.T) { - size := 256 // 256 bytes - pattern := make([]byte, size) - for i := 0; i < size; i++ { - if i%2 == 0 { - pattern[i] = 0xAA - } else { - pattern[i] = 0x55 - } - } - - bitmap := NewFileBitmapFromBytes(pattern) - invertedBitmap := Not(bitmap, false) - totalBits := size * 8 - - for i := 0; i < totalBits; i++ { - originalBit := bitmap.Get(i + 1) - invertedBit := invertedBitmap.Get(i + 1) - if originalBit == invertedBit { - t.Errorf("bit %d: expected inversion, got same value", i+1) - } - } -} - -// TestToArrayWithVariousBitSetsReturnsExpectedByteArray verifies that setting various bits produces the expected byte array. -func TestToArrayWithVariousBitSetsReturnsExpectedByteArray(t *testing.T) { - tests := []struct { - bitsToSet []int - expected []byte - }{ - {[]int{1, 8}, []byte{0b10000001, 0x00, 0x00, 0x00}}, - {[]int{9, 32}, []byte{0x00, 0b10000000, 0x00, 0b00000001}}, - {[]int{1, 8, 9, 32}, []byte{0b10000001, 0b10000000, 0x00, 0b00000001}}, - } - - for _, tt := range tests { - bitmap := NewFileBitmapFromBytes(make([]byte, 4)) // 4 bytes = 32 bits - for _, bit := range tt.bitsToSet { - bitmap.Set(bit) - } - actual := bitmap.ToArray() - if len(actual) != len(tt.expected) { - t.Errorf("expected array length %d, got %d", len(tt.expected), len(actual)) - } - for i, b := range tt.expected { - if actual[i] != b { - t.Errorf("at index %d, expected %08b, got %08b", i, b, actual[i]) - } - } - } -} - -// TestEnumeratorCorrectlyIteratesOverBits verifies that iterating over the internal byte slice -// yields the correct bit states. (Note: our Go implementation does not provide a dedicated enumerator, -// so we iterate over the underlying data.) -func TestEnumeratorCorrectlyIteratesOverBits(t *testing.T) { - tests := []struct { - bitmapBytes []byte - expectedBitStates []bool - }{ - { - []byte{0b10101010}, - []bool{true, false, true, false, true, false, true, false}, - }, - { - []byte{0b11110000}, - []bool{true, true, true, true, false, false, false, false}, - }, - { - []byte{0b00000000, 0b11111111}, - []bool{ - false, false, false, false, false, false, false, false, - true, true, true, true, true, true, true, true, - }, - }, - } - - for _, tt := range tests { - bitmap := NewFileBitmapFromBytes(tt.bitmapBytes) - var iterated []bool - for _, b := range bitmap.data { - for bitPos := 0; bitPos < 8; bitPos++ { - // Extract bit from most significant to least significant. - bit := (b & (1 << (7 - bitPos))) != 0 - iterated = append(iterated, bit) - } - } - if len(iterated) != len(tt.expectedBitStates) { - t.Errorf("expected %d bits, got %d", len(tt.expectedBitStates), len(iterated)) - } - for i, expected := range tt.expectedBitStates { - if iterated[i] != expected { - t.Errorf("bit %d: expected %v, got %v", i, expected, iterated[i]) - } - } - } -} - -// TestGivenARangeWhenCreatingFileBitmapProperBitsAreSet verifies that FromActiveRange -// properly sets bits in a given range and panics on invalid ranges. -func TestGivenARangeWhenCreatingFileBitmapProperBitsAreSet(t *testing.T) { - tests := []struct { - from int - to int - shouldPanic bool - }{ - {0, 10, true}, - {15, 10, true}, - {1, 10, false}, - {5, 15, false}, - {5, 5, false}, - } - - for _, tt := range tests { - if tt.shouldPanic { - didPanic := false - func() { - defer func() { - if r := recover(); r != nil { - didPanic = true - } - }() - _ = FromActiveRange(tt.from, tt.to) - }() - if !didPanic { - t.Errorf("expected panic for range (%d, %d)", tt.from, tt.to) - } - } else { - bitmap := FromActiveRange(tt.from, tt.to) - if bitmap.BitCount() < tt.to { - t.Errorf("expected bitmap bit count to be >= %d, got %d", tt.to, bitmap.BitCount()) - } - for x := 1; x <= bitmap.BitCount(); x++ { - bit := bitmap.Get(x) - expected := (x >= tt.from && x <= tt.to) - if bit != expected { - t.Errorf("bit %d: expected %v, got %v", x, expected, bit) - } - } - } - } -} - -// TestGivenTwoRangesWhenIntersectingFileBitmapsResultIsExpected verifies that the -// IntersectsWith method correctly identifies overlapping ranges. -func TestGivenTwoRangesWhenIntersectingFileBitmapsResultIsExpected(t *testing.T) { - tests := []struct { - from1, to1 int - from2, to2 int - intersect bool - }{ - {1, 10, 11, 15, false}, - {11, 15, 1, 10, false}, - {1, 10, 8, 15, true}, - {8, 15, 10, 15, true}, - {8, 15, 15, 16, true}, - {30, 36, 35, 35, true}, - } - - for _, tt := range tests { - bitmap1 := FromActiveRange(tt.from1, tt.to1) - bitmap2 := FromActiveRange(tt.from2, tt.to2) - result := bitmap1.IntersectsWith(bitmap2) - if result != tt.intersect { - t.Errorf("Intersection of range (%d, %d) and (%d, %d): expected %v, got %v", - tt.from1, tt.to1, tt.from2, tt.to2, tt.intersect, result) - } - } -} diff --git a/civisibility/utils/git.go b/civisibility/utils/git.go index b0e5996..774331d 100644 --- a/civisibility/utils/git.go +++ b/civisibility/utils/git.go @@ -16,6 +16,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/DataDog/ddtest/civisibility/constants" @@ -24,12 +25,8 @@ import ( // MaxPackFileSizeInMb is the maximum size of a pack file in megabytes. const MaxPackFileSizeInMb = 3 -// localGitData holds various pieces of information about the local Git repository, -// including the source root, repository URL, branch, commit SHA, author and committer details, and commit message. -type localGitData struct { - SourceRoot string - RepositoryURL string - Branch string +// localCommitData holds information about a single commit in the local Git repository. +type localCommitData struct { CommitSha string AuthorDate time.Time AuthorName string @@ -40,21 +37,55 @@ type localGitData struct { CommitMessage string } +// localGitData holds various pieces of information about the local Git repository, +// including the source root, repository URL, branch, commit SHA, author and committer details, and commit message. +type localGitData struct { + localCommitData + SourceRoot string + RepositoryURL string + Branch string +} + +// gitVersionData holds the major, minor, and patch version numbers of the Git executable. +type gitVersionData struct { + major int + minor int + patch int + err error +} + var ( + // gitCommandMutex is a mutex used to synchronize access to Git commands to prevent lock errors in git + gitCommandMutex sync.Mutex + // regexpSensitiveInfo is a regular expression used to match and filter out sensitive information from URLs. regexpSensitiveInfo = regexp.MustCompile("(https?://|ssh?://)[^/]*@") + // Constants for base branch detection algorithm + possibleBaseBranches = []string{"main", "master", "preprod", "prod", "dev", "development", "trunk"} + + // BASE_LIKE_BRANCH_FILTER - regex to check if the branch name is similar to a possible base branch + baseLikeBranchFilter = regexp.MustCompile(`^(main|master|preprod|prod|dev|development|trunk|release\/.*|hotfix\/.*)$`) + + // Cached data + // isGitFoundValue is a boolean flag indicating whether the Git executable is available on the system. isGitFoundValue bool // gitFinder is a sync.Once instance used to ensure that the Git executable is only checked once. - gitFinder sync.Once + gitFinderOnce sync.Once - // Constants for base branch detection algorithm - possibleBaseBranches = []string{"main", "master", "preprod", "prod", "dev", "development", "trunk"} + // gitVersion is a sync.Once instance used to ensure that the Git version is only retrieved once. + gitVersionOnce sync.Once - // BASE_LIKE_BRANCH_FILTER - regex to check if the branch name is similar to a possible base branch - baseLikeBranchFilter = regexp.MustCompile(`^(main|master|preprod|prod|dev|development|trunk|release\/.*|hotfix\/.*)$`) + // gitVersionValue holds the version of the Git executable installed on the system. + gitVersionValue gitVersionData + + // isAShallowCloneRepositoryOnce is a sync.Once instance used to ensure that the check for a shallow clone repository is only performed once. + isAShallowCloneRepositoryOnce atomic.Pointer[sync.Once] + + // isAShallowCloneRepositoryValue is a boolean flag indicating whether the repository is a shallow clone. + isAShallowCloneRepositoryValue bool ) // branchMetrics holds metrics for evaluating base branch candidates @@ -66,7 +97,7 @@ type branchMetrics struct { // isGitFound checks if the Git executable is available on the system. func isGitFound() bool { - gitFinder.Do(func() { + gitFinderOnce.Do(func() { _, err := exec.LookPath("git") isGitFoundValue = err == nil if err != nil { @@ -81,6 +112,8 @@ func execGit(args ...string) (val []byte, err error) { if !isGitFound() { return nil, errors.New("git executable not found") } + gitCommandMutex.Lock() + defer gitCommandMutex.Unlock() return exec.Command("git", args...).CombinedOutput() } @@ -102,19 +135,30 @@ func execGitStringWithInput(input string, args ...string) (val string, err error // getGitVersion retrieves the version of the Git executable installed on the system. func getGitVersion() (major int, minor int, patch int, err error) { - out, lerr := execGitString("--version") - if lerr != nil { - return 0, 0, 0, lerr - } - out = strings.TrimSpace(strings.ReplaceAll(out, "git version ", "")) - versionParts := strings.Split(out, ".") - if len(versionParts) < 3 { - return 0, 0, 0, errors.New("invalid git version") - } - major, _ = strconv.Atoi(versionParts[0]) - minor, _ = strconv.Atoi(versionParts[1]) - patch, _ = strconv.Atoi(versionParts[2]) - return major, minor, patch, nil + gitVersionOnce.Do(func() { + out, lerr := execGitString("--version") + if lerr != nil { + gitVersionValue = gitVersionData{err: lerr} + return + } + out = strings.TrimSpace(strings.ReplaceAll(out, "git version ", "")) + versionParts := strings.Split(out, ".") + if len(versionParts) < 3 { + gitVersionValue = gitVersionData{err: errors.New("invalid git version")} + return + } + major, _ = strconv.Atoi(versionParts[0]) + minor, _ = strconv.Atoi(versionParts[1]) + patch, _ = strconv.Atoi(versionParts[2]) + gitVersionValue = gitVersionData{ + major: major, + minor: minor, + patch: patch, + err: nil, + } + }) + + return gitVersionValue.major, gitVersionValue.minor, gitVersionValue.patch, gitVersionValue.err } // getLocalGitData retrieves information about the local Git repository from the current HEAD. @@ -138,6 +182,14 @@ func getLocalGitData() (localGitData, error) { if out, err := execGitString("config", "--global", "--add", "safe.directory", gitDir); err != nil { slog.Debug("civisibility.git: error while setting permissions to git folder", "gitDir", gitDir, "out", out, "error", err.Error()) } + // if the git folder contains with a `/.git` then we also add permission to the parent. + if strings.HasSuffix(gitDir, "/.git") { + parentGitDir := strings.TrimSuffix(gitDir, "/.git") + slog.Debug("civisibility.git: setting permissions to git folder", "parentGitDir", parentGitDir) + if out, err := execGitString("config", "--global", "--add", "safe.directory", parentGitDir); err != nil { + slog.Debug("civisibility.git: error while setting permissions to git folder", "parentGitDir", parentGitDir, "out", out, "error", err.Error()) + } + } } else { slog.Debug("civisibility.git: error getting the parent git folder.") } @@ -195,6 +247,82 @@ func getLocalGitData() (localGitData, error) { return gitData, nil } +// fetchCommitData retrieves commit data for a specific commit SHA in a shallow clone Git repository. +func fetchCommitData(commitSha string) (localCommitData, error) { + commitData := localCommitData{} + + // let's do a first check to see if the repository is a shallow clone + slog.Debug("civisibility.fetchCommitData: checking if the repository is a shallow clone") + isAShallowClone, err := isAShallowCloneRepository() + if err != nil { + return commitData, fmt.Errorf("civisibility.fetchCommitData: error checking if the repository is a shallow clone: %s", err) + } + + // if the git repo is a shallow clone, we try to fecth the commit sha data + if isAShallowClone { + // let's check the git version >= 2.27.0 (git --version) to see if we can unshallow the repository + slog.Debug("civisibility.fetchCommitData: checking the git version") + major, minor, patch, err := getGitVersion() + if err != nil { + return commitData, fmt.Errorf("civisibility.fetchCommitData: error getting the git version: %s", err) + } + slog.Debug("civisibility.fetchCommitData: git version", "major", major, "minor", minor, "patch", patch) + if major < 2 || (major == 2 && minor < 27) { + slog.Debug("civisibility.fetchCommitData: the git version is less than 2.27.0 we cannot unshallow the repository") + return commitData, nil + } + + // let's get the remote name + remoteName, err := getRemoteName() + if err != nil { + return commitData, fmt.Errorf("civisibility.fetchCommitData: error getting the remote name: %s\n%s", err, remoteName) + } + if remoteName == "" { + // if the origin name is empty, we fallback to "origin" + remoteName = "origin" + } + slog.Debug("civisibility.fetchCommitData: remote name", "remoteName", remoteName) + + // let's fetch the missing commits and trees from a commit sha + // git fetch --update-shallow --filter="blob:none" --recurse-submodules=no --no-write-fetch-head + slog.Debug("civisibility.fetchCommitData: fetching the missing commits and trees from the last month") + if fetchOutput, fetchErr := execGitString( + "fetch", "--update-shallow", "--filter=blob:none", "--recurse-submodules=no", "--no-write-fetch-head", remoteName, commitSha); fetchErr != nil { + return commitData, fmt.Errorf("civisibility.fetchCommitData: error: %s\n%s", fetchErr, fetchOutput) + } + } + + // Get commit details from the latest commit using git log (git show -s --format='%H","%aI","%an","%ae","%cI","%cn","%ce","%B') + slog.Debug("civisibility.git: getting the latest commit details") + out, err := execGitString("show", commitSha, "-s", "--format=%H\",\"%at\",\"%an\",\"%ae\",\"%ct\",\"%cn\",\"%ce\",\"%B") + if err != nil { + return commitData, err + } + + // Split the output into individual components + outArray := strings.Split(out, "\",\"") + if len(outArray) < 8 { + return commitData, errors.New("git log failed") + } + + // Parse author and committer dates from Unix timestamp + authorUnixDate, _ := strconv.ParseInt(outArray[1], 10, 64) + committerUnixDate, _ := strconv.ParseInt(outArray[4], 10, 64) + + // Populate the localGitData struct with the parsed information + commitData.CommitSha = outArray[0] + commitData.AuthorDate = time.Unix(authorUnixDate, 0) + commitData.AuthorName = outArray[2] + commitData.AuthorEmail = outArray[3] + commitData.CommitterDate = time.Unix(committerUnixDate, 0) + commitData.CommitterName = outArray[5] + commitData.CommitterEmail = outArray[6] + commitData.CommitMessage = strings.Trim(outArray[7], "\n") + + slog.Debug("civisibility.fetchCommitData: was completed successfully") + return commitData, nil +} + // GetLastLocalGitCommitShas retrieves the commit SHAs of the last 1000 commits in the local Git repository. func GetLastLocalGitCommitShas() []string { // git log --format=%H -n 1000 --since=\"1 month ago\" @@ -213,7 +341,7 @@ func UnshallowGitRepository() (bool, error) { slog.Debug("civisibility.unshallow: checking if the repository is a shallow clone") isAShallowClone, err := isAShallowCloneRepository() if err != nil { - return false, fmt.Errorf("civisibility.unshallow: error checking if the repository is a shallow clone: %s", err.Error()) + return false, fmt.Errorf("civisibility.unshallow: error checking if the repository is a shallow clone: %s", err) } // if the git repo is not a shallow clone, we can return early @@ -226,7 +354,7 @@ func UnshallowGitRepository() (bool, error) { slog.Debug("civisibility.unshallow: the repository is a shallow clone, checking if there are more than one commit in the logs") hasMoreThanOneCommits, err := hasTheGitLogHaveMoreThanOneCommits() if err != nil { - return false, fmt.Errorf("civisibility.unshallow: error checking if the git log has more than one commit: %s", err.Error()) + return false, fmt.Errorf("civisibility.unshallow: error checking if the git log has more than one commit: %s", err) } // if there are more than 1 commits, we can return early @@ -239,7 +367,7 @@ func UnshallowGitRepository() (bool, error) { slog.Debug("civisibility.unshallow: checking the git version") major, minor, patch, err := getGitVersion() if err != nil { - return false, fmt.Errorf("civisibility.unshallow: error getting the git version: %s", err.Error()) + return false, fmt.Errorf("civisibility.unshallow: error getting the git version: %s", err) } slog.Debug("civisibility.unshallow: git version", "major", major, "minor", minor, "patch", patch) if major < 2 || (major == 2 && minor < 27) { @@ -250,27 +378,27 @@ func UnshallowGitRepository() (bool, error) { // after asking for 2 logs lines, if the git log command returns just one commit sha, we reconfigure the repo // to ask for git commits and trees of the last month (no blobs) - // let's get the origin name (git config --default origin --get clone.defaultRemoteName) - originName, err := execGitString("config", "--default", "origin", "--get", "clone.defaultRemoteName") + // let's get the remote name + remoteName, err := getRemoteName() if err != nil { - return false, fmt.Errorf("civisibility.unshallow: error getting the origin name: %s\n%s", err.Error(), originName) + return false, fmt.Errorf("civisibility.unshallow: error getting the remote name: %s\n%s", err, remoteName) } - if originName == "" { + if remoteName == "" { // if the origin name is empty, we fallback to "origin" - originName = "origin" + remoteName = "origin" } - slog.Debug("civisibility.unshallow: origin name", "originName", originName) + slog.Debug("civisibility.unshallow: remote name", "remoteName", remoteName) // let's get the sha of the HEAD (git rev-parse HEAD) headSha, err := execGitString("rev-parse", "HEAD") if err != nil { - return false, fmt.Errorf("civisibility.unshallow: error getting the HEAD sha: %s\n%s", err.Error(), headSha) + return false, fmt.Errorf("civisibility.unshallow: error getting the HEAD sha: %s\n%s", err, headSha) } if headSha == "" { // if the HEAD is empty, we fallback to the current branch (git branch --show-current) headSha, err = execGitString("branch", "--show-current") if err != nil { - return false, fmt.Errorf("civisibility.unshallow: error getting the current branch: %s\n%s", err.Error(), headSha) + return false, fmt.Errorf("civisibility.unshallow: error getting the current branch: %s\n%s", err, headSha) } } slog.Debug("civisibility.unshallow: HEAD sha", "headSha", headSha) @@ -278,7 +406,7 @@ func UnshallowGitRepository() (bool, error) { // let's fetch the missing commits and trees from the last month // git fetch --shallow-since="1 month ago" --update-shallow --filter="blob:none" --recurse-submodules=no $(git config --default origin --get clone.defaultRemoteName) $(git rev-parse HEAD) slog.Debug("civisibility.unshallow: fetching the missing commits and trees from the last month") - fetchOutput, err := execGitString("fetch", "--shallow-since=\"1 month ago\"", "--update-shallow", "--filter=blob:none", "--recurse-submodules=no", originName, headSha) + fetchOutput, err := execGitString("fetch", "--shallow-since=\"1 month ago\"", "--update-shallow", "--filter=blob:none", "--recurse-submodules=no", remoteName, headSha) // let's check if the last command was unsuccessful if err != nil || fetchOutput == "" { @@ -296,7 +424,7 @@ func UnshallowGitRepository() (bool, error) { if err == nil { // let's try the alternative: git fetch --shallow-since="1 month ago" --update-shallow --filter="blob:none" --recurse-submodules=no $(git config --default origin --get clone.defaultRemoteName) $(git rev-parse --abbrev-ref --symbolic-full-name @{upstream}) slog.Debug("civisibility.unshallow: fetching the missing commits and trees from the last month using the remote branch name") - fetchOutput, err = execGitString("fetch", "--shallow-since=\"1 month ago\"", "--update-shallow", "--filter=blob:none", "--recurse-submodules=no", originName, remoteBranchName) + fetchOutput, err = execGitString("fetch", "--shallow-since=\"1 month ago\"", "--update-shallow", "--filter=blob:none", "--recurse-submodules=no", remoteName, remoteBranchName) } } @@ -311,14 +439,16 @@ func UnshallowGitRepository() (bool, error) { // let's try the last fallback: git fetch --shallow-since="1 month ago" --update-shallow --filter="blob:none" --recurse-submodules=no $(git config --default origin --get clone.defaultRemoteName) slog.Debug("civisibility.unshallow: fetching the missing commits and trees from the last month using the origin name") - fetchOutput, err = execGitString("fetch", "--shallow-since=\"1 month ago\"", "--update-shallow", "--filter=blob:none", "--recurse-submodules=no", originName) + fetchOutput, err = execGitString("fetch", "--shallow-since=\"1 month ago\"", "--update-shallow", "--filter=blob:none", "--recurse-submodules=no", remoteName) } if err != nil { - return false, fmt.Errorf("civisibility.unshallow: error: %s\n%s", err.Error(), fetchOutput) + return false, fmt.Errorf("civisibility.unshallow: error: %s\n%s", err, fetchOutput) } slog.Debug("civisibility.unshallow: was completed successfully") + tmpso := sync.Once{} + isAShallowCloneRepositoryOnce.Store(&tmpso) return true, nil } @@ -341,7 +471,7 @@ func GetGitDiff(baseCommit, headCommit string) (string, error) { // let's ensure we have all the branch names from the remote fetchOut, err := execGitString("fetch", remoteOut, baseCommit, "--depth=1") if err != nil { - slog.Debug("civisibility.git: error fetching", "remote", remoteOut, "baseCommit", baseCommit, "fetchOut", fetchOut, "error", err.Error()) + slog.Debug("civisibility.git: error fetching", "remoteOut", remoteOut, "baseCommit", baseCommit, "fetchOut", fetchOut, "error", err.Error()) } // then let's get the remote branch name @@ -349,10 +479,10 @@ func GetGitDiff(baseCommit, headCommit string) (string, error) { } } - slog.Debug("civisibility.git: getting the diff between commits", "baseCommit", baseCommit, "headCommit", headCommit) + slog.Debug("civisibility.git: getting the diff between", "baseCommit", baseCommit, "headCommit", headCommit) out, err := execGitString("diff", "-U0", "--word-diff=porcelain", baseCommit, headCommit) if err != nil { - return "", fmt.Errorf("civisibility.git: error getting the diff from %s to %s: %s | %s", baseCommit, headCommit, err.Error(), out) + return "", fmt.Errorf("civisibility.git: error getting the diff from %s to %s: %s | %s", baseCommit, headCommit, err, out) } if out == "" { return "", fmt.Errorf("civisibility.git: error getting the diff from %s to %s: empty output", baseCommit, headCommit) @@ -376,13 +506,26 @@ func filterSensitiveInfo(url string) string { // isAShallowCloneRepository checks if the local Git repository is a shallow clone. func isAShallowCloneRepository() (bool, error) { - // git rev-parse --is-shallow-repository - out, err := execGitString("rev-parse", "--is-shallow-repository") - if err != nil { - return false, err - } + var fErr error + var sOnce *sync.Once + sOnce = isAShallowCloneRepositoryOnce.Load() + if sOnce == nil { + sOnce = &sync.Once{} + isAShallowCloneRepositoryOnce.Store(sOnce) + } + sOnce.Do(func() { + // git rev-parse --is-shallow-repository + out, err := execGitString("rev-parse", "--is-shallow-repository") + if err != nil { + isAShallowCloneRepositoryValue = false + fErr = err + return + } - return strings.TrimSpace(out) == "true", nil + isAShallowCloneRepositoryValue = strings.TrimSpace(out) == "true" + }) + + return isAShallowCloneRepositoryValue, fErr } // hasTheGitLogHaveMoreThanOneCommits checks if the local Git repository has more than one commit. @@ -412,6 +555,7 @@ func getObjectsSha(commitsToInclude []string, commitsToExclude []string) []strin return strings.Split(out, "\n") } +// CreatePackFiles creates pack files from the given commits to include and exclude. func CreatePackFiles(commitsToInclude []string, commitsToExclude []string) []string { // get the objects shas to send objectsShas := getObjectsSha(commitsToInclude, commitsToExclude) @@ -426,18 +570,38 @@ func CreatePackFiles(commitsToInclude []string, commitsToExclude []string) []str objectsShasString += objectSha + "\n" } - // get a temporary path to store the pack files - temporaryPath, err := os.MkdirTemp("", "pack-objects") - if err != nil { - slog.Warn("civisibility: error creating temporary directory", "error", err.Error()) - return nil + workingDirectory := func() string { + wd, err := os.Getwd() + if err != nil { + return "." + } + return wd + } + + var temporaryPath string + var out string + var err error + + // Git can throw a cross device error if the temporal folder is in a different drive than the .git folder (eg. symbolic link) + // to handle this edge case, we first try with a temp folder and if we fail then we try in the working directory folder. + for _, folder := range []string{"", workingDirectory()} { + // get a temporary path to store the pack files + temporaryPath, err = os.MkdirTemp(folder, ".dd-pack-objects") + if err != nil { + slog.Warn("civisibility: error creating temporary directory", "folder", folder, "error", err.Error()) + continue + } + + // git pack-objects --compression=9 --max-pack-size={MaxPackFileSizeInMb}m "{temporaryPath}" + out, err = execGitStringWithInput(objectsShasString, + "pack-objects", "--compression=9", "--max-pack-size="+strconv.Itoa(MaxPackFileSizeInMb)+"m", temporaryPath+"/") + if err == nil { + break + } } - // git pack-objects --compression=9 --max-pack-size={MaxPackFileSizeInMb}m "{temporaryPath}" - out, err := execGitStringWithInput(objectsShasString, - "pack-objects", "--compression=9", "--max-pack-size="+strconv.Itoa(MaxPackFileSizeInMb)+"m", temporaryPath+"/") if err != nil { - slog.Warn("civisibility: error creating pack files", "error", err.Error()) + slog.Warn("civisibility: error creating pack files in", "temporaryPath", temporaryPath, "error", err.Error(), "output", out) return nil } @@ -448,7 +612,7 @@ func CreatePackFiles(commitsToInclude []string, commitsToExclude []string) []str // check if the pack file exists if _, err := os.Stat(file); os.IsNotExist(err) { - slog.Warn("civisibility: pack file not found", "packFile", packFiles[i]) + slog.Warn("civisibility: pack file not found", "file", packFiles[i]) continue } @@ -531,7 +695,7 @@ func findFallbackDefaultBranch(remoteName string) string { return "" } -// GetBaseBranchSha detects the base branch SHA using the algorithm from algorithm.md +// GetBaseBranchSha detects the base branch SHA using the algorithm func GetBaseBranchSha(defaultBranch string) (string, error) { if !isGitFound() { return "", errors.New("git executable not found") diff --git a/civisibility/utils/net/client.go b/civisibility/utils/net/client.go index ae162a0..d4f5a29 100644 --- a/civisibility/utils/net/client.go +++ b/civisibility/utils/net/client.go @@ -55,6 +55,8 @@ type ( repositoryURL string commitSha string commitMessage string + headCommitSha string + headCommitMessage string branchName string testConfigurations testConfigurations headers map[string]string @@ -214,16 +216,18 @@ func NewClientWithServiceNameAndSubdomain(serviceName, subdomain string) Client } return &client{ - id: id, - agentless: agentlessEnabled, - baseURL: baseURL, - environment: environment, - serviceName: serviceName, - workingDirectory: ciTags[constants.CIWorkspacePath], - repositoryURL: ciTags[constants.GitRepositoryURL], - commitSha: ciTags[constants.GitCommitSHA], - commitMessage: ciTags[constants.GitCommitMessage], - branchName: bName, + id: id, + agentless: agentlessEnabled, + baseURL: baseURL, + environment: environment, + serviceName: serviceName, + workingDirectory: ciTags[constants.CIWorkspacePath], + repositoryURL: ciTags[constants.GitRepositoryURL], + commitSha: ciTags[constants.GitCommitSHA], + commitMessage: ciTags[constants.GitCommitMessage], + headCommitSha: ciTags[constants.GitHeadCommit], + headCommitMessage: ciTags[constants.GitHeadMessage], + branchName: bName, testConfigurations: testConfigurations{ OsPlatform: ciTags[constants.OSPlatform], OsVersion: ciTags[constants.OSVersion], diff --git a/civisibility/utils/net/http.go b/civisibility/utils/net/http.go index 34d1264..d5fee04 100644 --- a/civisibility/utils/net/http.go +++ b/civisibility/utils/net/http.go @@ -100,26 +100,31 @@ type RequestHandler struct { // This also permits orchestrion to disable tracing on this client. // See https://golang.org/pkg/net/http/#DefaultTransport . // Except we use a higher timeout for this -var defaultHTTPClient = http.Client{ - Timeout: 45 * time.Second, - Transport: &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - }).DialContext, - ForceAttemptHTTP2: true, - MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - }, +var defaultHTTPClient = createNewHTTPClient() + +// createNewHTTPClient creates a new HTTP client with custom transport settings. +func createNewHTTPClient() *http.Client { + return &http.Client{ + Timeout: 45 * time.Second, + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + }, + } } // NewRequestHandler creates a new RequestHandler with a default HTTP client. func NewRequestHandler() *RequestHandler { return &RequestHandler{ - Client: &defaultHTTPClient, + Client: defaultHTTPClient, } } @@ -170,8 +175,7 @@ func (rh *RequestHandler) internalSendRequest(config *RequestConfig, attempt int for _, f := range config.Files { fileNames = append(fileNames, f.FieldName) } - slog.Debug("ciVisibilityHttpClient: new request with files", "method", config.Method, "url", config.URL, "attempt", attempt, "maxRetries", config.MaxRetries, "files", fileNames) - + slog.Debug("ciVisibilityHttpClient: new request with files", "method", config.Method, "url", config.URL, "attempt", attempt, "maxRetries", config.MaxRetries, "fileNames", fileNames) req, err = http.NewRequest(config.Method, config.URL, bytes.NewBuffer(body)) if err != nil { return true, nil, err @@ -204,7 +208,7 @@ func (rh *RequestHandler) internalSendRequest(config *RequestConfig, attempt int strBody = strBody[:4096] + "..." // Truncate for logging } } - slog.Debug("ciVisibilityHttpClient: new request with body", "method", config.Method, "url", config.URL, "attempt", attempt, "maxRetries", config.MaxRetries, "compressed", config.Compressed, "format", config.Format, "bodySize", len(serializedBody), "body", strBody) + slog.Debug("ciVisibilityHttpClient: new request with body", "method", config.Method, "url", config.URL, "attempt", attempt, "maxRetries", config.MaxRetries, "compressed", config.Compressed, "format", config.Format, "bytes", len(serializedBody), "body", strBody) req, err = http.NewRequest(config.Method, config.URL, bytes.NewBuffer(serializedBody)) if err != nil { @@ -246,9 +250,7 @@ func (rh *RequestHandler) internalSendRequest(config *RequestConfig, attempt int return false, nil, nil } // Close response body - defer func() { - _ = resp.Body.Close() - }() + defer resp.Body.Close() // Capture the status code statusCode := resp.StatusCode @@ -312,7 +314,7 @@ func (rh *RequestHandler) internalSendRequest(config *RequestConfig, attempt int } } - slog.Debug("ciVisibilityHttpClient: response received", "method", config.Method, "url", config.URL, "statusCode", resp.StatusCode, "format", responseFormat, "bodySize", len(responseBody)) + slog.Debug("ciVisibilityHttpClient: response received", "method", config.Method, "url", config.URL, "statusCode", resp.StatusCode, "format", responseFormat, "bytes", len(responseBody)) // Determine if we can unmarshal based on status code (2xx) canUnmarshal := statusCode >= 200 && statusCode < 300 @@ -359,9 +361,7 @@ func compressData(data []byte) ([]byte, error) { if err != nil { return nil, err } - if err := writer.Close(); err != nil { - return nil, err - } + writer.Close() return buf.Bytes(), nil } @@ -369,36 +369,38 @@ func compressData(data []byte) ([]byte, error) { func decompressData(data []byte) ([]byte, error) { reader, err := gzip.NewReader(bytes.NewReader(data)) if err != nil { - return nil, fmt.Errorf("failed to create gzip reader: %s", err.Error()) + return nil, fmt.Errorf("failed to create gzip reader: %s", err) } - defer func() { - _ = reader.Close() - }() + defer reader.Close() decompressedData, err := io.ReadAll(reader) if err != nil { - return nil, fmt.Errorf("failed to decompress data: %s", err.Error()) + return nil, fmt.Errorf("failed to decompress data: %s", err) } return decompressedData, nil } // exponentialBackoff performs an exponential backoff with retries. func exponentialBackoff(retryCount int, initialDelay time.Duration) { + time.Sleep(getExponentialBackoffDuration(retryCount, initialDelay)) +} + +// getExponentialBackoffDuration calculates the backoff duration based on the retry count and initial delay. +func getExponentialBackoffDuration(retryCount int, initialDelay time.Duration) time.Duration { maxDelay := 10 * time.Second delay := initialDelay * (1 << uint(retryCount)) // Exponential backoff if delay > maxDelay { delay = maxDelay } - time.Sleep(delay) + return delay } // prepareContent prepares the content for a FormFile by serializing it if needed. func prepareContent(content interface{}, contentType string) ([]byte, error) { - switch contentType { - case ContentTypeJSON: + if contentType == ContentTypeJSON { return serializeData(content, FormatJSON) - case ContentTypeMessagePack: + } else if contentType == ContentTypeMessagePack { return serializeData(content, FormatMessagePack) - case ContentTypeOctetStream: + } else if contentType == ContentTypeOctetStream { // For binary data, ensure it's already in byte format if data, ok := content.([]byte); ok { return data, nil diff --git a/civisibility/utils/net/known_tests_api.go b/civisibility/utils/net/known_tests_api.go index 79bcf37..7818fcd 100644 --- a/civisibility/utils/net/known_tests_api.go +++ b/civisibility/utils/net/known_tests_api.go @@ -71,13 +71,13 @@ func (c *client) GetKnownTests() (*KnownTestsResponseData, error) { response, err := c.handler.SendRequest(*request) if err != nil { - return nil, fmt.Errorf("sending known tests request: %s", err.Error()) + return nil, fmt.Errorf("sending known tests request: %s", err) } var responseObject knownTestsResponse err = response.Unmarshal(&responseObject) if err != nil { - return nil, fmt.Errorf("unmarshalling known tests response: %s", err.Error()) + return nil, fmt.Errorf("unmarshalling known tests response: %s", err) } return &responseObject.Data.Attributes, nil diff --git a/civisibility/utils/net/logs_api.go b/civisibility/utils/net/logs_api.go index 632e5cb..f2aa054 100644 --- a/civisibility/utils/net/logs_api.go +++ b/civisibility/utils/net/logs_api.go @@ -38,7 +38,7 @@ func (c *client) SendLogs(logsPayload io.Reader) error { response, responseErr := c.handler.SendRequest(request) if responseErr != nil { - return fmt.Errorf("failed to send logs: %s", responseErr.Error()) + return fmt.Errorf("failed to send logs: %s", responseErr) } if response.StatusCode < 200 || response.StatusCode >= 300 { diff --git a/civisibility/utils/net/searchcommits_api.go b/civisibility/utils/net/searchcommits_api.go index 46f9616..aae8c5c 100644 --- a/civisibility/utils/net/searchcommits_api.go +++ b/civisibility/utils/net/searchcommits_api.go @@ -48,7 +48,6 @@ func (c *client) GetCommits(localCommits []string) ([]string, error) { } request := c.getPostRequestConfig(searchCommitsURLPath, body) - response, err := c.handler.SendRequest(*request) if err != nil { return nil, fmt.Errorf("sending search commits request: %s", err.Error()) diff --git a/civisibility/utils/net/sendpackfiles_api.go b/civisibility/utils/net/sendpackfiles_api.go index 597bb36..2381cd8 100644 --- a/civisibility/utils/net/sendpackfiles_api.go +++ b/civisibility/utils/net/sendpackfiles_api.go @@ -59,7 +59,7 @@ func (c *client) SendPackFiles(commitSha string, packFiles []string) (bytes int6 for _, file := range packFiles { fileContent, fileErr := os.ReadFile(file) if fileErr != nil { - err = fmt.Errorf("failed to read pack file: %s", fileErr.Error()) + err = fmt.Errorf("failed to read pack file: %s", fileErr) return } @@ -82,7 +82,7 @@ func (c *client) SendPackFiles(commitSha string, packFiles []string) (bytes int6 response, responseErr := c.handler.SendRequest(request) if responseErr != nil { - err = fmt.Errorf("failed to send packfile request: %s", responseErr.Error()) + err = fmt.Errorf("failed to send packfile request: %s", responseErr) return } diff --git a/civisibility/utils/net/settings_api.go b/civisibility/utils/net/settings_api.go index 4b606bf..27c1b35 100644 --- a/civisibility/utils/net/settings_api.go +++ b/civisibility/utils/net/settings_api.go @@ -65,6 +65,7 @@ type ( Enabled bool `json:"enabled"` AttemptToFixRetries int `json:"attempt_to_fix_retries"` } `json:"test_management"` + SubtestFeaturesEnabled bool `json:"-"` } ) @@ -92,15 +93,15 @@ func (c *client) GetSettings() (*SettingsResponseData, error) { response, err := c.handler.SendRequest(*request) if err != nil { - return nil, fmt.Errorf("sending get settings request: %s", err.Error()) + return nil, fmt.Errorf("sending get settings request: %s", err) } - slog.Debug("civisibility.settings", "response", string(response.Body)) + slog.Debug("civisibility.settings", "responseBody", string(response.Body)) var responseObject settingsResponse err = response.Unmarshal(&responseObject) if err != nil { - return nil, fmt.Errorf("unmarshalling settings response: %s", err.Error()) + return nil, fmt.Errorf("unmarshalling settings response: %s", err) } return &responseObject.Data.Attributes, nil diff --git a/civisibility/utils/net/skippable.go b/civisibility/utils/net/skippable.go index 2b76a1c..bf42eaa 100644 --- a/civisibility/utils/net/skippable.go +++ b/civisibility/utils/net/skippable.go @@ -80,13 +80,13 @@ func (c *client) GetSkippableTests() (correlationID string, skippables map[strin response, err := c.handler.SendRequest(*request) if err != nil { - return "", nil, fmt.Errorf("sending skippable tests request: %s", err.Error()) + return "", nil, fmt.Errorf("sending skippable tests request: %s", err) } var responseObject skippableResponse err = response.Unmarshal(&responseObject) if err != nil { - return "", nil, fmt.Errorf("unmarshalling skippable tests response: %s", err.Error()) + return "", nil, fmt.Errorf("unmarshalling skippable tests response: %s", err) } skippableTestsMap := map[string]map[string][]SkippableResponseDataAttributes{} diff --git a/civisibility/utils/net/test_management_tests_api.go b/civisibility/utils/net/test_management_tests_api.go index 82d96dd..9918574 100644 --- a/civisibility/utils/net/test_management_tests_api.go +++ b/civisibility/utils/net/test_management_tests_api.go @@ -30,6 +30,7 @@ type ( CommitSha string `json:"sha"` Module string `json:"module,omitempty"` CommitMessage string `json:"commit_message"` + Branch string `json:"branch"` } testManagementTestsResponse struct { @@ -68,14 +69,27 @@ func (c *client) GetTestManagementTests() (*TestManagementTestsResponseDataModul return nil, fmt.Errorf("civisibility.GetTestManagementTests: repository URL is required") } + // we use the head commit SHA if it is set, otherwise we use the commit SHA + commitSha := c.commitSha + if c.headCommitSha != "" { + commitSha = c.headCommitSha + } + + // we use the head commit message if it is set, otherwise we use the commit message + commitMessage := c.commitMessage + if c.headCommitMessage != "" { + commitMessage = c.headCommitMessage + } + body := testManagementTestsRequest{ Data: testManagementTestsRequestHeader{ ID: c.id, Type: testManagementTestsRequestType, Attributes: testManagementTestsRequestData{ RepositoryURL: c.repositoryURL, - CommitSha: c.commitSha, - CommitMessage: c.commitMessage, + CommitSha: commitSha, + CommitMessage: commitMessage, + Branch: c.branchName, }, }, } @@ -84,26 +98,14 @@ func (c *client) GetTestManagementTests() (*TestManagementTestsResponseDataModul response, err := c.handler.SendRequest(*request) if err != nil { - return nil, fmt.Errorf("sending known tests request: %s", err.Error()) + return nil, fmt.Errorf("sending known tests request: %s", err) } var responseObject testManagementTestsResponse err = response.Unmarshal(&responseObject) if err != nil { - return nil, fmt.Errorf("unmarshalling test management tests response: %s", err.Error()) + return nil, fmt.Errorf("unmarshalling test management tests response: %s", err) } - if responseObject.Data.Attributes.Modules != nil { - for _, module := range responseObject.Data.Attributes.Modules { - if module.Suites == nil { - continue - } - for _, suite := range module.Suites { - if suite.Tests == nil { - continue - } - } - } - } return &responseObject.Data.Attributes, nil } From 3e8573f734894c757025cb6589fce0895aab989a Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 7 Jan 2026 14:13:01 +0100 Subject: [PATCH 2/2] lint issues --- civisibility/utils/net/http.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/civisibility/utils/net/http.go b/civisibility/utils/net/http.go index d5fee04..66d4887 100644 --- a/civisibility/utils/net/http.go +++ b/civisibility/utils/net/http.go @@ -250,7 +250,7 @@ func (rh *RequestHandler) internalSendRequest(config *RequestConfig, attempt int return false, nil, nil } // Close response body - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() // Capture the status code statusCode := resp.StatusCode @@ -361,7 +361,9 @@ func compressData(data []byte) ([]byte, error) { if err != nil { return nil, err } - writer.Close() + if err := writer.Close(); err != nil { + return nil, err + } return buf.Bytes(), nil } @@ -371,7 +373,7 @@ func decompressData(data []byte) ([]byte, error) { if err != nil { return nil, fmt.Errorf("failed to create gzip reader: %s", err) } - defer reader.Close() + defer func() { _ = reader.Close() }() decompressedData, err := io.ReadAll(reader) if err != nil { return nil, fmt.Errorf("failed to decompress data: %s", err) @@ -396,11 +398,12 @@ func getExponentialBackoffDuration(retryCount int, initialDelay time.Duration) t // prepareContent prepares the content for a FormFile by serializing it if needed. func prepareContent(content interface{}, contentType string) ([]byte, error) { - if contentType == ContentTypeJSON { + switch contentType { + case ContentTypeJSON: return serializeData(content, FormatJSON) - } else if contentType == ContentTypeMessagePack { + case ContentTypeMessagePack: return serializeData(content, FormatMessagePack) - } else if contentType == ContentTypeOctetStream { + case ContentTypeOctetStream: // For binary data, ensure it's already in byte format if data, ok := content.([]byte); ok { return data, nil @@ -409,8 +412,9 @@ func prepareContent(content interface{}, contentType string) ([]byte, error) { return io.ReadAll(reader) } return nil, errors.New("content must be []byte or an io.Reader for octet-stream content type") + default: + return nil, errors.New("unsupported content type for serialization") } - return nil, errors.New("unsupported content type for serialization") } // createMultipartFormData creates a multipart form data request body with the given files.