diff --git a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go index e74cf218fd0..6301252985e 100644 --- a/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go +++ b/vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go @@ -5,7 +5,6 @@ import ( "context" "encoding/json" "flag" - "fmt" "log/slog" "net/http" "os" @@ -32,7 +31,6 @@ var ( jsonDir = flag.String("nvd-json-dir", "", "Path to directory containing NVD CVE JSON files to examine.") parsedCPEDictionary = flag.String("cpe-repos", "", "Path to JSON mapping of CPEs to repos generated by cpe-repo-gen") outDir = flag.String("out-dir", "", "Path to output results.") - outFormat = flag.String("out-format", "OSV", "Format to output {OSV,PackageInfo}") workers = flag.Int("workers", 10, "The number of concurrent workers to use for processing CVEs.") gcsWorkers = flag.Int("gcs-workers", 30, "The number of concurrent workers to use for GCS uploads.") rejectFailed = flag.Bool("reject-failed", false, "If set, OSV records with a failed conversion outcome will not be generated.") @@ -51,10 +49,6 @@ var ( func main() { flag.Parse() - if !slices.Contains([]string{"OSV", "PackageInfo"}, *outFormat) { - fmt.Fprintf(os.Stderr, "Unsupported output format: %s\n", *outFormat) - os.Exit(1) - } if *outDir != "" { if err := os.MkdirAll(*outDir, 0755); err != nil { @@ -202,18 +196,7 @@ func processCVE(cve models.NVDCVE, vpRepoCache *c.VPRepoCache, repoTagsCache git } metrics.Repos = repos - var outcome models.ConversionOutcome - var vuln *vulns.Vulnerability - var finalMetrics *models.ConversionMetrics - switch *outFormat { - case "OSV": - vuln, finalMetrics, outcome = nvd.CVEToOSV(cve, repos, vpRepoCache, repoTagsCache, metrics) - case "PackageInfo": - outcome = nvd.CVEToPackageInfo(cve, repos, repoTagsCache, *outDir, metrics) - finalMetrics = metrics - } - - return vuln, finalMetrics, outcome + return nvd.CVEToOSV(cve, repos, vpRepoCache, repoTagsCache, metrics) } func worker(wg *sync.WaitGroup, jobs <-chan models.NVDCVE, gcsHelper *gcs.Helper, outDir string, vpRepoCache *c.VPRepoCache, repoTagsCache git.RepoTagsCache) { diff --git a/vulnfeeds/cmd/pypi/main.go b/vulnfeeds/cmd/pypi/main.go index 37b4a5ecc4b..40a72793765 100644 --- a/vulnfeeds/cmd/pypi/main.go +++ b/vulnfeeds/cmd/pypi/main.go @@ -21,7 +21,6 @@ import ( "fmt" "io/fs" "log/slog" - "net/http" "os" "path/filepath" "strings" @@ -220,7 +219,7 @@ func main() { func generatePyPIAffected(cve models.NVDCVE, pkg string, validVersions []string, purl string, metrics *models.ConversionMetrics) *vulns.Vulnerability { id := "PYSEC-0000-" + cve.ID - versions := conversion.ExtractVersionInfo(cve, validVersions, http.DefaultClient, metrics, nil) + versions := conversion.ExtractVersionInfo(cve, validVersions, metrics) pkgInfo := vulns.PackageInfo{ PkgName: pkg, diff --git a/vulnfeeds/conversion/nvd/converter.go b/vulnfeeds/conversion/nvd/converter.go index dc4c2e60bb7..3b23ffc3c26 100644 --- a/vulnfeeds/conversion/nvd/converter.go +++ b/vulnfeeds/conversion/nvd/converter.go @@ -3,18 +3,13 @@ package nvd import ( "cmp" - "encoding/json" "errors" - "log/slog" "maps" "net/http" - "os" - "path/filepath" "slices" "strings" c "github.com/google/osv/vulnfeeds/conversion" - "github.com/google/osv/vulnfeeds/conversion/writer" "github.com/google/osv/vulnfeeds/git" "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/utility" @@ -166,117 +161,6 @@ func CVEToOSV(cve models.NVDCVE, repos []string, vpRepoCache *c.VPRepoCache, cac return v, metrics, metrics.Outcome } -// CVEToPackageInfo takes an NVD CVE record and outputs a PackageInfo struct in a file in the specified directory. -func CVEToPackageInfo(cve models.NVDCVE, repos []string, cache git.RepoTagsCache, directory string, metrics *models.ConversionMetrics) models.ConversionOutcome { - CPEs := c.CPEs(cve) - // The vendor name and product name are used to construct the output `vulnDir` below, so need to be set to *something* to keep the output tidy. - maybeVendorName := "ENOCPE" - maybeProductName := "ENOCPE" - - if len(CPEs) > 0 { - CPE, err := c.ParseCPE(CPEs[0]) // For naming the subdirectory used for output. - maybeVendorName = CPE.Vendor - maybeProductName = CPE.Product - if err != nil { - return models.NoRanges - } - } - - // more often than not, this yields a VersionInfo with AffectedVersions and no AffectedCommits. - versions := c.ExtractVersionInfo(cve, nil, http.DefaultClient, metrics, cache) - if metrics.Outcome == models.Error { - return models.Error - } - - if len(versions.AffectedVersions) != 0 { - // There are some AffectedVersions to try and resolve to AffectedCommits. - if len(repos) == 0 { - metrics.AddNote("No affected ranges for %q, and no repos to try and convert %+v to tags with", maybeProductName, versions.AffectedVersions) - return models.NoRepos - } - logger.Info("Trying to convert version tags to commits", slog.String("cve", string(cve.ID)), slog.Any("versions", versions), slog.Any("repos", repos)) - c.VersionInfoToCommits(&versions, repos, cache, metrics) - if metrics.Outcome == models.Error { - return models.Error - } - } - - hasAnyFixedCommits := false - for _, repo := range repos { - if versions.HasFixedCommits(repo) { - hasAnyFixedCommits = true - } - } - - if versions.HasFixedVersions() && !hasAnyFixedCommits { - metrics.AddNote("Failed to convert fixed version tags to commits: %+v", versions) - return models.NoCommitRanges - } - - hasAnyLastAffectedCommits := false - for _, repo := range repos { - if versions.HasLastAffectedCommits(repo) { - hasAnyLastAffectedCommits = true - } - } - - if versions.HasLastAffectedVersions() && !hasAnyLastAffectedCommits && !hasAnyFixedCommits { - metrics.AddNote("Failed to convert last_affected version tags to commits: %+v", versions) - return models.NoCommitRanges - } - - if len(versions.AffectedCommits) == 0 { - metrics.AddNote("No affected commit ranges determined for %q", maybeProductName) - return models.NoCommitRanges - } - - versions.AffectedVersions = nil // these have served their purpose and are not required in the resulting output. - - slices.SortStableFunc(versions.AffectedCommits, models.AffectedCommitCompare) - - if metrics.Outcome == models.Error { - return metrics.Outcome - } - - var pkgInfos []vulns.PackageInfo - pi := vulns.PackageInfo{VersionInfo: versions} - pkgInfos = append(pkgInfos, pi) // combine-to-osv expects a serialised *array* of PackageInfo - - vulnDir := filepath.Join(directory, maybeVendorName, maybeProductName) - err := os.MkdirAll(vulnDir, 0755) - if err != nil { - logger.Warn("Failed to create dir", slog.Any("err", err)) - } - - outputFile := filepath.Join(vulnDir, string(cve.ID)+".nvd"+models.Extension) - f, err := os.Create(outputFile) - if err != nil { - logger.Warn("Failed to open for writing", slog.String("path", outputFile), slog.Any("err", err)) - } - defer f.Close() - - encoder := json.NewEncoder(f) - encoder.SetIndent("", " ") - err = encoder.Encode(&pkgInfos) - - if err != nil { - logger.Warn("Failed to encode PackageInfo", slog.String("path", outputFile), slog.Any("err", err)) - } - - logger.Info("Generated PackageInfo record", slog.String("cve", string(cve.ID)), slog.String("product", maybeProductName)) - - metricsFile, err := writer.CreateMetricsFile(cve.ID, vulnDir) - if err != nil { - logger.Warn("Failed to create metrics file", slog.String("path", metricsFile.Name()), slog.Any("err", err)) - } - err = writer.WriteMetricsFile(metrics, metricsFile) - if err != nil { - logger.Warn("Failed to write metrics file", slog.String("path", metricsFile.Name()), slog.Any("err", err)) - } - - return metrics.Outcome -} - // FindRepos attempts to find the source code repositories for a given CVE. func FindRepos(cve models.NVDCVE, vpRepoCache *c.VPRepoCache, repoTagsCache git.RepoTagsCache, metrics *models.ConversionMetrics, httpClient *http.Client) []string { // Find repos diff --git a/vulnfeeds/conversion/versions.go b/vulnfeeds/conversion/versions.go index 350cb1c1223..c4387b3723d 100644 --- a/vulnfeeds/conversion/versions.go +++ b/vulnfeeds/conversion/versions.go @@ -870,17 +870,8 @@ func ExtractVersionsFromCPEs(cve models.NVDCVE, validVersions []string, vpRepoCa } // ExtractVersionInfo extracts version information from a CVE and saves to a VersionInfo struct. -// This is mostly deprecated, but is still used by the Alpine, Debian, and PyPi converters. -func ExtractVersionInfo(cve models.NVDCVE, validVersions []string, httpClient *http.Client, metrics *models.ConversionMetrics, cache git.RepoTagsCache) (v models.VersionInfo) { - if commit, err := ExtractCommitsFromRefs(cve.References, httpClient, cache); err == nil { - v.AffectedCommits = append(v.AffectedCommits, commit...) - } - - if v.AffectedCommits != nil { - v.AffectedCommits = DeduplicateAffectedCommits(v.AffectedCommits) - metrics.AddNote("Extracted %d commits", len(v.AffectedCommits)) - } - +// This is mostly deprecated, but is still used by the PyPi converter. +func ExtractVersionInfo(cve models.NVDCVE, validVersions []string, metrics *models.ConversionMetrics) (v models.VersionInfo) { // Extract versions from CPEs. for _, config := range cve.Configurations { for _, node := range config.Nodes { diff --git a/vulnfeeds/conversion/versions_test.go b/vulnfeeds/conversion/versions_test.go index bc896068261..f1906a0fa8f 100644 --- a/vulnfeeds/conversion/versions_test.go +++ b/vulnfeeds/conversion/versions_test.go @@ -726,7 +726,6 @@ func TestExtractVersionInfo(t *testing.T) { inputCVEItem: loadTestData2("CVE-2022-32746"), inputValidVersions: []string{}, expectedVersionInfo: models.VersionInfo{ - AffectedCommits: []models.AffectedCommit(nil), AffectedVersions: []models.AffectedVersion{ { Introduced: "4.3.0", @@ -752,7 +751,6 @@ func TestExtractVersionInfo(t *testing.T) { inputCVEItem: loadTestData2("CVE-2022-0090"), inputValidVersions: []string{}, expectedVersionInfo: models.VersionInfo{ - AffectedCommits: []models.AffectedCommit(nil), AffectedVersions: []models.AffectedVersion{ { Introduced: "", @@ -778,7 +776,6 @@ func TestExtractVersionInfo(t *testing.T) { inputCVEItem: loadTestData2("CVE-2022-1122"), inputValidVersions: []string{}, expectedVersionInfo: models.VersionInfo{ - AffectedCommits: []models.AffectedCommit(nil), AffectedVersions: []models.AffectedVersion{ { Introduced: "", @@ -794,13 +791,7 @@ func TestExtractVersionInfo(t *testing.T) { inputCVEItem: loadTestData2("CVE-2022-25929"), inputValidVersions: []string{}, expectedVersionInfo: models.VersionInfo{ - AffectedCommits: []models.AffectedCommit{ - { - Repo: "https://github.com/joewalnes/smoothie", - Introduced: "0", - Fixed: "8e0920d50da82f4b6e605d56f41b69fbb9606a98", - }, - }, + AffectedVersions: []models.AffectedVersion{ { Introduced: "1.31.0", @@ -816,33 +807,7 @@ func TestExtractVersionInfo(t *testing.T) { inputCVEItem: loadTestData2("CVE-2022-29194"), inputValidVersions: []string{}, expectedVersionInfo: models.VersionInfo{ - AffectedCommits: []models.AffectedCommit{ - { - Repo: "https://github.com/tensorflow/tensorflow", - Introduced: "0", - Fixed: "0516d4d8bced506cae97dc3cb45dbd2fe4311f26", - }, - { - Repo: "https://github.com/tensorflow/tensorflow", - Introduced: "0", - Fixed: "33ed2b11cb8e879d86c371700e6573db1814a69e", - }, - { - Repo: "https://github.com/tensorflow/tensorflow", - Introduced: "0", - Fixed: "8a20d54a3c1bfa38c03ea99a2ad3c1b0a45dfa95", - }, - { - Repo: "https://github.com/tensorflow/tensorflow", - Introduced: "0", - Fixed: "cff267650c6a1b266e4b4500f69fbc49cdd773c5", - }, - { - Repo: "https://github.com/tensorflow/tensorflow", - Introduced: "0", - Fixed: "dd7b8a3c1714d0052ce4b4a2fd8dcef927439a24", - }, - }, + AffectedCommits: []models.AffectedCommit(nil), AffectedVersions: []models.AffectedVersion{ { Introduced: "", @@ -878,7 +843,6 @@ func TestExtractVersionInfo(t *testing.T) { inputCVEItem: loadTestData2("CVE-2022-46285"), inputValidVersions: []string{}, expectedVersionInfo: models.VersionInfo{ - AffectedCommits: []models.AffectedCommit{{Repo: "https://gitlab.freedesktop.org/xorg/lib/libxpm", Introduced: "0", Fixed: "a3a7c6dcc3b629d7650148"}}, AffectedVersions: []models.AffectedVersion{{Introduced: "", Fixed: "3.5.15"}}, }, expectedNotes: []string{}, @@ -887,7 +851,6 @@ func TestExtractVersionInfo(t *testing.T) { description: "A CVE with a different GitWeb reference URL that was not previously being extracted successfully", inputCVEItem: loadTestData2("CVE-2021-28429"), expectedVersionInfo: models.VersionInfo{ - AffectedCommits: []models.AffectedCommit{{Repo: "https://git.ffmpeg.org/ffmpeg.git", Introduced: "0", Fixed: "c94875471e3ba3dc396c6919ff3ec9b14539cd71"}}, AffectedVersions: []models.AffectedVersion{{Introduced: "", LastAffected: "4.3.2"}}, }, }, @@ -902,19 +865,6 @@ func TestExtractVersionInfo(t *testing.T) { description: "CVE with duplicate hashes", inputCVEItem: loadTestData2("CVE-2022-25761"), expectedVersionInfo: models.VersionInfo{ - AffectedCommits: []models.AffectedCommit{ - { - Repo: "https://github.com/open62541/open62541", - Introduced: "0", - Fixed: "3010bc67fbfd8de0921fc38c9efa146cd2e02c7f", - }, - { - Repo: "https://github.com/open62541/open62541", - Introduced: "0", - Fixed: "b79db1ac78146fc06b0b8435773d3967de2d659c", - }, - }, - AffectedVersions: []models.AffectedVersion{{Introduced: "", Fixed: "1.2.5"}}, }, }, @@ -923,8 +873,6 @@ func TestExtractVersionInfo(t *testing.T) { for _, tc := range tests { t.Run(tc.description, func(t *testing.T) { t.Parallel() - r := testutils.SetupVCR(t) - client := r.GetDefaultClient() if time.Now().Before(tc.disableExpiryDate) { t.Skipf("test %q: VersionInfo for %#v has been skipped due to known outage and will be reenabled on %s.", tc.description, tc.inputCVEItem, tc.disableExpiryDate) @@ -933,7 +881,7 @@ func TestExtractVersionInfo(t *testing.T) { t.Logf("test %q: VersionInfo for %#v has been enabled on %s.", tc.description, tc.inputCVEItem, tc.disableExpiryDate) } metrics := &models.ConversionMetrics{} - gotVersionInfo := ExtractVersionInfo(tc.inputCVEItem.CVE, tc.inputValidVersions, client, metrics, nil) + gotVersionInfo := ExtractVersionInfo(tc.inputCVEItem.CVE, tc.inputValidVersions, metrics) if diff := cmp.Diff(tc.expectedVersionInfo, gotVersionInfo); diff != "" { t.Errorf("test %q: VersionInfo for %#v was incorrect: %s", tc.description, tc.inputCVEItem, diff) } @@ -941,45 +889,6 @@ func TestExtractVersionInfo(t *testing.T) { } } -type roundTripperFunc func(*http.Request) (*http.Response, error) - -func (f roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) { - return f(req) -} - -func TestExtractVersionInfo_429(t *testing.T) { - requests := 0 - client := &http.Client{ - Transport: roundTripperFunc(func(req *http.Request) (*http.Response, error) { - requests++ - return &http.Response{ - StatusCode: http.StatusTooManyRequests, - Body: http.NoBody, - Request: req, - }, nil - }), - } - - cve := models.NVDCVE{ - References: []models.Reference{ - { - URL: "https://github.com/foo/bar/commit/1234567890abcdef1234567890abcdef12345678", - }, - }, - } - - metrics := &models.ConversionMetrics{} - gotVersionInfo := ExtractVersionInfo(cve, nil, client, metrics, nil) - - // Since it's a 429 and we retry, it should eventually fail and return no affected commits. - if len(gotVersionInfo.AffectedCommits) != 0 { - t.Errorf("Expected 0 affected commits, got %d", len(gotVersionInfo.AffectedCommits)) - } - if metrics.Outcome == models.Error { - t.Errorf("Did not expected outcome to be Error, got %v", metrics.Outcome) - } -} - func TestCPEs(t *testing.T) { tests := []struct { description string