chore(vulnfeeds): remove logic for GIT gen in pypi generation and remove CVEtoPackageInfo

vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go

@@ -5,7 +5,6 @@ import (
 	"context"
 	"encoding/json"
 	"flag"
-	"fmt"
 	"log/slog"
 	"net/http"
 	"os"
@@ -32,7 +31,6 @@ var (
 	jsonDir             = flag.String("nvd-json-dir", "", "Path to directory containing NVD CVE JSON files to examine.")
 	parsedCPEDictionary = flag.String("cpe-repos", "", "Path to JSON mapping of CPEs to repos generated by cpe-repo-gen")
 	outDir              = flag.String("out-dir", "", "Path to output results.")
-	outFormat           = flag.String("out-format", "OSV", "Format to output {OSV,PackageInfo}")
 	workers             = flag.Int("workers", 10, "The number of concurrent workers to use for processing CVEs.")
 	gcsWorkers          = flag.Int("gcs-workers", 30, "The number of concurrent workers to use for GCS uploads.")
 	rejectFailed        = flag.Bool("reject-failed", false, "If set, OSV records with a failed conversion outcome will not be generated.")
@@ -51,10 +49,6 @@ var (
 
 func main() {
 	flag.Parse()
-	if !slices.Contains([]string{"OSV", "PackageInfo"}, *outFormat) {
-		fmt.Fprintf(os.Stderr, "Unsupported output format: %s\n", *outFormat)
-		os.Exit(1)
-	}
 
 	if *outDir != "" {
 		if err := os.MkdirAll(*outDir, 0755); err != nil {
@@ -202,18 +196,7 @@ func processCVE(cve models.NVDCVE, vpRepoCache *c.VPRepoCache, repoTagsCache git
 	}
 	metrics.Repos = repos
 
-	var outcome models.ConversionOutcome
-	var vuln *vulns.Vulnerability
-	var finalMetrics *models.ConversionMetrics
-	switch *outFormat {
-	case "OSV":
-		vuln, finalMetrics, outcome = nvd.CVEToOSV(cve, repos, vpRepoCache, repoTagsCache, metrics)
-	case "PackageInfo":
-		outcome = nvd.CVEToPackageInfo(cve, repos, repoTagsCache, *outDir, metrics)
-		finalMetrics = metrics
-	}
-
-	return vuln, finalMetrics, outcome
+	return nvd.CVEToOSV(cve, repos, vpRepoCache, repoTagsCache, metrics)
 }
 
 func worker(wg *sync.WaitGroup, jobs <-chan models.NVDCVE, gcsHelper *gcs.Helper, outDir string, vpRepoCache *c.VPRepoCache, repoTagsCache git.RepoTagsCache) {

vulnfeeds/cmd/pypi/main.go

@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"io/fs"
 	"log/slog"
-	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
@@ -220,7 +219,7 @@ func main() {
 
 func generatePyPIAffected(cve models.NVDCVE, pkg string, validVersions []string, purl string, metrics *models.ConversionMetrics) *vulns.Vulnerability {
 	id := "PYSEC-0000-" + cve.ID
-	versions := conversion.ExtractVersionInfo(cve, validVersions, http.DefaultClient, metrics, nil)
+	versions := conversion.ExtractVersionInfo(cve, validVersions, metrics)
 
 	pkgInfo := vulns.PackageInfo{
 		PkgName:     pkg,

vulnfeeds/conversion/nvd/converter.go

@@ -3,18 +3,13 @@ package nvd
 
 import (
 	"cmp"
-	"encoding/json"
 	"errors"
-	"log/slog"
 	"maps"
 	"net/http"
-	"os"
-	"path/filepath"
 	"slices"
 	"strings"
 
 	c "github.com/google/osv/vulnfeeds/conversion"
-	"github.com/google/osv/vulnfeeds/conversion/writer"
 	"github.com/google/osv/vulnfeeds/git"
 	"github.com/google/osv/vulnfeeds/models"
 	"github.com/google/osv/vulnfeeds/utility"
@@ -166,117 +161,6 @@ func CVEToOSV(cve models.NVDCVE, repos []string, vpRepoCache *c.VPRepoCache, cac
 	return v, metrics, metrics.Outcome
 }
 
-// CVEToPackageInfo takes an NVD CVE record and outputs a PackageInfo struct in a file in the specified directory.
-func CVEToPackageInfo(cve models.NVDCVE, repos []string, cache git.RepoTagsCache, directory string, metrics *models.ConversionMetrics) models.ConversionOutcome {
-	CPEs := c.CPEs(cve)
-	// The vendor name and product name are used to construct the output `vulnDir` below, so need to be set to *something* to keep the output tidy.
-	maybeVendorName := "ENOCPE"
-	maybeProductName := "ENOCPE"
-
-	if len(CPEs) > 0 {
-		CPE, err := c.ParseCPE(CPEs[0]) // For naming the subdirectory used for output.
-		maybeVendorName = CPE.Vendor
-		maybeProductName = CPE.Product
-		if err != nil {
-			return models.NoRanges
-		}
-	}
-
-	// more often than not, this yields a VersionInfo with AffectedVersions and no AffectedCommits.
-	versions := c.ExtractVersionInfo(cve, nil, http.DefaultClient, metrics, cache)
-	if metrics.Outcome == models.Error {
-		return models.Error
-	}
-
-	if len(versions.AffectedVersions) != 0 {
-		// There are some AffectedVersions to try and resolve to AffectedCommits.
-		if len(repos) == 0 {
-			metrics.AddNote("No affected ranges for %q, and no repos to try and convert %+v to tags with", maybeProductName, versions.AffectedVersions)
-			return models.NoRepos
-		}
-		logger.Info("Trying to convert version tags to commits", slog.String("cve", string(cve.ID)), slog.Any("versions", versions), slog.Any("repos", repos))
-		c.VersionInfoToCommits(&versions, repos, cache, metrics)
-		if metrics.Outcome == models.Error {
-			return models.Error
-		}
-	}
-
-	hasAnyFixedCommits := false
-	for _, repo := range repos {
-		if versions.HasFixedCommits(repo) {
-			hasAnyFixedCommits = true
-		}
-	}
-
-	if versions.HasFixedVersions() && !hasAnyFixedCommits {
-		metrics.AddNote("Failed to convert fixed version tags to commits: %+v", versions)
-		return models.NoCommitRanges
-	}
-
-	hasAnyLastAffectedCommits := false
-	for _, repo := range repos {
-		if versions.HasLastAffectedCommits(repo) {
-			hasAnyLastAffectedCommits = true
-		}
-	}
-
-	if versions.HasLastAffectedVersions() && !hasAnyLastAffectedCommits && !hasAnyFixedCommits {
-		metrics.AddNote("Failed to convert last_affected version tags to commits: %+v", versions)
-		return models.NoCommitRanges
-	}
-
-	if len(versions.AffectedCommits) == 0 {
-		metrics.AddNote("No affected commit ranges determined for %q", maybeProductName)
-		return models.NoCommitRanges
-	}
-
-	versions.AffectedVersions = nil // these have served their purpose and are not required in the resulting output.
-
-	slices.SortStableFunc(versions.AffectedCommits, models.AffectedCommitCompare)
-
-	if metrics.Outcome == models.Error {
-		return metrics.Outcome
-	}
-
-	var pkgInfos []vulns.PackageInfo
-	pi := vulns.PackageInfo{VersionInfo: versions}
-	pkgInfos = append(pkgInfos, pi) // combine-to-osv expects a serialised *array* of PackageInfo
-
-	vulnDir := filepath.Join(directory, maybeVendorName, maybeProductName)
-	err := os.MkdirAll(vulnDir, 0755)
-	if err != nil {
-		logger.Warn("Failed to create dir", slog.Any("err", err))
-	}
-
-	outputFile := filepath.Join(vulnDir, string(cve.ID)+".nvd"+models.Extension)
-	f, err := os.Create(outputFile)
-	if err != nil {
-		logger.Warn("Failed to open for writing", slog.String("path", outputFile), slog.Any("err", err))
-	}
-	defer f.Close()
-
-	encoder := json.NewEncoder(f)
-	encoder.SetIndent("", "  ")
-	err = encoder.Encode(&pkgInfos)
-
-	if err != nil {
-		logger.Warn("Failed to encode PackageInfo", slog.String("path", outputFile), slog.Any("err", err))
-	}
-
-	logger.Info("Generated PackageInfo record", slog.String("cve", string(cve.ID)), slog.String("product", maybeProductName))
-
-	metricsFile, err := writer.CreateMetricsFile(cve.ID, vulnDir)
-	if err != nil {
-		logger.Warn("Failed to create metrics file", slog.String("path", metricsFile.Name()), slog.Any("err", err))
-	}
-	err = writer.WriteMetricsFile(metrics, metricsFile)
-	if err != nil {
-		logger.Warn("Failed to write metrics file", slog.String("path", metricsFile.Name()), slog.Any("err", err))
-	}
-
-	return metrics.Outcome
-}
-
 // FindRepos attempts to find the source code repositories for a given CVE.
 func FindRepos(cve models.NVDCVE, vpRepoCache *c.VPRepoCache, repoTagsCache git.RepoTagsCache, metrics *models.ConversionMetrics, httpClient *http.Client) []string {
 	// Find repos

vulnfeeds/conversion/versions.go

@@ -870,17 +870,8 @@ func ExtractVersionsFromCPEs(cve models.NVDCVE, validVersions []string, vpRepoCa
 }
 
 // ExtractVersionInfo extracts version information from a CVE and saves to a VersionInfo struct.
-// This is mostly deprecated, but is still used by the Alpine, Debian, and PyPi converters.
-func ExtractVersionInfo(cve models.NVDCVE, validVersions []string, httpClient *http.Client, metrics *models.ConversionMetrics, cache git.RepoTagsCache) (v models.VersionInfo) {
-	if commit, err := ExtractCommitsFromRefs(cve.References, httpClient, cache); err == nil {
-		v.AffectedCommits = append(v.AffectedCommits, commit...)
-	}
-
-	if v.AffectedCommits != nil {
-		v.AffectedCommits = DeduplicateAffectedCommits(v.AffectedCommits)
-		metrics.AddNote("Extracted %d commits", len(v.AffectedCommits))
-	}
-
+// This is mostly deprecated, but is still used by the PyPi converter.
+func ExtractVersionInfo(cve models.NVDCVE, validVersions []string, metrics *models.ConversionMetrics) (v models.VersionInfo) {
 	// Extract versions from CPEs.
 	for _, config := range cve.Configurations {
 		for _, node := range config.Nodes {

vulnfeeds/conversion/versions_test.go

@@ -726,7 +726,6 @@ func TestExtractVersionInfo(t *testing.T) {
 			inputCVEItem:       loadTestData2("CVE-2022-32746"),
 			inputValidVersions: []string{},
 			expectedVersionInfo: models.VersionInfo{
-				AffectedCommits: []models.AffectedCommit(nil),
 				AffectedVersions: []models.AffectedVersion{
 					{
 						Introduced:   "4.3.0",
@@ -752,7 +751,6 @@ func TestExtractVersionInfo(t *testing.T) {
 			inputCVEItem:       loadTestData2("CVE-2022-0090"),
 			inputValidVersions: []string{},
 			expectedVersionInfo: models.VersionInfo{
-				AffectedCommits: []models.AffectedCommit(nil),
 				AffectedVersions: []models.AffectedVersion{
 					{
 						Introduced:   "",
@@ -778,7 +776,6 @@ func TestExtractVersionInfo(t *testing.T) {
 			inputCVEItem:       loadTestData2("CVE-2022-1122"),
 			inputValidVersions: []string{},
 			expectedVersionInfo: models.VersionInfo{
-				AffectedCommits: []models.AffectedCommit(nil),
 				AffectedVersions: []models.AffectedVersion{
 					{
 						Introduced:   "",
@@ -794,13 +791,7 @@ func TestExtractVersionInfo(t *testing.T) {
 			inputCVEItem:       loadTestData2("CVE-2022-25929"),
 			inputValidVersions: []string{},
 			expectedVersionInfo: models.VersionInfo{
-				AffectedCommits: []models.AffectedCommit{
-					{
-						Repo:       "https://github.com/joewalnes/smoothie",
-						Introduced: "0",
-						Fixed:      "8e0920d50da82f4b6e605d56f41b69fbb9606a98",
-					},
-				},
+
 				AffectedVersions: []models.AffectedVersion{
 					{
 						Introduced:   "1.31.0",
@@ -816,33 +807,7 @@ func TestExtractVersionInfo(t *testing.T) {
 			inputCVEItem:       loadTestData2("CVE-2022-29194"),
 			inputValidVersions: []string{},
 			expectedVersionInfo: models.VersionInfo{
-				AffectedCommits: []models.AffectedCommit{
-					{
-						Repo:       "https://github.com/tensorflow/tensorflow",
-						Introduced: "0",
-						Fixed:      "0516d4d8bced506cae97dc3cb45dbd2fe4311f26",
-					},
-					{
-						Repo:       "https://github.com/tensorflow/tensorflow",
-						Introduced: "0",
-						Fixed:      "33ed2b11cb8e879d86c371700e6573db1814a69e",
-					},
-					{
-						Repo:       "https://github.com/tensorflow/tensorflow",
-						Introduced: "0",
-						Fixed:      "8a20d54a3c1bfa38c03ea99a2ad3c1b0a45dfa95",
-					},
-					{
-						Repo:       "https://github.com/tensorflow/tensorflow",
-						Introduced: "0",
-						Fixed:      "cff267650c6a1b266e4b4500f69fbc49cdd773c5",
-					},
-					{
-						Repo:       "https://github.com/tensorflow/tensorflow",
-						Introduced: "0",
-						Fixed:      "dd7b8a3c1714d0052ce4b4a2fd8dcef927439a24",
-					},
-				},
+				AffectedCommits: []models.AffectedCommit(nil),
 				AffectedVersions: []models.AffectedVersion{
 					{
 						Introduced:   "",
@@ -878,7 +843,6 @@ func TestExtractVersionInfo(t *testing.T) {
 			inputCVEItem:       loadTestData2("CVE-2022-46285"),
 			inputValidVersions: []string{},
 			expectedVersionInfo: models.VersionInfo{
-				AffectedCommits:  []models.AffectedCommit{{Repo: "https://gitlab.freedesktop.org/xorg/lib/libxpm", Introduced: "0", Fixed: "a3a7c6dcc3b629d7650148"}},
 				AffectedVersions: []models.AffectedVersion{{Introduced: "", Fixed: "3.5.15"}},
 			},
 			expectedNotes: []string{},
@@ -887,7 +851,6 @@ func TestExtractVersionInfo(t *testing.T) {
 			description:  "A CVE with a different GitWeb reference URL that was not previously being extracted successfully",
 			inputCVEItem: loadTestData2("CVE-2021-28429"),
 			expectedVersionInfo: models.VersionInfo{
-				AffectedCommits:  []models.AffectedCommit{{Repo: "https://git.ffmpeg.org/ffmpeg.git", Introduced: "0", Fixed: "c94875471e3ba3dc396c6919ff3ec9b14539cd71"}},
 				AffectedVersions: []models.AffectedVersion{{Introduced: "", LastAffected: "4.3.2"}},
 			},
 		},
@@ -902,19 +865,6 @@ func TestExtractVersionInfo(t *testing.T) {
 			description:  "CVE with duplicate hashes",
 			inputCVEItem: loadTestData2("CVE-2022-25761"),
 			expectedVersionInfo: models.VersionInfo{
-				AffectedCommits: []models.AffectedCommit{
-					{
-						Repo:       "https://github.com/open62541/open62541",
-						Introduced: "0",
-						Fixed:      "3010bc67fbfd8de0921fc38c9efa146cd2e02c7f",
-					},
-					{
-						Repo:       "https://github.com/open62541/open62541",
-						Introduced: "0",
-						Fixed:      "b79db1ac78146fc06b0b8435773d3967de2d659c",
-					},
-				},
-
 				AffectedVersions: []models.AffectedVersion{{Introduced: "", Fixed: "1.2.5"}},
 			},
 		},
@@ -923,8 +873,6 @@ func TestExtractVersionInfo(t *testing.T) {
 	for _, tc := range tests {
 		t.Run(tc.description, func(t *testing.T) {
 			t.Parallel()
-			r := testutils.SetupVCR(t)
-			client := r.GetDefaultClient()
 
 			if time.Now().Before(tc.disableExpiryDate) {
 				t.Skipf("test %q: VersionInfo for %#v has been skipped due to known outage and will be reenabled on %s.", tc.description, tc.inputCVEItem, tc.disableExpiryDate)
@@ -933,53 +881,14 @@ func TestExtractVersionInfo(t *testing.T) {
 				t.Logf("test %q: VersionInfo for %#v has been enabled on %s.", tc.description, tc.inputCVEItem, tc.disableExpiryDate)
 			}
 			metrics := &models.ConversionMetrics{}
-			gotVersionInfo := ExtractVersionInfo(tc.inputCVEItem.CVE, tc.inputValidVersions, client, metrics, nil)
+			gotVersionInfo := ExtractVersionInfo(tc.inputCVEItem.CVE, tc.inputValidVersions, metrics)
 			if diff := cmp.Diff(tc.expectedVersionInfo, gotVersionInfo); diff != "" {
 				t.Errorf("test %q: VersionInfo for %#v was incorrect: %s", tc.description, tc.inputCVEItem, diff)
 			}
 		})
 	}
 }
 
-type roundTripperFunc func(*http.Request) (*http.Response, error)
-
-func (f roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) {
-	return f(req)
-}
-
-func TestExtractVersionInfo_429(t *testing.T) {
-	requests := 0
-	client := &http.Client{
-		Transport: roundTripperFunc(func(req *http.Request) (*http.Response, error) {
-			requests++
-			return &http.Response{
-				StatusCode: http.StatusTooManyRequests,
-				Body:       http.NoBody,
-				Request:    req,
-			}, nil
-		}),
-	}
-
-	cve := models.NVDCVE{
-		References: []models.Reference{
-			{
-				URL: "https://github.com/foo/bar/commit/1234567890abcdef1234567890abcdef12345678",
-			},
-		},
-	}
-
-	metrics := &models.ConversionMetrics{}
-	gotVersionInfo := ExtractVersionInfo(cve, nil, client, metrics, nil)
-
-	// Since it's a 429 and we retry, it should eventually fail and return no affected commits.
-	if len(gotVersionInfo.AffectedCommits) != 0 {
-		t.Errorf("Expected 0 affected commits, got %d", len(gotVersionInfo.AffectedCommits))
-	}
-	if metrics.Outcome == models.Error {
-		t.Errorf("Did not expected outcome to be Error, got %v", metrics.Outcome)
-	}
-}
-
 func TestCPEs(t *testing.T) {
 	tests := []struct {
 		description  string