diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/combine-to-osv.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/combine-to-osv.yaml index 6ab13e680c4..124751dd550 100644 --- a/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/combine-to-osv.yaml +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb-test/combine-to-osv.yaml @@ -16,3 +16,5 @@ spec: value: osv-test-cve-osv-conversion - name: OUTPUT_GCS_BUCKET value: osv-test-cve-osv-conversion + - name: NUM_WORKERS + value: "1000" diff --git a/deployment/clouddeploy/gke-workers/environments/oss-vdb/combine-to-osv.yaml b/deployment/clouddeploy/gke-workers/environments/oss-vdb/combine-to-osv.yaml index 59604ef3283..15be287c44d 100644 --- a/deployment/clouddeploy/gke-workers/environments/oss-vdb/combine-to-osv.yaml +++ b/deployment/clouddeploy/gke-workers/environments/oss-vdb/combine-to-osv.yaml @@ -15,3 +15,7 @@ spec: value: oss-vdb - name: OUTPUT_GCS_BUCKET value: cve-osv-conversion + - name: INPUT_GCS_BUCKET + value: cve-osv-conversion + - name: NUM_WORKERS + value: "1000" \ No newline at end of file diff --git a/vulnfeeds/cmd/combine-to-osv/main.go b/vulnfeeds/cmd/combine-to-osv/main.go index e1c97a81c07..d8dfb595b48 100644 --- a/vulnfeeds/cmd/combine-to-osv/main.go +++ b/vulnfeeds/cmd/combine-to-osv/main.go @@ -4,24 +4,27 @@ package main import ( "cmp" "context" - "errors" "flag" "fmt" + "io" "io/fs" "log/slog" "os" + "path" "path/filepath" "slices" "strings" + "sync" + "sync/atomic" "cloud.google.com/go/storage" "github.com/google/osv/vulnfeeds/conversion" "github.com/google/osv/vulnfeeds/conversion/writer" + "github.com/google/osv/vulnfeeds/gcs-tools" "github.com/google/osv/vulnfeeds/models" "github.com/google/osv/vulnfeeds/utility" "github.com/google/osv/vulnfeeds/utility/logger" "github.com/ossf/osv-schema/bindings/go/osvschema" - "google.golang.org/api/iterator" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/types/known/structpb" ) @@ -32,190 +35,334 @@ const ( defaultNVDOSVPath = "nvd" ) -func main() { - logger.InitGlobalLogger() - defer logger.Close() - - cve5Path := flag.String("cve5-path", defaultCVE5Path, "Path to CVE5 OSV files") - nvdPath := flag.String("nvd-path", defaultNVDOSVPath, "Path to NVD OSV files") - osvOutputPath := flag.String("osv-output-path", defaultOSVOutputPath, "Local output path of combined OSV files, or GCS prefix if uploading.") - outputBucketName := flag.String("output-bucket", "osv-test-cve-osv-conversion", "The GCS bucket to write to.") - overridesBucketName := flag.String("overrides-bucket", "osv-test-cve-osv-conversion", "The GCS bucket to read overrides from.") - uploadToGCS := flag.Bool("upload-to-gcs", false, "If true, upload to GCS bucket instead of writing to local disk.") - numWorkers := flag.Int("workers", 64, "Number of workers to process records") - syncDeletions := flag.Bool("sync-deletions", false, "If false, do not delete files in bucket that are not local") - flag.Parse() +// CVEWorkItem represents a unit of work for a single CVE. +type CVEWorkItem struct { + ID models.CVEID + CVE5Path string + NVDPath string +} - err := os.MkdirAll(*osvOutputPath, 0755) - if err != nil { - logger.Fatal("Can't create output path", slog.Any("err", err)) +func cveIDFromPath(p string) models.CVEID { + var base string + if strings.HasPrefix(p, "gs://") { + base = path.Base(p) + } else { + base = filepath.Base(p) + } + id := strings.TrimSuffix(base, ".json") + if strings.HasPrefix(id, "CVE-") { + return models.CVEID(id) } - // Load CVE5 OSVs - allCVE5 := loadOSV(*cve5Path) - // Load NVD OSVs - allNVD := loadOSV(*nvdPath) - debianCVEs, err := listBucketObjects("osv-test-debian-osv", "/debian-cve-osv") - if err != nil { - logger.Warn("Failed to list debian cves", slog.Any("err", err)) - } else { - for i, filename := range debianCVEs { - cve := extractCVEName(filename, "DEBIAN-") - if cve != "" { - debianCVEs[i] = cve + return "" +} + +func listObjects(ctx context.Context, client *storage.Client, pathStr string) ([]string, error) { + if strings.HasPrefix(pathStr, "gs://") { + trimmed := strings.TrimPrefix(pathStr, "gs://") + bucketName, prefix, _ := strings.Cut(trimmed, "/") + bucket := client.Bucket(bucketName) + objs, err := gcs.ListBucketObjects(ctx, bucket, prefix) + if err != nil { + return nil, err + } + var fullPaths []string + for _, obj := range objs { + if strings.HasSuffix(obj, "/") || !strings.HasSuffix(obj, ".json") || strings.HasSuffix(obj, ".metrics.json") { + continue } + fullPaths = append(fullPaths, fmt.Sprintf("gs://%s/%s", bucketName, obj)) } + + return fullPaths, nil } - // run extract file name on each element in debianCVEs and alpineCVEs. - alpineCVEs, err := listBucketObjects("osv-test-cve-osv-conversion", "/alpine") - if err != nil { - logger.Warn("Failed to list alpine cves", slog.Any("err", err)) - } else { - for i, filename := range alpineCVEs { - cve := extractCVEName(filename, "ALPINE-") - if cve != "" { - alpineCVEs[i] = cve - } + var files []string + err := filepath.WalkDir(pathStr, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() && strings.HasSuffix(p, ".json") && !strings.HasSuffix(p, ".metrics.json") { + files = append(files, p) } - } - // this ensures the creation of CVEs even if they don't have packages - // to ensure Alpine and Debian CVEs have an upstream CVE. - // linter is compaining that we aren't appending to the same slice, but we - // just want to combine these two arrays with a more descriptive name. - mandatoryCVEIDs := append(debianCVEs, alpineCVEs...) //nolint:gocritic - combinedData := combineIntoOSV(allCVE5, allNVD, mandatoryCVEIDs) + return nil + }) - ctx := context.Background() + return files, err +} - vulnerabilities := make([]*osvschema.Vulnerability, 0, len(combinedData)) - for _, v := range combinedData { - vulnerabilities = append(vulnerabilities, v) - } +func readVulnerability(ctx context.Context, client *storage.Client, fullPath string) (*osvschema.Vulnerability, error) { + if strings.HasPrefix(fullPath, "gs://") { + trimmed := strings.TrimPrefix(fullPath, "gs://") + bucketName, objName, _ := strings.Cut(trimmed, "/") + rc, err := client.Bucket(bucketName).Object(objName).NewReader(ctx) + if err != nil { + return nil, err + } + defer rc.Close() + file, err := io.ReadAll(rc) + if err != nil { + return nil, err + } + var vuln osvschema.Vulnerability + if err := protojson.Unmarshal(file, &vuln); err != nil { + return nil, err + } - writer.UploadVulnsToGCS(ctx, "OSV files", *uploadToGCS, *outputBucketName, *overridesBucketName, *numWorkers, *osvOutputPath, vulnerabilities, *syncDeletions) -} + return &vuln, nil + } -// extractCVEName extracts the CVE name from a given filename and prefix. -// It returns an empty string if the filename does not start with "CVE". -func extractCVEName(filename string, prefix string) string { - cleaned := strings.TrimPrefix(filename, prefix) - cleaned = strings.TrimSuffix(cleaned, ".json") - pre := strings.Split(cleaned, "-") - if pre[0] != "CVE" { - return "" + file, err := os.ReadFile(fullPath) + if err != nil { + return nil, err + } + var vuln osvschema.Vulnerability + if err := protojson.Unmarshal(file, &vuln); err != nil { + return nil, err } - return cleaned + return &vuln, nil } -// listBucketObjects lists the names of all objects in a Google Cloud Storage bucket. -// It does not download the file contents. -func listBucketObjects(bucketName string, prefix string) ([]string, error) { - ctx := context.Background() - client, err := storage.NewClient(ctx) - if err != nil { - return nil, fmt.Errorf("storage.NewClient: %w", err) - } - defer client.Close() - bucket := client.Bucket(bucketName) - it := bucket.Objects(ctx, &storage.Query{Prefix: prefix}) - var filenames []string - for { - attrs, err := it.Next() - if errors.Is(err, iterator.Done) { - break // All objects have been listed. - } - if err != nil { - return nil, fmt.Errorf("bucket.Objects: %w", err) - } - filenames = append(filenames, attrs.Name) +func combineIntoOSV(cve5 *osvschema.Vulnerability, nvd *osvschema.Vulnerability) *osvschema.Vulnerability { + var baseOSV *osvschema.Vulnerability + if cve5 != nil && nvd != nil { + baseOSV = combineTwoOSVRecords(cve5, nvd) + } else if cve5 != nil { + baseOSV = cve5 + } else if nvd != nil { + baseOSV = nvd + } else { + return nil } - return filenames, nil + return baseOSV } -// loadOSV recursively loads all OSV vulnerabilities from a given directory path. -// It walks the directory, reads each ".json" file, and decodes it into an osvschema.Vulnerability object. -// The function returns a map of CVE IDs to their corresponding Vulnerability objects. -// Files that are not ".json" files, directories, or files ending in ".metrics.json" are skipped. -// The function will log warnings for files that fail to open or decode, and will terminate if it fails to walk the directory. -func loadOSV(osvPath string) map[models.CVEID]*osvschema.Vulnerability { - allVulns := make(map[models.CVEID]*osvschema.Vulnerability) - logger.Info("Loading OSV records", slog.String("path", osvPath)) - err := filepath.WalkDir(osvPath, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err +func readAndCombineWorker(ctx context.Context, client *storage.Client, workChan <-chan *CVEWorkItem, vulnChan chan<- *osvschema.Vulnerability) { + for work := range workChan { + var cve5, nvd *osvschema.Vulnerability + var cve5Err, nvdErr error + var readVulnsWg sync.WaitGroup + + if work.CVE5Path != "" { + readVulnsWg.Add(1) + go func() { + defer readVulnsWg.Done() + cve5, cve5Err = readVulnerability(ctx, client, work.CVE5Path) + if cve5Err != nil { + logger.Error("Failed to read CVE5", slog.String("id", string(work.ID)), slog.Any("err", cve5Err)) + } + }() } - if d.IsDir() || !strings.HasSuffix(path, ".json") || strings.HasSuffix(path, ".metrics.json") { - return nil + + if work.NVDPath != "" { + readVulnsWg.Add(1) + go func() { + defer readVulnsWg.Done() + nvd, nvdErr = readVulnerability(ctx, client, work.NVDPath) + if nvdErr != nil { + logger.Error("Failed to read NVD", slog.String("id", string(work.ID)), slog.Any("err", nvdErr)) + } + }() } - file, err := os.ReadFile(path) - if err != nil { - logger.Warn("Failed to open OSV JSON file", slog.String("path", path), slog.Any("err", err)) - return nil + readVulnsWg.Wait() + + if cve5Err != nil || nvdErr != nil { + continue } - var vuln osvschema.Vulnerability - decodeErr := protojson.Unmarshal(file, &vuln) - if decodeErr != nil { - logger.Error("Failed to decode, skipping", slog.String("file", path), slog.Any("err", decodeErr)) - return nil + combined := combineIntoOSV(cve5, nvd) + if combined != nil { + vulnChan <- combined } - allVulns[models.CVEID(vuln.GetId())] = &vuln + } +} - return nil - }) +func main() { + logger.InitGlobalLogger() + defer logger.Close() + cve5Path := flag.String("cve5-path", defaultCVE5Path, "Path to CVE5 OSV files") + nvdPath := flag.String("nvd-path", defaultNVDOSVPath, "Path to NVD OSV files") + osvOutputPath := flag.String("osv-output-path", defaultOSVOutputPath, "Local output path of combined OSV files, or GCS prefix if uploading.") + outputBucketName := flag.String("output-bucket", "osv-test-cve-osv-conversion", "The GCS bucket to write to.") + overridesBucketName := flag.String("overrides-bucket", "osv-test-cve-osv-conversion", "The GCS bucket to read overrides from.") + uploadToGCS := flag.Bool("upload-to-gcs", false, "If true, upload to GCS bucket instead of writing to local disk.") + numWorkers := flag.Int("workers", 64, "Number of workers to process records") + syncDeletions := flag.Bool("sync-deletions", false, "If false, do not delete files in bucket that are not local") + flag.Parse() + + err := os.MkdirAll(*osvOutputPath, 0755) if err != nil { - logger.Fatal("Failed to walk OSV directory", slog.String("path", osvPath), slog.Any("err", err)) + logger.Fatal("Can't create output path", slog.Any("err", err)) } - return allVulns -} - -// combineIntoOSV creates OSV entry by combining loaded CVEs from NVD and PackageInfo information from security advisories. -func combineIntoOSV(cve5osv map[models.CVEID]*osvschema.Vulnerability, nvdosv map[models.CVEID]*osvschema.Vulnerability, mandatoryCVEIDs []string) map[models.CVEID]*osvschema.Vulnerability { - osvRecords := make(map[models.CVEID]*osvschema.Vulnerability) - - // Iterate through CVEs from security advisories (cve5) as the base - for cveID, cve5 := range cve5osv { - var baseOSV *osvschema.Vulnerability - nvd, ok := nvdosv[cveID] + ctx := context.Background() + client, err := storage.NewClient(ctx) + if err != nil { + logger.Fatal("Failed to create storage client", slog.Any("err", err)) + } + defer client.Close() - if ok { - baseOSV = combineTwoOSVRecords(cve5, nvd) - // The CVE is processed, so remove it from the nvdosv map to avoid re-processing. - delete(nvdosv, cveID) - } else { - baseOSV = cve5 + // List CVE5 and NVD objects + var cve5Files, nvdFiles []string + var cve5ListErr, nvdListErr error + logger.Info("Starting to list CVE5 and NVD objects") + var listObjWg sync.WaitGroup + listObjWg.Add(2) + + go func() { + defer listObjWg.Done() + logger.Info("Listing CVE5 objects", slog.String("path", *cve5Path)) + cve5Files, cve5ListErr = listObjects(ctx, client, *cve5Path) + if cve5ListErr != nil { + logger.Fatal("Failed to list CVE5 objects", slog.Any("err", cve5ListErr)) } + }() + + go func() { + defer listObjWg.Done() + logger.Info("Listing NVD objects", slog.String("path", *nvdPath)) + nvdFiles, nvdListErr = listObjects(ctx, client, *nvdPath) + if nvdListErr != nil { + logger.Fatal("Failed to list NVD objects", slog.Any("err", nvdListErr)) + } + }() + + listObjWg.Wait() + + // Build work items + logger.Info("Starting to build work items") + workItems := make(map[models.CVEID]*CVEWorkItem) + for _, f := range cve5Files { + id := cveIDFromPath(f) + if id != "" { + if _, ok := workItems[id]; !ok { + workItems[id] = &CVEWorkItem{ID: id} + } + workItems[id].CVE5Path = f + } + } - if len(baseOSV.GetAffected()) == 0 || !hasRanges(baseOSV.GetAffected()) { - // check if part exists. - if !slices.Contains(mandatoryCVEIDs, string(cveID)) { - continue + for _, f := range nvdFiles { + id := cveIDFromPath(f) + if id != "" { + if _, ok := workItems[id]; !ok { + workItems[id] = &CVEWorkItem{ID: id} } + workItems[id].NVDPath = f } - osvRecords[cveID] = baseOSV } - // Add any remaining CVEs from NVD that were not in the advisory data. - for cveID, nvd := range nvdosv { - if len(nvd.GetAffected()) == 0 || !hasRanges(nvd.GetAffected()) { - continue + logger.Info("Total CVE Work Items to process", slog.Int("count", len(workItems))) + + // Start Upload Pool + var outBkt, overridesBkt *storage.BucketHandle + var gcsHelper *gcs.Helper + if *uploadToGCS { + outBkt = client.Bucket(*outputBucketName) + if *overridesBucketName != "" { + overridesBkt = client.Bucket(*overridesBucketName) + } + gcsHelper, err = gcs.InitUploadPool(ctx, *numWorkers, *outputBucketName) + if err != nil { + logger.Fatal("Failed to initialize GCS upload pool", slog.Any("err", err)) + } + defer gcsHelper.CloseAndWait() + } + + // Start Channels + // Channel Flow: + // + // [workItems] ---workChan---> [readAndCombineWorker] ---vulnChan---> [Collector] ---outputVulnChan---> [VulnWorker] ---> [GCS/Disk] + // | + // +---> (filters invalid) + logger.Info("Starting processing channels and workers") + vulnChan := make(chan *osvschema.Vulnerability, *numWorkers) + uploadVulnsChan := make(chan *osvschema.Vulnerability, *numWorkers) + workChan := make(chan *CVEWorkItem, *numWorkers) + + // Start VulnWorkers (Upload side) + var uploadVulnsWg sync.WaitGroup + var successCount atomic.Uint64 + for range *numWorkers { + uploadVulnsWg.Add(1) + go func() { + defer uploadVulnsWg.Done() + writer.VulnWorker(ctx, uploadVulnsChan, outBkt, overridesBkt, gcsHelper, *osvOutputPath, &successCount) + }() + } + + // Interpose Collector to gather valid IDs + var validIDs []string + var collectValidIDsWg sync.WaitGroup + collectValidIDsWg.Add(1) + totalWork := len(workItems) + go func() { + defer collectValidIDsWg.Done() + count := 0 + for v := range vulnChan { + count++ + if len(v.GetAffected()) > 0 { + validIDs = append(validIDs, v.GetId()) + } + uploadVulnsChan <- v + if count%1000 == 0 { + logger.Info("Processed CVEs", slog.Int("count", count), slog.Int("total", totalWork), slog.Int("percent", (count*100)/totalWork)) + } + } + close(uploadVulnsChan) + }() + + // Start ReadAndCombineWorkers (Read side) + var readAndCombineWg sync.WaitGroup + for range *numWorkers { + readAndCombineWg.Add(1) + go func() { + defer readAndCombineWg.Done() + readAndCombineWorker(ctx, client, workChan, vulnChan) + }() + } + + // Feed Work + go func() { + for _, work := range workItems { + workChan <- work } - osvRecords[cveID] = nvd + close(workChan) + }() + + // Wait for reads to finish + readAndCombineWg.Wait() + close(vulnChan) + + // Wait for collector and uploads to finish + collectValidIDsWg.Wait() + uploadVulnsWg.Wait() + + logger.Info("Successfully processed OSV files", slog.Int("count", len(validIDs))) + if outBkt == nil && gcsHelper == nil { + logger.Info("Successfully wrote records to disk", slog.Uint64("count", successCount.Load())) } - return osvRecords + // Handle Deletion + if *syncDeletions && *uploadToGCS { + writer.HandleDeletion(ctx, outBkt, *osvOutputPath, validIDs) + } } // combineTwoOSVRecords takes two osv records and combines them into one func combineTwoOSVRecords(cve5 *osvschema.Vulnerability, nvd *osvschema.Vulnerability) *osvschema.Vulnerability { baseOSV := cve5 + if baseOSV.GetDetails() == "" && nvd.GetDetails() != "" { + baseOSV.Details = nvd.GetDetails() + } + if baseOSV.GetSummary() == "" && nvd.GetSummary() != "" { + baseOSV.Summary = nvd.GetSummary() + } combinedAffected := pickAffectedInformation(cve5.GetAffected(), nvd.GetAffected()) baseOSV.Affected = combinedAffected @@ -876,16 +1023,6 @@ func pickBestRange(cve5Range *osvschema.Range, nvdRange *osvschema.Range) *osvsc return merged } -func hasRanges(affected []*osvschema.Affected) bool { - for _, a := range affected { - if len(a.GetRanges()) > 0 { - return true - } - } - - return false -} - // getRangeBoundaryVersions extracts the introduced, last_affected and fixed versions from a slice of OSV events. // It iterates through the events and returns the last non-empty "introduced", "last_affected" and "fixed" versions found. func getRangeBoundaryVersions(events []*osvschema.Event) (introduced, lastAffected, fixed string) { diff --git a/vulnfeeds/cmd/combine-to-osv/main_test.go b/vulnfeeds/cmd/combine-to-osv/main_test.go index aaf0139f004..90f0e9873b8 100644 --- a/vulnfeeds/cmd/combine-to-osv/main_test.go +++ b/vulnfeeds/cmd/combine-to-osv/main_test.go @@ -1,136 +1,18 @@ package main import ( - "path/filepath" "sort" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - "github.com/google/osv/vulnfeeds/models" "github.com/ossf/osv-schema/bindings/go/osvschema" "google.golang.org/protobuf/testing/protocmp" "google.golang.org/protobuf/types/known/structpb" "google.golang.org/protobuf/types/known/timestamppb" ) -const testdataPath = "../../test_data/combine-to-osv" - -func TestLoadOSV(t *testing.T) { - cve5Path := filepath.Join(testdataPath, "cve5") - allVulns := loadOSV(cve5Path) - - if len(allVulns) != 4 { - t.Errorf("Expected 4 vulnerabilities, got %d", len(allVulns)) - } - - if _, ok := allVulns["CVE-2023-1234"]; !ok { - t.Error("Expected to load CVE-2023-1234") - } -} - -func TestCombineIntoOSV(t *testing.T) { - cve5Path := filepath.Join(testdataPath, "cve5") - nvdPath := filepath.Join(testdataPath, "nvd") - - cve5osv := loadOSV(cve5Path) - nvdosv := loadOSV(nvdPath) - nvdosvCopy := make(map[models.CVEID]*osvschema.Vulnerability) - for k, v := range nvdosv { - nvdosvCopy[k] = v - } - noPkgCVEs := []string{"CVE-2023-0003"} - - combined := combineIntoOSV(cve5osv, nvdosvCopy, noPkgCVEs) - - // Expected results - // CVE-2023-1234: merged - // CVE-2023-0001: from cve5 only - // CVE-2023-0002: from nvd only - // CVE-2023-0003: from cve5, no affected, but in noPkgCVEs - // CVE-2023-0004: from cve5, no affected, not in noPkgCVEs, so skipped - if len(combined) != 2 { - t.Errorf("Expected 2 combined vulnerabilities, got %d", len(combined)) - } - - // Test case 1: Merged CVE - cve1234, ok := combined["CVE-2023-1234"] - if !ok { - t.Fatal("Expected combined map to contain CVE-2023-1234") - } - - // Check modified and published dates - expectedModified, _ := time.Parse(time.RFC3339, "2023-01-02T12:00:00Z") - if !cve1234.GetModified().AsTime().Equal(expectedModified) { - t.Errorf("CVE-2023-1234: expected modified time %v, got %v", expectedModified, cve1234.GetModified()) - } - expectedPublished, _ := time.Parse(time.RFC3339, "2023-01-01T09:00:00Z") - if !cve1234.GetPublished().AsTime().Equal(expectedPublished) { - t.Errorf("CVE-2023-1234: expected published time %v, got %v", expectedPublished, cve1234.GetPublished()) - } - - // Check references - if len(cve1234.GetReferences()) != 2 { - t.Errorf("CVE-2023-1234: expected 2 references, got %d", len(cve1234.GetReferences())) - } - - // Check aliases - if len(cve1234.GetAliases()) != 2 { - t.Errorf("CVE-2023-1234: expected 2 aliases, got %d", len(cve1234.GetAliases())) - } - - // Check affected (based on pickAffectedInformation logic) - var affectedForRepoA *osvschema.Affected - foundAffected := false - for _, a := range cve1234.GetAffected() { - if len(a.GetRanges()) > 0 && a.GetRanges()[0].GetRepo() == "https://example.com/repo/a" { - affectedForRepoA = a - foundAffected = true - - break - } - } - if !foundAffected { - t.Fatal("Did not find affected for repo https://example.com/repo/a") - } - - expectedRange := &osvschema.Range{ - Type: osvschema.Range_GIT, - Repo: "https://example.com/repo/a", - Events: []*osvschema.Event{ - {Introduced: "1.0.0"}, - {Fixed: "1.0.1"}, - }, - } - - // The current logic for pickAffectedInformation when len(cveRanges) == 1 && len(nvdRanges) == 1 - // is to prefer cve5 data. - if diff := cmp.Diff(expectedRange, affectedForRepoA.GetRanges()[0], protocmp.Transform()); diff != "" { - t.Errorf("CVE-2023-1234: affected range mismatch (-want +got):\n%s", diff) - } - - // Test case 2: CVE only in cve5 (has no ranges, so it should be skipped) - if _, ok = combined["CVE-2023-0001"]; ok { - t.Error("Expected combined map to NOT contain CVE-2023-0001 because it has no ranges") - } - - // Test case 3: CVE only in nvd (has no ranges, so it should be skipped) - if _, ok = combined["CVE-2023-0002"]; ok { - t.Error("Expected combined map to NOT contain CVE-2023-0002 because it has no ranges") - } - - // Test case 4: No ranges, in noPkgCVEs (should be kept) - if _, ok = combined["CVE-2023-0003"]; !ok { - t.Error("Expected combined map to contain CVE-2023-0003") - } - - // Test case 5: No ranges, not in noPkgCVEs (should be skipped) - if _, ok = combined["CVE-2023-0004"]; ok { - t.Error("Expected combined map to NOT contain CVE-2023-0004") - } -} - func TestPickAffectedInformation(t *testing.T) { repoA := "https://example.com/repo/a" repoB := "https://example.com/repo/b" diff --git a/vulnfeeds/cmd/combine-to-osv/run_combine_to_osv_convert.sh b/vulnfeeds/cmd/combine-to-osv/run_combine_to_osv_convert.sh index 3330d96af00..4a0a4594a0d 100755 --- a/vulnfeeds/cmd/combine-to-osv/run_combine_to_osv_convert.sh +++ b/vulnfeeds/cmd/combine-to-osv/run_combine_to_osv_convert.sh @@ -21,26 +21,14 @@ OUTPUT_BUCKET="${OUTPUT_GCS_BUCKET:=cve-osv-conversion}" NUM_WORKERS="${NUM_WORKERS:=64}" OSV_OUTPUT="osv-output" -NVD_OSV_OUTPUT="nvd" -CVE5_OSV_OUTPUT="cve5" echo "Setup initial directories" -rm -rf $NVD_OSV_OUTPUT && mkdir -p $NVD_OSV_OUTPUT rm -rf $OSV_OUTPUT && mkdir -p $OSV_OUTPUT -rm -rf $CVE5_OSV_OUTPUT && mkdir -p $CVE5_OSV_OUTPUT - -echo "Begin syncing NVD data from GCS bucket ${INPUT_BUCKET}" -gcloud --no-user-output-enabled storage -q cp "gs://${INPUT_BUCKET}/nvd-osv/CVE-????-*.json" "${NVD_OSV_OUTPUT}" -echo "Successfully synced from GCS bucket" - -echo "Begin syncing CVE5 data from GCS bucket ${INPUT_BUCKET}" -gcloud --no-user-output-enabled storage -q cp "gs://${INPUT_BUCKET}/cve5/CVE-????-*.json" "${CVE5_OSV_OUTPUT}" -echo "Successfully synced from GCS bucket" echo "Run combine-to-osv" ./combine-to-osv \ - -cve5-path "$CVE5_OSV_OUTPUT" \ - -nvd-path "$NVD_OSV_OUTPUT" \ + -cve5-path "gs://${INPUT_BUCKET}/cve5/" \ + -nvd-path "gs://${INPUT_BUCKET}/nvd-osv/" \ -osv-output-path "$OSV_OUTPUT" \ -upload-to-gcs \ -output-bucket "${OUTPUT_BUCKET}" \ diff --git a/vulnfeeds/conversion/writer/writer.go b/vulnfeeds/conversion/writer/writer.go index f30fed98e7f..ce5826d66c6 100644 --- a/vulnfeeds/conversion/writer/writer.go +++ b/vulnfeeds/conversion/writer/writer.go @@ -256,7 +256,11 @@ func UploadVulnsToGCS( } if doDeletions { - HandleDeletion(ctx, outBkt, osvOutputPath, vulnerabilities) + validIDs := make([]string, len(vulnerabilities)) + for i, v := range vulnerabilities { + validIDs[i] = v.GetId() + } + HandleDeletion(ctx, outBkt, osvOutputPath, validIDs) } gcsHelper, err = gcs.InitUploadPool(ctx, numWorkers, outputBucketName) @@ -292,7 +296,7 @@ func UploadVulnsToGCS( } } -func HandleDeletion(ctx context.Context, outBkt *storage.BucketHandle, osvOutputPath string, vulnerabilities []*osvschema.Vulnerability) { +func HandleDeletion(ctx context.Context, outBkt *storage.BucketHandle, osvOutputPath string, validVulnIDs []string) { // Check if any need to be deleted bucketObjects, err := gcs.ListBucketObjects(ctx, outBkt, osvOutputPath) if err != nil { @@ -300,8 +304,8 @@ func HandleDeletion(ctx context.Context, outBkt *storage.BucketHandle, osvOutput return } vulnFilenames := make(map[string]bool) - for _, v := range vulnerabilities { - filename := v.GetId() + ".json" + for _, id := range validVulnIDs { + filename := id + ".json" filePath := path.Join(osvOutputPath, filename) vulnFilenames[filePath] = true } diff --git a/vulnfeeds/conversion/writer/writer_test.go b/vulnfeeds/conversion/writer/writer_test.go index 72fb09c4ca1..2eb29d2aa4e 100644 --- a/vulnfeeds/conversion/writer/writer_test.go +++ b/vulnfeeds/conversion/writer/writer_test.go @@ -334,12 +334,9 @@ func TestHandleDeletion(t *testing.T) { } w2.Close() - vulnerabilities := []*osvschema.Vulnerability{ - {Id: "CVE-2023-1111"}, - {Id: "CVE-2023-3333"}, - } + validIDs := []string{"CVE-2023-1111", "CVE-2023-3333"} - HandleDeletion(ctx, bkt, "", vulnerabilities) + HandleDeletion(ctx, bkt, "", validIDs) // CVE-2023-1111.json should still exist if _, err := bkt.Object("CVE-2023-1111.json").Attrs(ctx); err != nil { diff --git a/vulnfeeds/models/cve.go b/vulnfeeds/models/cve.go index 875896834dd..f2393c004a1 100644 --- a/vulnfeeds/models/cve.go +++ b/vulnfeeds/models/cve.go @@ -146,7 +146,7 @@ type CVE5 struct { func EnglishDescription(descriptions []LangString) string { for _, desc := range descriptions { - if desc.Lang == "en" { + if desc.Lang == "en" || strings.HasPrefix(desc.Lang, "en-") { return desc.Value } }