perf(debian-cve-convert): reduce memory usage and streamline Debian CVE converter

deployment/clouddeploy/gke-workers/base/feeds/debian-cve-convert.yaml

@@ -21,6 +21,15 @@ spec:
           - name: debian-cve-convert
             image: debian-cve-convert
             imagePullPolicy: Always
+            command: ["/root/debian"]
+            args:
+            - "--input-bucket=$(INPUT_GCS_BUCKET)"
+            - "--output-bucket=$(OUTPUT_GCS_BUCKET)"
+            - "--output-path=debian-cve-osv"
+            - "--cve-path=cve_jsons/"
+            - "--workers=$(NUM_WORKERS)"
+            - "--upload-to-gcs"
+            - "--sync-deletions"
             resources:
               requests:
                 cpu: "2"

vulnfeeds/cmd/converters/debian/Dockerfile

@@ -29,8 +29,5 @@ FROM gcr.io/google.com/cloudsdktool/google-cloud-cli:alpine@sha256:3b058be4e039d
 
 WORKDIR /root/
 COPY --from=GO_BUILD /src/debian ./
-COPY ./cmd/converters/debian/run_debian_convert.sh ./
+ENTRYPOINT ["/root/debian"]
 
-RUN chmod 755 ./run_debian_convert.sh
-
-ENTRYPOINT ["/root/run_debian_convert.sh"]

vulnfeeds/cmd/converters/debian/main.go

@@ -10,12 +10,15 @@ import (
 	"log/slog"
 	"net/http"
 	"os"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
 
+	"cloud.google.com/go/storage"
 	"github.com/google/osv/vulnfeeds/conversion/writer"
 	"github.com/google/osv/vulnfeeds/faulttolerant"
+	gcs "github.com/google/osv/vulnfeeds/gcs-tools"
 	"github.com/google/osv/vulnfeeds/models"
 	"github.com/google/osv/vulnfeeds/utility/logger"
 	"github.com/google/osv/vulnfeeds/vulns"
@@ -35,26 +38,59 @@ func main() {
 	logger.InitGlobalLogger()
 	defer logger.Close()
 
+	cvePath := flag.String("cve-path", defaultCvePath, "Path to CVE JSON files.")
+	inputBucketName := flag.String("input-bucket", "", "The GCS bucket to download NVD CVE data from. If set, downloads data before processing.")
+
 	debianOutputPath := flag.String("output-path", debianOutputPathDefault, "Path to output OSV files.")
 	outputBucketName := flag.String("output-bucket", outputBucketDefault, "The GCS bucket to write to.")
 	numWorkers := flag.Int("workers", 64, "Number of workers to process records")
-	uploadToGCS := flag.Bool("upload-to-gcs", false, "If true, do not write to GCS bucket and instead write to local disk.")
+	uploadToGCS := flag.Bool("upload-to-gcs", false, "If true, write to GCS bucket.")
 	syncDeletions := flag.Bool("sync-deletions", false, "If false, do not delete files in bucket that are not local")
 	flag.Parse()
 
+	ctx := context.Background()
+
+	if *inputBucketName != "" {
+		logger.Info("Downloading NVD CVE data from GCS bucket", slog.String("bucket", *inputBucketName), slog.String("dest", *cvePath))
+		storageClient, err := storage.NewClient(ctx)
+		if err != nil {
+			logger.Fatal("Failed to create GCS client", slog.Any("err", err))
+		}
+		defer storageClient.Close()
+
+		bkt := storageClient.Bucket(*inputBucketName)
+		// TODO: It's inefficient to write all of this to a folder and then read it from the folder again.
+		// It can just be 1 operation, and nothing should need to touch disk.
+		if err := gcs.DownloadBucket(ctx, bkt, "nvd/", *cvePath); err != nil {
+			logger.Fatal("Failed to download NVD CVE data from GCS", slog.Any("err", err))
+		}
+
+		logger.Info("Successfully downloaded NVD CVE data from GCS")
+	}
+
 	err := os.MkdirAll(*debianOutputPath, 0755)
 	if err != nil {
 		logger.Fatal("Can't create output path", slog.Any("err", err))
 	}
 
+	vulnerabilities, err := buildDebianVulnerabilities(*cvePath)
+	if err != nil {
+		logger.Fatal("Failed to build Debian vulnerabilities", slog.Any("err", err))
+	}
+	runtime.GC()
+	writer.UploadVulnsToGCS(ctx, "Debian CVEs", *uploadToGCS, *outputBucketName, "", *numWorkers, *debianOutputPath, vulnerabilities, *syncDeletions)
+	logger.Info("Debian CVE conversion succeeded.")
+}
+
+func buildDebianVulnerabilities(cvePath string) ([]*osvschema.Vulnerability, error) {
 	debianData, err := downloadDebianSecurityTracker()
 	if err != nil {
-		logger.Fatal("Failed to download/parse Debian Security Tracker json file", slog.Any("err", err))
+		return nil, fmt.Errorf("failed to download/parse Debian Security Tracker json file: %w", err)
 	}
 
 	debianReleaseMap, err := getDebianReleaseMap()
 	if err != nil {
-		logger.Fatal("Failed to get Debian distro info data", slog.Any("err", err))
+		return nil, fmt.Errorf("failed to get Debian distro info data: %w", err)
 	}
 
 	targetCVEs := make(map[string]bool)
@@ -66,7 +102,7 @@ func main() {
 		}
 	}
 
-	allCVEs := vulns.LoadTargetCVEs(defaultCvePath, targetCVEs)
+	allCVEs := vulns.LoadTargetCVEMetadata(cvePath, targetCVEs)
 	osvCVEs := generateOSVFromDebianTracker(debianData, debianReleaseMap, allCVEs)
 
 	vulnerabilities := make([]*osvschema.Vulnerability, 0, len(osvCVEs))
@@ -78,13 +114,11 @@ func main() {
 		vulnerabilities = append(vulnerabilities, v.Vulnerability)
 	}
 
-	ctx := context.Background()
-	writer.UploadVulnsToGCS(ctx, "Debian CVEs", *uploadToGCS, *outputBucketName, "", *numWorkers, *debianOutputPath, vulnerabilities, *syncDeletions)
-	logger.Info("Debian CVE conversion succeeded.")
+	return vulnerabilities, nil
 }
 
 // generateOSVFromDebianTracker converts Debian Security Tracker entries to OSV format.
-func generateOSVFromDebianTracker(debianData DebianSecurityTrackerData, debianReleaseMap map[string]string, allCVEs map[models.CVEID]models.Vulnerability) map[string]*vulns.Vulnerability {
+func generateOSVFromDebianTracker(debianData DebianSecurityTrackerData, debianReleaseMap map[string]string, allCVEs map[models.CVEID]vulns.VulnerabilityMetadata) map[string]*vulns.Vulnerability {
 	logger.Info("Converting Debian Security Tracker data to OSV.")
 	osvCves := make(map[string]*vulns.Vulnerability)
 
@@ -132,12 +166,18 @@ func generateOSVFromDebianTracker(debianData DebianSecurityTrackerData, debianRe
 					},
 				}
 
-				if !currentNVDCVE.CVE.Published.IsZero() {
-					v.Published = timestamppb.New(currentNVDCVE.CVE.Published.Time)
+				if !currentNVDCVE.Published.IsZero() {
+					v.Published = timestamppb.New(currentNVDCVE.Published)
+				}
+
+				if !currentNVDCVE.Modified.IsZero() {
+					v.Modified = timestamppb.New(currentNVDCVE.Modified)
+				} else if !currentNVDCVE.Published.IsZero() {
+					v.Modified = timestamppb.New(currentNVDCVE.Published)
 				}
 
-				if currentNVDCVE.CVE.Metrics != nil {
-					v.AddSeverity(currentNVDCVE.CVE.Metrics)
+				if currentNVDCVE.Metrics != nil {
+					v.AddSeverity(currentNVDCVE.Metrics)
 				}
 
 				osvCves[cveID] = v

vulnfeeds/cmd/converters/debian/main_test.go

@@ -38,26 +38,31 @@ func sortAffected(affected []*osvschema.Affected) {
 	})
 }
 
-func loadTestData(t *testing.T, cveName string) models.Vulnerability {
+func loadTestMetadata(t *testing.T, cveName string) vulns.VulnerabilityMetadata {
 	t.Helper()
 	fileName := fmt.Sprintf("../../../test_data/nvdcve-2.0/%s.json", cveName)
 	file, err := os.Open(fileName)
 	if err != nil {
 		t.Fatalf("Failed to load test data from %q: %#v", fileName, err)
 	}
+	defer file.Close()
 	var nvdCves models.CVEAPIJSON20Schema
 	err = json.NewDecoder(file).Decode(&nvdCves)
 	if err != nil {
 		t.Fatalf("Failed to decode %q: %+v", fileName, err)
 	}
 	for _, vulnerability := range nvdCves.Vulnerabilities {
 		if string(vulnerability.CVE.ID) == cveName {
-			return vulnerability
+			return vulns.VulnerabilityMetadata{
+				Published: vulnerability.CVE.Published.Time,
+				Modified:  vulnerability.CVE.LastModified.Time,
+				Metrics:   vulnerability.CVE.Metrics,
+			}
 		}
 	}
 	t.Fatalf("test data doesn't contain %q", cveName)
 
-	return models.Vulnerability{}
+	return vulns.VulnerabilityMetadata{}
 }
 
 func TestGenerateOSVFromDebianTracker(t *testing.T) {
@@ -77,10 +82,10 @@ func TestGenerateOSVFromDebianTracker(t *testing.T) {
 		"bookworm": "12",
 		"trixie":   "13",
 	}
-	cveStuff := map[models.CVEID]models.Vulnerability{
-		"CVE-2014-1424": loadTestData(t, "CVE-2014-1424"),
-		"CVE-2017-6507": loadTestData(t, "CVE-2017-6507"),
-		"CVE-2016-1585": loadTestData(t, "CVE-2016-1585"),
+	cveStuff := map[models.CVEID]vulns.VulnerabilityMetadata{
+		"CVE-2014-1424": loadTestMetadata(t, "CVE-2014-1424"),
+		"CVE-2017-6507": loadTestMetadata(t, "CVE-2017-6507"),
+		"CVE-2016-1585": loadTestMetadata(t, "CVE-2016-1585"),
 	}
 	got := generateOSVFromDebianTracker(trackerData, releaseMap, cveStuff)
 

vulnfeeds/vulns/vulns.go

@@ -21,14 +21,17 @@ import (
 	"encoding/json"
 	"errors"
 	"io"
+	"io/fs"
 	"log/slog"
 	"maps"
 	"net/url"
 	"os"
 	"path"
+	"path/filepath"
 	"slices"
 	"strings"
 	"sync"
+	"time"
 
 	goccyyaml "github.com/goccy/go-yaml"
 	"google.golang.org/protobuf/encoding/protojson"
@@ -863,6 +866,98 @@ func CheckQuality(text string) QualityCheck {
 	return Success
 }
 
+// VulnerabilityMetadata contains lightweight metadata (publication time, last modified time, and severity metrics) for a CVE.
+type VulnerabilityMetadata struct {
+	Published time.Time
+	Modified  time.Time
+	Metrics   *models.CVEItemMetrics
+}
+
+type lightNVDItem struct {
+	CVE struct {
+		ID           models.CVEID           `json:"id"`
+		Published    models.NVDTime         `json:"published"`
+		LastModified models.NVDTime         `json:"lastModified"`
+		Metrics      *models.CVEItemMetrics `json:"metrics"`
+	} `json:"cve"`
+}
+
+type lightNVDSchema struct {
+	Vulnerabilities []lightNVDItem `json:"vulnerabilities"`
+}
+
+// LoadTargetCVEMetadata loads publication dates, last modified dates, and severity metrics for target CVE IDs,
+// avoiding memory-heavy decoding of CPE match configurations, descriptions, and references.
+func LoadTargetCVEMetadata(cvePath string, targetCVEs map[string]bool) map[models.CVEID]VulnerabilityMetadata {
+	type cveMeta struct {
+		id   models.CVEID
+		meta VulnerabilityMetadata
+	}
+
+	metaChan := make(chan cveMeta)
+	var wg sync.WaitGroup
+
+	err := filepath.WalkDir(cvePath, func(filePath string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if d.IsDir() || !strings.HasSuffix(d.Name(), ".json") {
+			return nil
+		}
+
+		// Spawning one goroutine per file is acceptable here because the number of files
+		// is small (typically one per year of CVE data).
+		wg.Add(1)
+		go func(p string) {
+			defer wg.Done()
+			file, err := os.Open(p)
+			if err != nil {
+				logger.Error("Failed to open CVE JSON", slog.String("path", p), slog.Any("err", err))
+				return
+			}
+			defer file.Close()
+
+			var nvdcve lightNVDSchema
+			if err := json.NewDecoder(file).Decode(&nvdcve); err != nil {
+				logger.Error("Failed to decode JSON", slog.String("file", p), slog.Any("err", err))
+				return
+			}
+
+			for _, item := range nvdcve.Vulnerabilities {
+				if targetCVEs == nil || targetCVEs[string(item.CVE.ID)] {
+					metaChan <- cveMeta{
+						id: item.CVE.ID,
+						meta: VulnerabilityMetadata{
+							Published: item.CVE.Published.Time,
+							Modified:  item.CVE.LastModified.Time,
+							Metrics:   item.CVE.Metrics,
+						},
+					}
+				}
+			}
+			logger.Info("Loaded "+filepath.Base(p), slog.String("cve", filepath.Base(p)))
+		}(filePath)
+
+		return nil
+	})
+
+	if err != nil {
+		logger.Fatal("Failed to walk CVE path", slog.String("path", cvePath), slog.Any("err", err))
+	}
+
+	go func() {
+		wg.Wait()
+		close(metaChan)
+	}()
+
+	result := make(map[models.CVEID]VulnerabilityMetadata)
+	for item := range metaChan {
+		result[item.id] = item.meta
+	}
+
+	return result
+}
+
 // LoadAllCVEs loads the downloaded CVE's from the NVD database into memory.
 func LoadAllCVEs(cvePath string) map[models.CVEID]models.Vulnerability {
 	return LoadTargetCVEs(cvePath, nil)