Feat/CVE correlation

api/app.py

@@ -61,6 +61,9 @@ def create_app() -> Flask:
     # ------------------------------------------------------------------ #
     # Database Management                                                   #
     # ------------------------------------------------------------------ #
+    with app.app_context():
+        db = DatabaseManager()
+        db.run_migrations()
 
     @app.teardown_appcontext
     def close_db(error=None):
@@ -162,7 +165,7 @@ def internal_error(exc):
         logger.error("Unhandled exception: %s", exc)
         return jsonify({"error": "Internal server error"}), 500
 
-    logger.info("OpenShield API created — %d blueprints registered", len(app.blueprints))
+    logger.info("OpenShield API created - %d blueprints registered", len(app.blueprints))
     return app
 
 

api/models/finding.py

@@ -42,6 +42,9 @@ class Finding:
     scan_id: Optional[str] = None
     playbook: Optional[str] = None
     metadata: Dict[str, Any] = field(default_factory=dict)
+    cve_references: List[Dict[str, Any]] = field(default_factory=list)
+    cvss_score: Optional[float] = None
+    exploit_available: bool = False
     id: Optional[int] = None
 
     def to_dict(self) -> Dict[str, Any]:
@@ -61,6 +64,9 @@ def to_dict(self) -> Dict[str, Any]:
             "scan_id": self.scan_id,
             "playbook": self.playbook,
             "metadata": self.metadata,
+            "cve_references": self.cve_references,
+            "cvss_score": self.cvss_score,
+            "exploit_available": self.exploit_available,
         }
 
 
@@ -140,6 +146,9 @@ def create_tables(self) -> None:
                     playbook        TEXT,
                     frameworks      JSONB,
                     metadata        JSONB,
+                    cve_references  JSONB DEFAULT '[]',
+                    cvss_score      FLOAT DEFAULT NULL,
+                    exploit_available BOOLEAN DEFAULT FALSE,
                     detected_at     TIMESTAMPTZ NOT NULL
                 );
             """)
@@ -154,6 +163,27 @@ def create_tables(self) -> None:
         conn.commit()
         logger.info("Database tables created / verified")
 
+    def run_migrations(self) -> None:
+        """Add CVE columns if they don't exist.
+        Safe to call on every startup - uses IF NOT EXISTS.
+        """
+        conn = self._get_conn()
+        try:
+            with conn.cursor() as cur:
+                # Ensure we are in the right schema
+                cur.execute("SET search_path TO openshield, public;")
+                cur.execute("""
+                    ALTER TABLE findings
+                        ADD COLUMN IF NOT EXISTS cve_references    JSONB   DEFAULT '[]',
+                        ADD COLUMN IF NOT EXISTS cvss_score        FLOAT   DEFAULT NULL,
+                        ADD COLUMN IF NOT EXISTS exploit_available BOOLEAN DEFAULT FALSE
+                """)
+            conn.commit()
+            logger.info("CVE migrations applied successfully")
+        except Exception as e:
+            logger.error("Failed to run CVE migrations: %s", e)
+            conn.rollback()
+
     # ------------------------------------------------------------------ #
     # Write                                                                 #
     # ------------------------------------------------------------------ #
@@ -183,8 +213,9 @@ def save_scan(self, scan_result: Dict[str, Any]) -> None:
                         (scan_id, rule_id, rule_name, severity, category,
                          resource_id, resource_name, resource_type,
                          description, remediation, playbook,
-                         frameworks, metadata, detected_at)
-                    VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
+                         frameworks, metadata, cve_references,
+                         cvss_score, exploit_available, detected_at)
+                    VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
                     """,
                     (
                         f.get("scan_id"),
@@ -200,6 +231,9 @@ def save_scan(self, scan_result: Dict[str, Any]) -> None:
                         f.get("playbook"),
                         json.dumps(f.get("frameworks", {})),
                         json.dumps(f.get("metadata", {})),
+                        json.dumps(f.get("cve_references", [])),
+                        f.get("cvss_score"),
+                        f.get("exploit_available", False),
                         f.get("detected_at"),
                     ),
                 )
@@ -257,7 +291,7 @@ def get_scans(self) -> List[Dict[str, Any]]:
     # ------------------------------------------------------------------ #
 
     def get_score(self) -> int:
-        """Return a 0–100 security posture score based on open findings.
+        """Return a 0-100 security posture score based on open findings.
 
         HIGH findings deduct 10 points each, MEDIUM 5, LOW 2.
         Score floors at 0.
@@ -274,6 +308,38 @@ def get_score(self) -> int:
         )
         return max(0, 100 - deduction)
 
+    def get_cve_summary(self) -> Dict[str, Any]:
+        """Return high-level summary of CVE findings for the dashboard."""
+        conn = self._get_conn()
+        with conn.cursor() as cur:
+            cur.execute("""
+                SELECT 
+                    COUNT(*) as total_findings,
+                    COUNT(CASE WHEN exploit_available = TRUE THEN 1 END) as exploit_count,
+                    MAX(cvss_score) as max_cvss_score,
+                    AVG(cvss_score) as avg_cvss_score,
+                    COUNT(CASE WHEN cvss_score >= 9.0 THEN 1 END) as critical_cve_count
+                FROM findings
+            """)
+            row = cur.fetchone()
+
+        if not row:
+            return {
+                "total_findings": 0,
+                "exploit_count": 0,
+                "max_cvss_score": None,
+                "avg_cvss_score": None,
+                "critical_cve_count": 0
+            }
+
+        return {
+            "total_findings": row[0],
+            "exploit_count": row[1],
+            "max_cvss_score": row[2],
+            "avg_cvss_score": round(row[3], 2) if row[3] is not None else None,
+            "critical_cve_count": row[4]
+        }
+
     def get_compliance_score(self, framework: str) -> Dict[str, Any]:
         """Return pass/fail breakdown against a compliance framework.
 

api/routes/findings.py

@@ -5,6 +5,7 @@
 from flask import Blueprint, g, jsonify, request
 
 from api.models.finding import DatabaseManager
+from scanner.cve_correlator import enrich_findings
 
 findings_bp = Blueprint("findings", __name__)
 logger = logging.getLogger(__name__)
@@ -22,10 +23,10 @@ def list_findings():
     """Return findings, optionally filtered by severity, category, or rule_id.
 
     Query parameters:
-        severity  — HIGH | MEDIUM | LOW | INFO
-        category  — Storage | Network | Identity | Database | Compute | KeyVault
-        rule_id   — e.g. AZ-STOR-001
-        scan_id   — UUID of a specific scan
+        severity  - HIGH | MEDIUM | LOW | INFO
+        category  - Storage | Network | Identity | Database | Compute | KeyVault
+        rule_id   - e.g. AZ-STOR-001
+        scan_id   - UUID of a specific scan
     """
     try:
         filters = {

api/routes/score.py

@@ -22,7 +22,7 @@ def _get_db() -> DatabaseManager:
 
 @score_bp.get("/api/score")
 def get_score():
-    """Return the overall security posture score (0–100).
+    """Return the overall security posture score (0-100).
 
     Score calculation:
         Starts at 100. Deducts 10 per HIGH finding, 5 per MEDIUM, 2 per LOW.
@@ -34,4 +34,16 @@ def get_score():
         return jsonify(result)
     except Exception as exc:
         logger.error("Failed to calculate score: %s", exc)
-        return jsonify({"error": "Failed to calculate score", "detail": str(exc)}), 500
+        return jsonify({"error": "Failed to calculate score", "detail": str(exc)}), 500
+
+
+@score_bp.get("/api/score/cve-summary")
+def get_cve_summary():
+    """Return high-level CVE summary for the dashboard."""
+    try:
+        db = _get_db()
+        result = db.get_cve_summary()
+        return jsonify(result)
+    except Exception as exc:
+        logger.error("Failed to fetch CVE summary: %s", exc)
+        return jsonify({"error": "Failed to fetch CVE summary", "detail": str(exc)}), 500

docs/cve_correlation_feature.md

@@ -0,0 +1,74 @@
+# OpenShield - CVE Correlation Feature Documentation
+
+## Overview
+
+The CVE Correlation feature integrates the MITRE National Vulnerability Database (NVD) API with the OpenShield scanner. It cross-references security misconfigurations discovered during scans with known Common Vulnerabilities and Exposures (CVEs), providing users with CVSS scores and exploit availability status.
+
+## Files Created and Modified
+
+### New Files (Core Logic)
+
+| File | Purpose |
+|---|---|
+| scanner/nvd_client.py | NVD API Integration. Handles low-level communication with MITRE NVD. Implements strict rate-limiting (7s gap), in-memory caching for performance, and exponential back-off for reliability. |
+| scanner/cve_correlator.py | Contextual Mapping. Maps OpenShield Rule IDs (e.g., AZ-STOR) to NVD search terms. Performs the logic of merging raw API results into finding objects. |
+| tests/test_nvd_client.py | Client Verification. Unit tests verifying parsing logic, 429 retry handling, and cache hits. |
+| tests/test_cve_correlator.py | Logic Verification. Unit tests ensuring Rule IDs map correctly and finding enrichment correctly identifies the highest risk. |
+
+### Modified Files (Integration)
+
+| File | Change | Why |
+|---|---|---|
+| scanner/engine.py | Enrichment-at-Source. Integrated enrich_findings directly into the scan lifecycle. | Performance: By enriching during the scan, CVE data is saved once to the database. The frontend does not have to wait for an NVD API call when loading the dashboard. |
+| api/models/finding.py | Updated Finding dataclass and added run_migrations and get_cve_summary. | Persistence: Adds cve_references, cvss_score, and exploit_available columns to PostgreSQL. get_cve_summary provides stats for dashboard widgets. |
+| api/app.py | Added db.run_migrations call at startup. | Auto-Deployment: Ensures the database schema is updated automatically on any environment where the app is launched. |
+| api/routes/score.py | Added GET /api/score/cve-summary endpoint. | Dashboard UI: Provides the frontend with high-level data like Total Known Exploits in a single lightweight request. |
+| api/routes/findings.py | Adjusted list_findings to return data from the database. | Clean API: Keeps the API response structure consistent while including the new enriched security data. |
+
+## Frontend Integration Design
+
+To ensure the frontend dashboard works perfectly, the architecture uses an Enrichment-at-Source model:
+
+1. Zero-Latency Dashboard Loads: The scan engine pre-enriches findings. When the frontend calls the API, it receives static data from the database. Response times are reduced from seconds to milliseconds.
+2. Dashboard-Ready Summary Endpoint: The /api/score/cve-summary endpoint allows the frontend to fetch high-level statistics (Total Findings, Exploit Count, Max CVSS) in one call instead of processing thousands of records locally.
+3. Actionable Risk (CISA KEV): The exploit_available flag uses the CISA Known Exploited Vulnerabilities catalogue, allowing the dashboard to highlight high-priority risks that are being exploited in the wild.
+4. Persistent Historical State: Enrichment happens at the time of scan, meaning the dashboard shows the CVE status as it existed on that day. This ensures accurate compliance and historical reporting.
+
+## Security and Compliance Audit
+
+1. No Hardcoded Secrets: All credentials (DATABASE_URL, JWT_SECRET) are handled via environment variables.
+2. SSRF Protection: NVD query parameters are sanitized and derived from internal static maps.
+3. SQL Safety: All database additions use parameterized queries to prevent injection.
+4. Character Quality: All non-ASCII characters and emojis were removed for pipeline compatibility.
+
+## Testing Strategy
+
+All logic is verified using the Python standard library unittest framework. All NVD HTTP calls are fully mocked to ensure stability.
+
+### Testing Rationale
+
+The 27 tests were selected to verify three critical areas of the API integration:
+
+1. Data Integrity (TestParseConveItem):
+   * Purpose: The NVD API response is deeply nested and contains multiple CVSS versions (v2, v3.0, v3.1).
+   * Rationale: We must guarantee the scanner always extracts the highest precision score available. We also verify description truncation to ensure unexpectedly long CVE descriptions do not exceed database column limits.
+
+2. System Stability (TestQueryNvd):
+   * Purpose: To prevent the scanner from being rate-limited or banned by MITRE.
+   * Rationale: We verify that the in-memory cache is used for repeated resource types. We also simulate 429 (Rate Limited) responses to confirm the exponential back-off logic works. Finally, we ensure that network failures return an empty list instead of raising exceptions, keeping the core scanner operational.
+
+3. Logic Correctness (TestGetNvdKeyword and TestEnrichFindings):
+   * Purpose: To verify the mapping engine and risk calculation.
+   * Rationale: We test the prefix-fallback mechanism to ensure the feature is future-proof for new rules. We also verify that when multiple CVEs match, the highest CVSS score is selected to highlight the maximum risk on the dashboard.
+
+4. Integration Safety (TestEnrichSingleFinding):
+   * Purpose: To ensure enrichment is non-destructive.
+   * Rationale: We verify that adding CVE data does not overwrite existing scanner fields like resource_id or base severity.
+
+### How to run the tests
+
+```bash
+python3 -m unittest tests/test_nvd_client.py tests/test_cve_correlator.py -v
+```
+
+Expected output: All tests passing, zero network calls made.