Add evidence-checker extension

artefact_enumerator.py

@@ -359,6 +359,23 @@ def _process_compliance_snapshot_of_artefact(
         if uncommitted_backlog_item:
             uncommitted_backlog_items.append(uncommitted_backlog_item)
 
+    if (
+        extensions_cfg.test_results
+        and extensions_cfg.test_results.enabled
+        and extensions_cfg.test_results.is_supported(artefact_kind=artefact.artefact_kind)
+    ):
+        compliance_snapshot, uncommitted_backlog_item = _create_backlog_item_for_extension(
+            finding_cfgs=finding_cfgs,
+            finding_types=(odg.model.Datatype.TEST_RESULT_FINDING,),
+            artefact=artefact,
+            compliance_snapshot=compliance_snapshot,
+            service=odg.extensions_cfg.Services.TEST_RESULT_FINDING,
+            interval_seconds=extensions_cfg.osid.interval,
+            now=now,
+        )
+        if uncommitted_backlog_item:
+            uncommitted_backlog_items.append(uncommitted_backlog_item)
+
     if (
         extensions_cfg.osid
         and extensions_cfg.osid.enabled

charts/extensions/Chart.yaml

@@ -44,3 +44,6 @@ dependencies:
   - name: odg-operator
     repository: file://charts/odg-operator
     condition: odg-operator.enabled
+  - name: test-evidence
+    repository: file://charts/test-evidence
+    condition: test-evidence.enabled

charts/extensions/charts/test-evidence/templates/test-results.yaml

@@ -0,0 +1,123 @@
+{{- $podName := "test-evidence" }}
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ $podName }}
+  namespace: {{ .Values.target_namespace | default .Release.Namespace }}
+  {{- if default dict (.Values.deployment).annotations }}
+  annotations:
+  {{- range $annotation, $value := .Values.deployment.annotations }}
+    {{ $annotation }}: {{ $value }}
+  {{- end }}
+  {{- end }}
+spec:
+  replicas: 0 # will be scaled automatically by backlog-controller
+  selector:
+    matchLabels:
+      app: {{ $podName }}
+      delivery-gear.gardener.cloud/service: testEvidence
+  template:
+    metadata:
+      labels:
+        app: {{ $podName }}
+        delivery-gear.gardener.cloud/service: testEvidence
+    spec:
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: ScheduleAnyway
+          labelSelector:
+            matchLabels:
+              app: {{ $podName }}
+      terminationGracePeriodSeconds: 300 # 5 min
+      containers:
+        - name: {{ $podName }}
+          image: {{ include "image" .Values.image }}
+          imagePullPolicy: IfNotPresent
+          command:
+            - python3
+            - -m
+            - test_evidence_extension
+          securityContext:
+            allowPrivilegeEscalation: false
+          env:
+            - name: SECRET_FACTORY_PATH
+              value: /secrets
+            - name: EXTENSIONS_CFG_PATH
+              value: /extensions_cfg/extensions_cfg
+            - name: FINDINGS_CFG_PATH
+              value: /findings_cfg/findings_cfg
+            - name: OCM_REPO_MAPPINGS_PATH
+              value: /ocm_repo_mappings/ocm_repo_mappings
+            - name: K8S_TARGET_NAMESPACE
+              value: {{ .Values.target_namespace | default .Release.Namespace }}
+          volumeMounts:
+            - name: github
+              mountPath: /secrets/github
+            - name: github-app
+              mountPath: /secrets/github-app
+            - name: kubernetes
+              mountPath: /secrets/kubernetes
+            - name: oci-registry
+              mountPath: /secrets/oci-registry
+            - name: extensions-cfg
+              mountPath: /extensions_cfg
+            - name: findings-cfg
+              mountPath: /findings_cfg
+            - name: ocm-repo-mappings
+              mountPath: /ocm_repo_mappings
+              readOnly: true
+          lifecycle:
+            preStop: # hook ensures that just created pods have at least enough time alive to add a termination signal handler
+              exec:
+                command:
+                  - sleep
+                  - "60"
+          resources:
+            requests:
+              memory: 100Mi
+              cpu: 250m
+            limits:
+              memory: 300Mi
+              cpu: 500m
+      volumes:
+        - name: github
+          secret:
+            secretName: secret-factory-github
+            optional: true
+        - name: github-app
+          secret:
+            secretName: secret-factory-github-app
+            optional: true
+        - name: kubernetes
+          secret:
+            secretName: secret-factory-kubernetes
+            optional: true # might use incluster config
+        - name: oci-registry
+          secret:
+            secretName: secret-factory-oci-registry
+            optional: true # oci authentication is optional
+        - name: extensions-cfg
+          configMap:
+            name: extensions-cfg
+        - name: findings-cfg
+          configMap:
+            name: findings-cfg
+        - name: ocm-repo-mappings
+          configMap:
+            name: ocm-repo-mappings
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: allow-egress-from-test-evidence
+  namespace: {{ .Values.target_namespace | default .Release.Namespace }}
+spec:
+  podSelector:
+    matchLabels:
+      app: {{ $podName }}
+  policyTypes:
+    - Egress
+  egress:
+    - {}  # Allows all egress traffic to any destination and port

charts/extensions/values.yaml

@@ -88,3 +88,10 @@ sast:
   image:
     repository: null
     tag: null
+test-evidence:
+  deployment:
+    annotations: []
+  enabled: false
+  image:
+    repository: null
+    tag: null

docs/extensions/evidence_checker.rst

@@ -0,0 +1,64 @@
+Evidence Checker Extension
+==========================
+
+Purpose
+-------
+
+Tests are a crucial part of software development, making sure the software's functionality stays intact despite the introduction of new features and enhancements. To prove the software's stability it is therefore often required to document and store test results of unit tests.
+
+Therefore, the aim of the evidence checker extension is to create a finding for an artefact/resource in case no test evidences from unit tests have been found for this artefact.
+
+Labels used for the extension
+-----------------------------
+
+The extension uses three labels in your component descriptor as described below.
+
+1. Purpose Label
+~~~~~~~~~~~~~~~~
+
+The purpose label defines the purpose of the artefact respectively what the artefact is used for. The value of the purpose label could be such as 'lint', 'sast' or 'test'. E.g., if the value is 'test' we know that this artefact is in fact a test evidence.
+
+.. code-block:: yaml
+
+    - name: gardener.cloud/purposes
+      signing: false
+      value:
+        - test
+
+2. Test Policy Label
+~~~~~~~~~~~~~~~~~~~~
+
+The test policy label defines whether an artefact requires unit tests - and therefore test evidences - or not. E.g., a helmchart does not require unit tests, while an OCI-image would require unit tests.
+
+.. code-block:: yaml
+
+   - name: gardener.cloud/test-policy
+     value: false / true
+
+3. Test Scope Label
+~~~~~~~~~~~~~~~~~~~
+
+The test scope label is set in the component descriptor of a test artefact (aka an artefact whose purpose label has the value 'test) and it defines for which other artefacts it represents the test evidence.
+
+If you do not specify any value in this label it is automatically assumed that the test artefact is valid for all artefacts within a component.
+
+.. code-block:: yaml
+
+    - name: gardener.cloud/test-scope
+              value:
+                - artefact-1
+                - artefact-2
+
+Functionality
+-------------
+
+1. The extension scans each artefact of a component and first validates whether the currently scanned artefact is a test evidence itself or not. This is achieved with the use of the gardener.cloud/purposes label. An artefact is considered a test evidence provided the label's value is 'test'.
+
+2. Provided the gardener.cloud/purposes label's has NOT been given the value 'test' the extension further checks, whether the artefact requires a test or not. This is achieved with the gardener.cloud/test-policy label which is either 'true' or 'false.
+
+3. In case an artefact has been identified as a test artefact (as described in step 1) the extension further checks, for which artefacts within the component the test-evidence is valid for. This can be defined with the help of the gardener.cloud/test-scope label.
+
+4. In case one of the artefacts identified in step 2 is not covered by the test artefacts(aka evidences) a finding will be created and test-evidences will have to be provided retrospectively.
+
+.. image:: evidence-checker.jpg
+   :alt: Evidence Checker Flow

odg/extensions_cfg.py

@@ -43,6 +43,7 @@ class Services(enum.StrEnum):
     PPMS = 'ppms'
     RESPONSIBLES = 'responsibles'
     SAST = 'sast'
+    TEST_EVIDENCE = 'testEvidence'
     ODG_OPERATOR = 'odg-operator'
     SBOM_GENERATOR = 'sbomGenerator'
 
@@ -984,6 +985,24 @@ def is_supported(
         return True
 
 
+@dataclasses.dataclass(kw_only=True)
+class TestEvidenceConfig(BacklogItemMixins):
+    service: Services = Services.TEST_EVIDENCE
+    delivery_service_url: str
+    external_artefacts_require_tests: bool
+    interval: int = 60 * 60 * 24 #
+    on_unsupported: WarningVerbosities = WarningVerbosities.WARNING
+
+    def is_supported(
+        self,
+        artefact_kind: odg.model.ArtefactKind,
+    ) -> bool:
+        if artefact_kind is not odg.model.ArtefactKind.RESOURCE:
+            return False
+
+        return True
+
+
 @dataclasses.dataclass(kw_only=True)
 class OsId(BacklogItemMixins):
     '''
@@ -1119,6 +1138,7 @@ class ExtensionsConfiguration:
     responsibles: ResponsiblesConfig | None
     sast: SASTConfig | None
     sbom_generator: SBOMGeneratorConfig | None
+    test_evidence: TestEvidenceConfig | None
     backlog_controller: BacklogControllerConfig = dataclasses.field(default_factory=BacklogControllerConfig) # noqa: E501
 
     @staticmethod

odg/extensions_cfg.yaml

@@ -105,3 +105,6 @@ sbom_generator:
   mappings:
    - prefix: ''
      group_id: 0 # <int> must be set
+
+test_evidence:
+  external_artefacts_require_tests: False

odg/findings.py

@@ -135,6 +135,15 @@ class SASTFindingSelector:
     sub_types: list[str]
 
 
+@dataclasses.dataclass
+class TestEvidenceFindingSelector:
+    '''
+    :param list[str] status:
+        List of regexes to determine matching test-evidence statuses.
+    '''
+    status: list[str]
+
+
 @dataclasses.dataclass
 class VulnerabilityFindingSelector:
     '''
@@ -195,6 +204,7 @@ class FindingCategorisation:
         | SASTFindingSelector
         | VulnerabilityFindingSelector
         | OsIdFindingSelector
+        | TestEvidenceFindingSelector
         | None
     )
 
@@ -557,6 +567,8 @@ def _validate(self):
                 self._validate_malware()
             case odg.model.Datatype.SAST_FINDING:
                 self._validate_sast()
+            case odg.model.Datatype.TEST_EVIDENCE_FINDING:
+                self._validate_test_result()
             case odg.model.Datatype.VULNERABILITY_FINDING:
                 self._validate_vulnerabilty()
             case odg.model.Datatype.INVENTORY_FINDING:
@@ -647,6 +659,18 @@ def _validate_malware(self):
         e.add_note('\n'.join(violations))
         raise e
 
+    def _validate_test_result(self):
+        violations = self._validate_categorisations(
+            expected_selector=TestEvidenceFindingSelector,
+        )
+
+        if not violations:
+            return
+
+        e = ModelValidationError('test evidence violations found:')
+        e.add_note('\n'.join(violations))
+        raise e
+
     def _validate_sast(self):
         violations = self._validate_categorisations(
             expected_selector=SASTFindingSelector,
@@ -930,3 +954,8 @@ def categorise_finding(
             for status in selector.status:
                 if re.fullmatch(status, finding_property, re.IGNORECASE):
                     return categorisation
+
+        elif isinstance(selector, TestEvidenceFindingSelector):
+            for status in selector.status:
+                if re.fullmatch(status, finding_property, re.IGNORECASE):
+                    return categorisation

odg/findings_cfg.yaml

@@ -46,6 +46,25 @@
         malware_names:
           - .*
 
+- type: finding/test-evidence
+  issues:
+    enable_assignees: False
+  categorisations:
+    - id: NONE
+      display_name: No Tests required
+      value: 0
+      allowed_processing_time: ~
+      rescoring: manual
+
+    - id: BLOCKER
+      display_name: No Test Evidence
+      value: 16
+      allowed_processing_time: 0
+      rescoring: manual
+      selector:
+        status:
+          - .*
+
 - type: finding/sast
   issues:
     enable_issues: False

odg/labels.py

@@ -76,6 +76,18 @@ class CveCategorisationLabel(Label):
     value: odg.cvss.CveCategorisation
 
 
+@dataclasses.dataclass(frozen=True)
+class TestScopeLabel(Label):
+    name = 'gardener.cloud/test-scope'
+    value: str # artefact-name
+
+
+@dataclasses.dataclass(frozen=True)
+class TestPolicyLabel(Label):
+    name = 'gardener.cloud/test-policy'
+    value: bool # is test required
+
+
 @functools.cache
 def _label_to_type() -> dict[str, Label]:
     own_module = sys.modules[__name__]