Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions tests/test_artifact_reproducibility.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
from __future__ import annotations

import json
from collections.abc import Callable
from dataclasses import dataclass
from pathlib import Path

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

Import pytest and define REPO_ROOT to allow for robust path resolution relative to the repository structure, ensuring tests are runnable from any directory.

import pytest

REPO_ROOT = Path(__file__).resolve().parents[1]

from scripts.generate_layered_admissibility_artifact import (
generate_layered_admissibility_artifact,
)


@dataclass(frozen=True)
class DeterministicArtifactSpec:
"""Configuration for a reproducible committed artifact check."""

name: str
committed_path: Path
regenerate: Callable[[Path], Path]


ARTIFACT_SPECS: tuple[DeterministicArtifactSpec, ...] = (
DeterministicArtifactSpec(
name="layered_admissibility_results",
committed_path=Path("artifacts/layered_admissibility_results.json"),
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

Use REPO_ROOT to anchor the artifact path. This prevents the test from failing if executed from a directory other than the repository root.

Suggested change
committed_path=Path("artifacts/layered_admissibility_results.json"),
committed_path=REPO_ROOT / "artifacts/layered_admissibility_results.json",

regenerate=generate_layered_admissibility_artifact,
),
)


def _load_json(path: Path) -> object:
return json.loads(path.read_text(encoding="utf-8"))


def test_committed_deterministic_artifacts_are_reproducible(tmp_path: Path) -> None:
for spec in ARTIFACT_SPECS:
regenerated_path = tmp_path / spec.committed_path.name
spec.regenerate(regenerated_path)

regenerated_payload = _load_json(regenerated_path)
committed_payload = _load_json(spec.committed_path)

assert regenerated_payload == committed_payload, (
f"Deterministic artifact drift detected for '{spec.name}'. "
f"Regenerate with: python scripts/generate_layered_admissibility_artifact.py"
)
Comment on lines +35 to +46
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

Refactor the test to use pytest.mark.parametrize. This improves test isolation and provides clearer reporting by treating each artifact as a separate test case, which aligns with the extensible design mentioned in the PR description.

Suggested change
def test_committed_deterministic_artifacts_are_reproducible(tmp_path: Path) -> None:
for spec in ARTIFACT_SPECS:
regenerated_path = tmp_path / spec.committed_path.name
spec.regenerate(regenerated_path)
regenerated_payload = _load_json(regenerated_path)
committed_payload = _load_json(spec.committed_path)
assert regenerated_payload == committed_payload, (
f"Deterministic artifact drift detected for '{spec.name}'. "
f"Regenerate with: python scripts/generate_layered_admissibility_artifact.py"
)
@pytest.mark.parametrize("spec", ARTIFACT_SPECS, ids=lambda s: s.name)
def test_committed_deterministic_artifacts_are_reproducible(spec: DeterministicArtifactSpec, tmp_path: Path) -> None:
regenerated_path = tmp_path / spec.committed_path.name
spec.regenerate(regenerated_path)
regenerated_payload = _load_json(regenerated_path)
committed_payload = _load_json(spec.committed_path)
assert regenerated_payload == committed_payload, (
f"Deterministic artifact drift detected for '{spec.name}'. "
f"Regenerate with: python scripts/generate_layered_admissibility_artifact.py"
)

Loading