diff --git a/src/agent_rules_kit/cli.py b/src/agent_rules_kit/cli.py index ee604c0..6670ce7 100644 --- a/src/agent_rules_kit/cli.py +++ b/src/agent_rules_kit/cli.py @@ -10,6 +10,8 @@ from agent_rules_kit import __version__ from agent_rules_kit.discovery import InstructionFile, discover_instruction_files +from agent_rules_kit.findings import Finding +from agent_rules_kit.governance import find_unsupported_claim_findings from agent_rules_kit.init_plan import InitPlan, build_init_plan from agent_rules_kit.init_write import InitWriteResult, write_init_files from agent_rules_kit.redaction import redact_secret_like_values @@ -111,14 +113,20 @@ def _run_check(repository_root: Path, *, output_format: str = "console") -> int: return 2 status = "ok" if instruction_files else "no_instruction_files" - payload = _build_check_payload(repository_root, instruction_files, status=status) + findings = find_unsupported_claim_findings(repository_root, instruction_files) + payload = _build_check_payload( + repository_root, + instruction_files, + findings=findings, + status=status, + ) if output_format == "json": _print_json(payload) elif output_format == "markdown": _print_markdown(payload) else: - return _print_console_check(repository_root, instruction_files) + return _print_console_check(repository_root, instruction_files, findings) return 0 if instruction_files else 1 @@ -126,6 +134,7 @@ def _run_check(repository_root: Path, *, output_format: str = "console") -> int: def _print_console_check( repository_root: Path, instruction_files: tuple[InstructionFile, ...], + findings: tuple[Finding, ...], ) -> int: print(f"agent-rules-kit check: {repository_root}") @@ -137,6 +146,15 @@ def _print_console_check( for instruction_file in instruction_files: print(f"- {instruction_file.path} [{instruction_file.kind.value}]") + if findings: + print("Findings:") + for finding in findings: + location = _format_finding_location(finding) + print( + f"- {finding.rule_id} [{finding.severity.value}] " + f"{location} - {redact_secret_like_values(finding.message)}" + ) + return 0 @@ -193,6 +211,7 @@ def _build_check_payload( repository_root: Path, instruction_files: tuple[InstructionFile, ...], *, + findings: tuple[Finding, ...], status: str, ) -> dict[str, object]: return { @@ -208,7 +227,9 @@ def _build_check_payload( ], "summary": { "supported_instruction_file_count": len(instruction_files), + "finding_count": len(findings), }, + "findings": [_build_finding_payload(finding) for finding in findings], "error": None, } @@ -224,7 +245,9 @@ def _build_check_error_payload( "instruction_files": [], "summary": { "supported_instruction_file_count": 0, + "finding_count": 0, }, + "findings": [], "error": { "message": redact_secret_like_values(str(error)), }, @@ -244,6 +267,7 @@ def _print_markdown(payload: dict[str, object]) -> None: "- Supported instruction files: " f"{payload['summary']['supported_instruction_file_count']}" ) + print(f"- Findings: {payload['summary']['finding_count']}") error = payload["error"] if error is not None: @@ -265,6 +289,51 @@ def _print_markdown(payload: dict[str, object]) -> None: kind = _markdown_value(str(instruction_file["kind"])) print(f"| {path} | {kind} |") + findings = payload["findings"] + if findings: + print() + print("## Findings") + print() + print("| Rule | Severity | Location | Message |") + print("| --- | --- | --- | --- |") + for finding in findings: + rule_id = _markdown_value(str(finding["rule_id"])) + severity = _markdown_value(str(finding["severity"])) + location = _markdown_value(_format_finding_payload_location(finding)) + message = _markdown_value(str(finding["message"])) + print(f"| {rule_id} | {severity} | {location} | {message} |") + + +def _build_finding_payload(finding: Finding) -> dict[str, str | int]: + payload = finding.to_dict() + + if "message" in payload: + payload["message"] = redact_secret_like_values(str(payload["message"])) + if "path" in payload: + payload["path"] = redact_secret_like_values(str(payload["path"])) + + return payload + + +def _format_finding_location(finding: Finding) -> str: + if finding.path is None: + return "repository" + if finding.line is None: + return redact_secret_like_values(finding.path) + return f"{redact_secret_like_values(finding.path)}:{finding.line}" + + +def _format_finding_payload_location(finding: dict[str, object]) -> str: + path_value = finding.get("path") + if path_value is None: + return "repository" + + line_value = finding.get("line") + if line_value is None: + return str(path_value) + + return f"{path_value}:{line_value}" + def _markdown_value(value: str) -> str: return redact_secret_like_values(value).replace("|", "\\|").replace("\n", " ") diff --git a/src/agent_rules_kit/governance.py b/src/agent_rules_kit/governance.py new file mode 100644 index 0000000..60e9b27 --- /dev/null +++ b/src/agent_rules_kit/governance.py @@ -0,0 +1,102 @@ +"""Instruction governance diagnostics.""" + +from __future__ import annotations + +import re +from pathlib import Path +from re import Pattern + +from agent_rules_kit.discovery import InstructionFile +from agent_rules_kit.findings import Finding, Severity + +UNSUPPORTED_CLAIM_RULE_ID = "AIRK-GOV006" +UNSUPPORTED_CLAIM_MESSAGE = ( + "Instruction file may contain an unsupported security or maturity claim." +) + +UNSUPPORTED_CLAIM_PATTERNS: tuple[Pattern[str], ...] = ( + re.compile(r"\bguarantee[sd]?\s+(security|safety)\b", re.IGNORECASE), + re.compile(r"\bguaranteed\s+(secure|safe|security|safety)\b", re.IGNORECASE), + re.compile( + r"\bmake[s]?\s+(the\s+)?(repository|repo|project|tool)\s+(secure|safe)\b", + re.IGNORECASE, + ), + re.compile(r"\bcomplete\s+secret\s+scann(?:er|ing)\b", re.IGNORECASE), + re.compile(r"\bproduction[- ]ready\b", re.IGNORECASE), + re.compile(r"\benterprise[- ]grade\b", re.IGNORECASE), +) + +NEGATED_UNSUPPORTED_CLAIM_CONTEXT_PATTERNS: tuple[Pattern[str], ...] = ( + re.compile( + r"\b(do not|don't|must not|should not|never|avoid|forbid|forbidden|no)\b" + r".{0,120}\b(" + r"claim[s]?|guarantee[sd]?|security|safety|secure|safe|" + r"production[- ]ready|enterprise[- ]grade|complete secret scann(?:er|ing)" + r")\b", + re.IGNORECASE, + ), + re.compile( + r"\bnot\s+(a\s+)?(" + r"security scanner|secret scanner|production[- ]ready|enterprise[- ]grade|" + r"secure|safe" + r")\b", + re.IGNORECASE, + ), +) + + +def find_unsupported_claim_findings( + repository_root: Path, + instruction_files: tuple[InstructionFile, ...], +) -> tuple[Finding, ...]: + """Return unsupported security or maturity claim findings. + + The rule is intentionally conservative and deterministic. It scans only + supported instruction files discovered by agent-rules-kit and does not + execute repository commands, call the network, or call an LLM. + """ + findings: list[Finding] = [] + + for instruction_file in instruction_files: + candidate = repository_root / instruction_file.path + + try: + text = candidate.read_text(encoding="utf-8") + except UnicodeDecodeError: + continue + + for line_number, line in enumerate(text.splitlines(), start=1): + if _contains_unsupported_claim(line): + findings.append( + Finding( + rule_id=UNSUPPORTED_CLAIM_RULE_ID, + severity=Severity.WARNING, + message=UNSUPPORTED_CLAIM_MESSAGE, + path=instruction_file.path, + line=line_number, + ) + ) + + return tuple(findings) + + +def _contains_unsupported_claim(line: str) -> bool: + has_claim = any( + pattern.search(line) is not None for pattern in UNSUPPORTED_CLAIM_PATTERNS + ) + if not has_claim: + return False + + return not any( + pattern.search(line) is not None + for pattern in NEGATED_UNSUPPORTED_CLAIM_CONTEXT_PATTERNS + ) + + +__all__ = [ + "NEGATED_UNSUPPORTED_CLAIM_CONTEXT_PATTERNS", + "UNSUPPORTED_CLAIM_MESSAGE", + "UNSUPPORTED_CLAIM_PATTERNS", + "UNSUPPORTED_CLAIM_RULE_ID", + "find_unsupported_claim_findings", +] diff --git a/tests/fixtures/repositories/unsupported-claim/AGENTS.md b/tests/fixtures/repositories/unsupported-claim/AGENTS.md new file mode 100644 index 0000000..5708204 --- /dev/null +++ b/tests/fixtures/repositories/unsupported-claim/AGENTS.md @@ -0,0 +1,6 @@ +# AGENTS.md + +Rules: + +- This process guarantees security for the repository. +- This project is enterprise-grade and production-ready. diff --git a/tests/test_cli.py b/tests/test_cli.py index f8a250d..fee6fea 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -419,6 +419,89 @@ def test_init_write_backs_up_existing_agents_file_before_replacing(self) -> None self.assertIn("- AGENTS.md [backup-and-replace]", text) self.assertIn("backup: AGENTS.md.agent-rules-kit.bak", text) + def test_check_console_reports_unsupported_security_claim_findings(self) -> None: + output = io.StringIO() + + with redirect_stdout(output): + exit_code = main(["check", str(FIXTURE_ROOT / "unsupported-claim")]) + + text = output.getvalue() + + self.assertEqual(exit_code, 0) + self.assertIn("Found 1 supported instruction file(s):", text) + self.assertIn("Findings:", text) + self.assertIn("AIRK-GOV006 [warning] AGENTS.md:5", text) + self.assertIn("AGENTS.md:6", text) + self.assertIn( + "Instruction file may contain an unsupported security or maturity claim.", + text, + ) + + def test_check_json_reports_unsupported_security_claim_findings(self) -> None: + output = io.StringIO() + + with redirect_stdout(output): + exit_code = main( + [ + "check", + str(FIXTURE_ROOT / "unsupported-claim"), + "--format", + "json", + ] + ) + + payload = json.loads(output.getvalue()) + + self.assertEqual(exit_code, 0) + self.assertEqual(payload["summary"]["finding_count"], 2) + self.assertEqual(len(payload["findings"]), 2) + self.assertEqual(payload["findings"][0]["rule_id"], "AIRK-GOV006") + self.assertEqual(payload["findings"][0]["severity"], "warning") + self.assertEqual(payload["findings"][0]["path"], "AGENTS.md") + self.assertEqual(payload["findings"][0]["line"], 5) + self.assertEqual(payload["findings"][1]["line"], 6) + + def test_check_markdown_reports_unsupported_security_claim_findings(self) -> None: + output = io.StringIO() + + with redirect_stdout(output): + exit_code = main( + [ + "check", + str(FIXTURE_ROOT / "unsupported-claim"), + "--format", + "markdown", + ] + ) + + text = output.getvalue() + + self.assertEqual(exit_code, 0) + self.assertIn("- Findings: 2", text) + self.assertIn("## Findings", text) + self.assertIn("| AIRK-GOV006 | warning | AGENTS.md:5 |", text) + self.assertIn("| AIRK-GOV006 | warning | AGENTS.md:6 |", text) + + def test_check_json_reports_empty_findings_for_clean_fixture(self) -> None: + output = io.StringIO() + + with redirect_stdout(output): + exit_code = main( + [ + "check", + str(FIXTURE_ROOT / "single-agent"), + "--format", + "json", + ] + ) + + payload = json.loads(output.getvalue()) + + self.assertEqual(exit_code, 0) + self.assertEqual(payload["summary"]["finding_count"], 0) + self.assertEqual(payload["findings"], []) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_governance.py b/tests/test_governance.py new file mode 100644 index 0000000..6d8dc7b --- /dev/null +++ b/tests/test_governance.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +from agent_rules_kit.discovery import discover_instruction_files +from agent_rules_kit.governance import find_unsupported_claim_findings + + +class GovernanceFindingTests(unittest.TestCase): + def test_reports_unsupported_security_and_maturity_claims(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + repository = Path(temporary_directory) + (repository / "AGENTS.md").write_text( + "\n".join( + [ + "# AGENTS.md", + "", + "Rules:", + "", + "- This process guarantees security for the repository.", + "- This project is enterprise-grade and production-ready.", + ] + ) + + "\n", + encoding="utf-8", + ) + + instruction_files = discover_instruction_files(repository) + findings = find_unsupported_claim_findings(repository, instruction_files) + + self.assertEqual( + [finding.rule_id for finding in findings], + ["AIRK-GOV006", "AIRK-GOV006"], + ) + self.assertEqual([finding.line for finding in findings], [5, 6]) + self.assertEqual([finding.path for finding in findings], ["AGENTS.md", "AGENTS.md"]) + + def test_ignores_negative_guidance_about_unsupported_claims(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + repository = Path(temporary_directory) + (repository / "AGENTS.md").write_text( + "\n".join( + [ + "# AGENTS.md", + "", + "Rules:", + "", + "- Do not claim this repository is enterprise-grade.", + "- Never say this process guarantees security.", + "- This tool is not a security scanner.", + "- Avoid production-ready claims.", + "- The tool must not claim complete secret scanning.", + ] + ) + + "\n", + encoding="utf-8", + ) + + instruction_files = discover_instruction_files(repository) + findings = find_unsupported_claim_findings(repository, instruction_files) + + self.assertEqual(findings, ()) + + def test_scans_only_supported_instruction_files(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + repository = Path(temporary_directory) + (repository / "README.md").write_text( + "This project is enterprise-grade and guarantees security.\n", + encoding="utf-8", + ) + + instruction_files = discover_instruction_files(repository) + findings = find_unsupported_claim_findings(repository, instruction_files) + + self.assertEqual(instruction_files, ()) + self.assertEqual(findings, ()) + + +if __name__ == "__main__": + unittest.main()