diff --git a/pyproject.toml b/pyproject.toml index e2af2b3..4d3ea6f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "context-compiler" -version = "0.6.17" +version = "0.6.18" description = "Deterministic conversational state engine for LLM applications." readme = "README.md" requires-python = ">=3.11" diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index 883eca2..a45f24b 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -57,9 +57,27 @@ For conformance transcript fixtures: * If `prompt_to_user` is a string → exact match * If `prompt_to_user` is `null` → any non-empty string is accepted +## State JSON fixtures + +For [`conformance/state-json/`](conformance/state-json/): + +Portable serialization contract coverage for `engine.export_json()` and +`engine.import_json(...)`, including canonical export payload shape and +deterministic validation/error boundaries. + +## Checkpoint fixtures + +For [`conformance/checkpoint/`](conformance/checkpoint/): + +Portable checkpoint import contract coverage for +`engine.import_checkpoint(...)`, including deterministic validation/error +boundaries, atomic failure behavior, and pending-clarification clearing semantics. + ## Source of truth Fixtures reflect current Python behavior and tests. +Property/fuzz invariants remain Python-local tests and are not part of the +portable fixture contract. ## Engine regression fixtures @@ -83,6 +101,12 @@ These fixtures cover preprocessor behavior (heuristic classification plus output They are exercised by [`tests/test_preprocessor_conformance.py`](../test_preprocessor_conformance.py), including deterministic replay and validation-boundary checks (only validated directive output may pass through). +Portable fixture scope: +- deterministic heuristic and validator input/output contracts intended for cross-language parity + +Python-local test scope: +- property/fuzz invariants and filesystem/template behaviors (for example `render_prompt` file-loading behavior) remain in Python unit/property tests and are not portable fixture requirements. + They validate: * heuristic classification determinism diff --git a/tests/fixtures/conformance/checkpoint/001_import_checkpoint_non_object_rejected.json b/tests/fixtures/conformance/checkpoint/001_import_checkpoint_non_object_rejected.json new file mode 100644 index 0000000..b9715bf --- /dev/null +++ b/tests/fixtures/conformance/checkpoint/001_import_checkpoint_non_object_rejected.json @@ -0,0 +1,24 @@ +{ + "id": "checkpoint_import_checkpoint_non_object_rejected", + "kind": "checkpoint", + "initial_state": { + "premise": null, + "policies": {}, + "version": 2 + }, + "action": { + "fn": "import_checkpoint", + "payload": [] + }, + "expected": { + "error": { + "type": "ValueError", + "message_contains": "Invalid checkpoint payload" + }, + "state": { + "premise": null, + "policies": {}, + "version": 2 + } + } +} diff --git a/tests/fixtures/conformance/checkpoint/002_import_checkpoint_unsupported_version_rejected.json b/tests/fixtures/conformance/checkpoint/002_import_checkpoint_unsupported_version_rejected.json new file mode 100644 index 0000000..2a9f1e0 --- /dev/null +++ b/tests/fixtures/conformance/checkpoint/002_import_checkpoint_unsupported_version_rejected.json @@ -0,0 +1,32 @@ +{ + "id": "checkpoint_import_checkpoint_unsupported_version_rejected", + "kind": "checkpoint", + "initial_state": { + "premise": null, + "policies": {}, + "version": 2 + }, + "action": { + "fn": "import_checkpoint", + "payload": { + "checkpoint_version": 2, + "authoritative_state": { + "premise": null, + "policies": {}, + "version": 2 + }, + "pending": null + } + }, + "expected": { + "error": { + "type": "ValueError", + "message_contains": "Unsupported checkpoint version" + }, + "state": { + "premise": null, + "policies": {}, + "version": 2 + } + } +} diff --git a/tests/fixtures/conformance/checkpoint/003_import_checkpoint_invalid_pending_shape_rejected.json b/tests/fixtures/conformance/checkpoint/003_import_checkpoint_invalid_pending_shape_rejected.json new file mode 100644 index 0000000..019a42d --- /dev/null +++ b/tests/fixtures/conformance/checkpoint/003_import_checkpoint_invalid_pending_shape_rejected.json @@ -0,0 +1,34 @@ +{ + "id": "checkpoint_import_checkpoint_invalid_pending_shape_rejected", + "kind": "checkpoint", + "initial_state": { + "premise": null, + "policies": {}, + "version": 2 + }, + "action": { + "fn": "import_checkpoint", + "payload": { + "checkpoint_version": 1, + "authoritative_state": { + "premise": null, + "policies": {}, + "version": 2 + }, + "pending": { + "kind": "replacement" + } + } + }, + "expected": { + "error": { + "type": "ValueError", + "message_contains": "Invalid checkpoint payload" + }, + "state": { + "premise": null, + "policies": {}, + "version": 2 + } + } +} diff --git a/tests/fixtures/conformance/checkpoint/004_import_checkpoint_invalid_replacement_shape_rejected.json b/tests/fixtures/conformance/checkpoint/004_import_checkpoint_invalid_replacement_shape_rejected.json new file mode 100644 index 0000000..ce138f9 --- /dev/null +++ b/tests/fixtures/conformance/checkpoint/004_import_checkpoint_invalid_replacement_shape_rejected.json @@ -0,0 +1,40 @@ +{ + "id": "checkpoint_import_checkpoint_invalid_replacement_shape_rejected", + "kind": "checkpoint", + "initial_state": { + "premise": null, + "policies": {}, + "version": 2 + }, + "action": { + "fn": "import_checkpoint", + "payload": { + "checkpoint_version": 1, + "authoritative_state": { + "premise": null, + "policies": {}, + "version": 2 + }, + "pending": { + "kind": "replacement", + "replacement": { + "kind": "use_only", + "new_item": "kubectl", + "old_item": "docker" + }, + "prompt_to_user": "confirm?" + } + } + }, + "expected": { + "error": { + "type": "ValueError", + "message_contains": "Invalid checkpoint payload" + }, + "state": { + "premise": null, + "policies": {}, + "version": 2 + } + } +} diff --git a/tests/fixtures/conformance/checkpoint/005_import_checkpoint_invalid_authoritative_state_rejected_atomically.json b/tests/fixtures/conformance/checkpoint/005_import_checkpoint_invalid_authoritative_state_rejected_atomically.json new file mode 100644 index 0000000..285a1d9 --- /dev/null +++ b/tests/fixtures/conformance/checkpoint/005_import_checkpoint_invalid_authoritative_state_rejected_atomically.json @@ -0,0 +1,48 @@ +{ + "id": "checkpoint_import_checkpoint_invalid_authoritative_state_rejected_atomically", + "kind": "checkpoint", + "initial_state": { + "premise": null, + "policies": {}, + "version": 2 + }, + "prelude": [ + "use kubectl instead of docker" + ], + "action": { + "fn": "import_checkpoint", + "payload": { + "checkpoint_version": 1, + "authoritative_state": { + "premise": null, + "policies": [], + "version": 2 + }, + "pending": null + } + }, + "expected": { + "error": { + "type": "ValueError", + "message_contains": "Invalid state payload" + }, + "state": { + "premise": null, + "policies": {}, + "version": 2 + }, + "followup": { + "input": "maybe", + "decision": { + "kind": "clarify", + "state": null, + "prompt_to_user": "Did you mean to use \"kubectl\" instead?" + }, + "state": { + "premise": null, + "policies": {}, + "version": 2 + } + } + } +} diff --git a/tests/fixtures/conformance/checkpoint/006_import_checkpoint_pending_null_clears_existing_pending.json b/tests/fixtures/conformance/checkpoint/006_import_checkpoint_pending_null_clears_existing_pending.json new file mode 100644 index 0000000..6599bc7 --- /dev/null +++ b/tests/fixtures/conformance/checkpoint/006_import_checkpoint_pending_null_clears_existing_pending.json @@ -0,0 +1,50 @@ +{ + "id": "checkpoint_import_checkpoint_pending_null_clears_existing_pending", + "kind": "checkpoint", + "initial_state": { + "premise": null, + "policies": {}, + "version": 2 + }, + "prelude": [ + "use kubectl instead of docker" + ], + "action": { + "fn": "import_checkpoint", + "payload": { + "checkpoint_version": 1, + "authoritative_state": { + "premise": "baseline", + "policies": { + "pytest": "use" + }, + "version": 2 + }, + "pending": null + } + }, + "expected": { + "state": { + "premise": "baseline", + "policies": { + "pytest": "use" + }, + "version": 2 + }, + "followup": { + "input": "yes", + "decision": { + "kind": "passthrough", + "state": null, + "prompt_to_user": null + }, + "state": { + "premise": "baseline", + "policies": { + "pytest": "use" + }, + "version": 2 + } + } + } +} diff --git a/tests/fixtures/conformance/checkpoint/007_import_checkpoint_pending_absent_clears_existing_pending.json b/tests/fixtures/conformance/checkpoint/007_import_checkpoint_pending_absent_clears_existing_pending.json new file mode 100644 index 0000000..18c3cbe --- /dev/null +++ b/tests/fixtures/conformance/checkpoint/007_import_checkpoint_pending_absent_clears_existing_pending.json @@ -0,0 +1,49 @@ +{ + "id": "checkpoint_import_checkpoint_pending_absent_clears_existing_pending", + "kind": "checkpoint", + "initial_state": { + "premise": null, + "policies": {}, + "version": 2 + }, + "prelude": [ + "use kubectl instead of docker" + ], + "action": { + "fn": "import_checkpoint", + "payload": { + "checkpoint_version": 1, + "authoritative_state": { + "premise": "baseline", + "policies": { + "pytest": "use" + }, + "version": 2 + } + } + }, + "expected": { + "state": { + "premise": "baseline", + "policies": { + "pytest": "use" + }, + "version": 2 + }, + "followup": { + "input": "yes", + "decision": { + "kind": "passthrough", + "state": null, + "prompt_to_user": null + }, + "state": { + "premise": "baseline", + "policies": { + "pytest": "use" + }, + "version": 2 + } + } + } +} diff --git a/tests/fixtures/conformance/state-json/001_export_json_canonical_sorted_compact.json b/tests/fixtures/conformance/state-json/001_export_json_canonical_sorted_compact.json new file mode 100644 index 0000000..ba3a9a7 --- /dev/null +++ b/tests/fixtures/conformance/state-json/001_export_json_canonical_sorted_compact.json @@ -0,0 +1,28 @@ +{ + "id": "state_json_export_json_canonical_sorted_compact", + "kind": "state_json", + "initial_state": { + "premise": null, + "policies": {}, + "version": 2 + }, + "prelude": [ + "set premise concise", + "use zeta", + "use alpha" + ], + "action": { + "fn": "export_json" + }, + "expected": { + "payload": "{\"policies\":{\"alpha\":\"use\",\"zeta\":\"use\"},\"premise\":\"concise\",\"version\":2}", + "state": { + "premise": "concise", + "policies": { + "alpha": "use", + "zeta": "use" + }, + "version": 2 + } + } +} diff --git a/tests/fixtures/conformance/state-json/002_import_json_invalid_json_rejected.json b/tests/fixtures/conformance/state-json/002_import_json_invalid_json_rejected.json new file mode 100644 index 0000000..1153210 --- /dev/null +++ b/tests/fixtures/conformance/state-json/002_import_json_invalid_json_rejected.json @@ -0,0 +1,24 @@ +{ + "id": "state_json_import_json_invalid_json_rejected", + "kind": "state_json", + "initial_state": { + "premise": null, + "policies": {}, + "version": 2 + }, + "action": { + "fn": "import_json", + "payload": "{" + }, + "expected": { + "error": { + "type": "ValueError", + "message_contains": "Invalid JSON payload" + }, + "state": { + "premise": null, + "policies": {}, + "version": 2 + } + } +} diff --git a/tests/fixtures/conformance/state-json/003_import_json_non_object_rejected.json b/tests/fixtures/conformance/state-json/003_import_json_non_object_rejected.json new file mode 100644 index 0000000..872e7c2 --- /dev/null +++ b/tests/fixtures/conformance/state-json/003_import_json_non_object_rejected.json @@ -0,0 +1,24 @@ +{ + "id": "state_json_import_json_non_object_rejected", + "kind": "state_json", + "initial_state": { + "premise": null, + "policies": {}, + "version": 2 + }, + "action": { + "fn": "import_json", + "payload": "[\"not\",\"an\",\"object\"]" + }, + "expected": { + "error": { + "type": "ValueError", + "message_contains": "Invalid state payload" + }, + "state": { + "premise": null, + "policies": {}, + "version": 2 + } + } +} diff --git a/tests/fixtures/conformance/state-json/004_import_json_unsupported_version_rejected.json b/tests/fixtures/conformance/state-json/004_import_json_unsupported_version_rejected.json new file mode 100644 index 0000000..98c855f --- /dev/null +++ b/tests/fixtures/conformance/state-json/004_import_json_unsupported_version_rejected.json @@ -0,0 +1,24 @@ +{ + "id": "state_json_import_json_unsupported_version_rejected", + "kind": "state_json", + "initial_state": { + "premise": null, + "policies": {}, + "version": 2 + }, + "action": { + "fn": "import_json", + "payload": "{\"premise\":null,\"policies\":{},\"version\":1}" + }, + "expected": { + "error": { + "type": "ValueError", + "message_contains": "Unsupported state version" + }, + "state": { + "premise": null, + "policies": {}, + "version": 2 + } + } +} diff --git a/tests/fixtures/conformance/state-json/005_import_json_empty_normalized_policy_key_rejected_atomically.json b/tests/fixtures/conformance/state-json/005_import_json_empty_normalized_policy_key_rejected_atomically.json new file mode 100644 index 0000000..9584f5a --- /dev/null +++ b/tests/fixtures/conformance/state-json/005_import_json_empty_normalized_policy_key_rejected_atomically.json @@ -0,0 +1,29 @@ +{ + "id": "state_json_import_json_empty_normalized_policy_key_rejected_atomically", + "kind": "state_json", + "initial_state": { + "premise": null, + "policies": {}, + "version": 2 + }, + "prelude": [ + "use kubectl" + ], + "action": { + "fn": "import_json", + "payload": "{\"premise\":null,\"policies\":{\"Docker\":\"use\",\"a\":\"use\"},\"version\":2}" + }, + "expected": { + "error": { + "type": "ValueError", + "message_contains": "Invalid state payload" + }, + "state": { + "premise": null, + "policies": { + "kubectl": "use" + }, + "version": 2 + } + } +} diff --git a/tests/fixtures/conformance/state-json/006_import_json_valid_normalized_policy_key_accepted.json b/tests/fixtures/conformance/state-json/006_import_json_valid_normalized_policy_key_accepted.json new file mode 100644 index 0000000..c83600a --- /dev/null +++ b/tests/fixtures/conformance/state-json/006_import_json_valid_normalized_policy_key_accepted.json @@ -0,0 +1,22 @@ +{ + "id": "state_json_import_json_valid_normalized_policy_key_accepted", + "kind": "state_json", + "initial_state": { + "premise": null, + "policies": {}, + "version": 2 + }, + "action": { + "fn": "import_json", + "payload": "{\"premise\":null,\"policies\":{\"Docker\":\"use\"},\"version\":2}" + }, + "expected": { + "state": { + "premise": null, + "policies": { + "docker": "use" + }, + "version": 2 + } + } +} diff --git a/tests/fixtures/conformance/step/018_pending_affirmative_punctuation_token.json b/tests/fixtures/conformance/step/018_pending_affirmative_punctuation_token.json new file mode 100644 index 0000000..168c85b --- /dev/null +++ b/tests/fixtures/conformance/step/018_pending_affirmative_punctuation_token.json @@ -0,0 +1,33 @@ +{ + "id": "step_pending_affirmative_punctuation_token", + "kind": "step", + "initial_state": { + "premise": null, + "policies": {}, + "version": 2 + }, + "prelude": [ + "use kubectl instead of docker" + ], + "input": " okay!!! ", + "expected": { + "decision": { + "kind": "update", + "prompt_to_user": null, + "state": { + "premise": null, + "policies": { + "kubectl": "use" + }, + "version": 2 + } + }, + "state": { + "premise": null, + "policies": { + "kubectl": "use" + }, + "version": 2 + } + } +} diff --git a/tests/fixtures/conformance/step/019_pending_negative_punctuation_token.json b/tests/fixtures/conformance/step/019_pending_negative_punctuation_token.json new file mode 100644 index 0000000..bd03c8d --- /dev/null +++ b/tests/fixtures/conformance/step/019_pending_negative_punctuation_token.json @@ -0,0 +1,29 @@ +{ + "id": "step_pending_negative_punctuation_token", + "kind": "step", + "initial_state": { + "premise": null, + "policies": {}, + "version": 2 + }, + "prelude": [ + "use kubectl instead of docker" + ], + "input": " NOPE?? ", + "expected": { + "decision": { + "kind": "update", + "prompt_to_user": null, + "state": { + "premise": null, + "policies": {}, + "version": 2 + } + }, + "state": { + "premise": null, + "policies": {}, + "version": 2 + } + } +} diff --git a/tests/fixtures/preprocessor/canonical-directive-bracket-wrapper.json b/tests/fixtures/preprocessor/canonical-directive-bracket-wrapper.json new file mode 100644 index 0000000..44d956d --- /dev/null +++ b/tests/fixtures/preprocessor/canonical-directive-bracket-wrapper.json @@ -0,0 +1,8 @@ +{ + "name": "canonical-directive-bracket-wrapper", + "input": "[prohibit peanuts]", + "expected": { + "classification": "directive", + "output": "prohibit peanuts" + } +} diff --git a/tests/fixtures/preprocessor/canonical-directive-clear-state-period.json b/tests/fixtures/preprocessor/canonical-directive-clear-state-period.json new file mode 100644 index 0000000..ee769eb --- /dev/null +++ b/tests/fixtures/preprocessor/canonical-directive-clear-state-period.json @@ -0,0 +1,8 @@ +{ + "name": "canonical-directive-clear-state-period", + "input": "clear state.", + "expected": { + "classification": "directive", + "output": "clear state" + } +} diff --git a/tests/fixtures/preprocessor/canonical-directive-paren-wrapper.json b/tests/fixtures/preprocessor/canonical-directive-paren-wrapper.json new file mode 100644 index 0000000..3283b4d --- /dev/null +++ b/tests/fixtures/preprocessor/canonical-directive-paren-wrapper.json @@ -0,0 +1,8 @@ +{ + "name": "canonical-directive-paren-wrapper", + "input": "(clear state)", + "expected": { + "classification": "directive", + "output": "clear state" + } +} diff --git a/tests/fixtures/preprocessor/canonical-directive-reset-policies-bang.json b/tests/fixtures/preprocessor/canonical-directive-reset-policies-bang.json new file mode 100644 index 0000000..993f280 --- /dev/null +++ b/tests/fixtures/preprocessor/canonical-directive-reset-policies-bang.json @@ -0,0 +1,8 @@ +{ + "name": "canonical-directive-reset-policies-bang", + "input": "reset policies!", + "expected": { + "classification": "directive", + "output": "reset policies" + } +} diff --git a/tests/fixtures/preprocessor/list-prefix-directive-unknown.json b/tests/fixtures/preprocessor/list-prefix-directive-unknown.json new file mode 100644 index 0000000..ecdb2b5 --- /dev/null +++ b/tests/fixtures/preprocessor/list-prefix-directive-unknown.json @@ -0,0 +1,8 @@ +{ + "name": "list-prefix-directive-unknown", + "input": "1. use docker", + "expected": { + "classification": "unknown", + "output": null + } +} diff --git a/tests/fixtures/preprocessor/meta-prefix-directive-unknown.json b/tests/fixtures/preprocessor/meta-prefix-directive-unknown.json new file mode 100644 index 0000000..cedcaf7 --- /dev/null +++ b/tests/fixtures/preprocessor/meta-prefix-directive-unknown.json @@ -0,0 +1,8 @@ +{ + "name": "meta-prefix-directive-unknown", + "input": "example: use docker", + "expected": { + "classification": "unknown", + "output": null + } +} diff --git a/tests/fixtures/preprocessor/validator-invalid-json-shape-unknown.json b/tests/fixtures/preprocessor/validator-invalid-json-shape-unknown.json new file mode 100644 index 0000000..ef7bfe7 --- /dev/null +++ b/tests/fixtures/preprocessor/validator-invalid-json-shape-unknown.json @@ -0,0 +1,11 @@ +{ + "name": "validator-invalid-json-shape-unknown", + "kind": "validator", + "raw_output": { + "classification": "directive" + }, + "expected": { + "classification": "unknown", + "output": null + } +} diff --git a/tests/fixtures/preprocessor/validator-malformed-json-text-unknown.json b/tests/fixtures/preprocessor/validator-malformed-json-text-unknown.json new file mode 100644 index 0000000..da8ac6c --- /dev/null +++ b/tests/fixtures/preprocessor/validator-malformed-json-text-unknown.json @@ -0,0 +1,9 @@ +{ + "name": "validator-malformed-json-text-unknown", + "kind": "validator", + "raw_output": "{\"classification\":\"directive\",", + "expected": { + "classification": "unknown", + "output": null + } +} diff --git a/tests/fixtures/preprocessor/validator-malformed-sentinel-unknown.json b/tests/fixtures/preprocessor/validator-malformed-sentinel-unknown.json new file mode 100644 index 0000000..040bd5b --- /dev/null +++ b/tests/fixtures/preprocessor/validator-malformed-sentinel-unknown.json @@ -0,0 +1,9 @@ +{ + "name": "validator-malformed-sentinel-unknown", + "kind": "validator", + "raw_output": "", + "expected": { + "classification": "unknown", + "output": null + } +} diff --git a/tests/fixtures/preprocessor/validator-malformed-text-unknown.json b/tests/fixtures/preprocessor/validator-malformed-text-unknown.json new file mode 100644 index 0000000..0d6821b --- /dev/null +++ b/tests/fixtures/preprocessor/validator-malformed-text-unknown.json @@ -0,0 +1,9 @@ +{ + "name": "validator-malformed-text-unknown", + "kind": "validator", + "raw_output": "set premise to concise replies", + "expected": { + "classification": "unknown", + "output": null + } +} diff --git a/tests/fixtures/preprocessor/validator-multi-candidate-directive-unknown.json b/tests/fixtures/preprocessor/validator-multi-candidate-directive-unknown.json new file mode 100644 index 0000000..c22f4a0 --- /dev/null +++ b/tests/fixtures/preprocessor/validator-multi-candidate-directive-unknown.json @@ -0,0 +1,9 @@ +{ + "name": "validator-multi-candidate-directive-unknown", + "kind": "validator", + "raw_output": "prohibit peanuts and use almonds", + "expected": { + "classification": "unknown", + "output": null + } +} diff --git a/tests/fixtures/preprocessor/validator-sentinel-no-directive.json b/tests/fixtures/preprocessor/validator-sentinel-no-directive.json new file mode 100644 index 0000000..ed1b659 --- /dev/null +++ b/tests/fixtures/preprocessor/validator-sentinel-no-directive.json @@ -0,0 +1,9 @@ +{ + "name": "validator-sentinel-no-directive", + "kind": "validator", + "raw_output": "", + "expected": { + "classification": "no_directive", + "output": null + } +} diff --git a/tests/fixtures/preprocessor/validator-source-input-allow-safe-directive.json b/tests/fixtures/preprocessor/validator-source-input-allow-safe-directive.json new file mode 100644 index 0000000..4d6e112 --- /dev/null +++ b/tests/fixtures/preprocessor/validator-source-input-allow-safe-directive.json @@ -0,0 +1,10 @@ +{ + "name": "validator-source-input-allow-safe-directive", + "kind": "validator", + "raw_output": "prohibit peanuts", + "source_input": "what is a simple curry recipe?", + "expected": { + "classification": "directive", + "output": "prohibit peanuts" + } +} diff --git a/tests/fixtures/preprocessor/validator-source-input-block-change-premise-rewrite.json b/tests/fixtures/preprocessor/validator-source-input-block-change-premise-rewrite.json new file mode 100644 index 0000000..8e93a16 --- /dev/null +++ b/tests/fixtures/preprocessor/validator-source-input-block-change-premise-rewrite.json @@ -0,0 +1,10 @@ +{ + "name": "validator-source-input-block-change-premise-rewrite", + "kind": "validator", + "raw_output": "change premise to concise replies", + "source_input": "change premise concise replies", + "expected": { + "classification": "unknown", + "output": null + } +} diff --git a/tests/fixtures/preprocessor/validator-source-input-block-set-premise-rewrite.json b/tests/fixtures/preprocessor/validator-source-input-block-set-premise-rewrite.json new file mode 100644 index 0000000..0415083 --- /dev/null +++ b/tests/fixtures/preprocessor/validator-source-input-block-set-premise-rewrite.json @@ -0,0 +1,10 @@ +{ + "name": "validator-source-input-block-set-premise-rewrite", + "kind": "validator", + "raw_output": "set premise concise replies", + "source_input": "set premise to concise replies", + "expected": { + "classification": "unknown", + "output": null + } +} diff --git a/tests/fixtures/preprocessor/validator-structured-json-directive.json b/tests/fixtures/preprocessor/validator-structured-json-directive.json new file mode 100644 index 0000000..e69177e --- /dev/null +++ b/tests/fixtures/preprocessor/validator-structured-json-directive.json @@ -0,0 +1,9 @@ +{ + "name": "validator-structured-json-directive", + "kind": "validator", + "raw_output": "{\"classification\":\"directive\",\"output\":\"use docker\"}", + "expected": { + "classification": "directive", + "output": "use docker" + } +} diff --git a/tests/test_fixtures.py b/tests/test_fixtures.py index 2c30904..2434116 100644 --- a/tests/test_fixtures.py +++ b/tests/test_fixtures.py @@ -1,12 +1,20 @@ import json from pathlib import Path +import pytest + from context_compiler import compile_transcript, create_engine _STEP_FIXTURES_DIR = Path(__file__).resolve().parent / "fixtures" / "conformance" / "step" _TRANSCRIPT_FIXTURES_DIR = ( Path(__file__).resolve().parent / "fixtures" / "conformance" / "transcript" ) +_STATE_JSON_FIXTURES_DIR = ( + Path(__file__).resolve().parent / "fixtures" / "conformance" / "state-json" +) +_CHECKPOINT_FIXTURES_DIR = ( + Path(__file__).resolve().parent / "fixtures" / "conformance" / "checkpoint" +) def _json_files(dir_path: Path) -> list[Path]: @@ -75,3 +83,73 @@ def test_transcript_fixtures() -> None: normalized = {"state": result} assert normalized == fixture["expected"], fixture_id + + +def _apply_prelude(engine: object, prelude: object) -> None: + assert isinstance(prelude, list) + for prior_input in prelude: + assert isinstance(prior_input, str) + engine.step(prior_input) + + +def test_state_json_fixtures() -> None: + for path in _json_files(_STATE_JSON_FIXTURES_DIR): + fixture = _load(path) + fixture_id = fixture["id"] + + assert fixture["kind"] == "state_json", fixture_id + engine = create_engine(state=fixture["initial_state"]) + _apply_prelude(engine, fixture.get("prelude", [])) + + action = fixture["action"] + expected = fixture["expected"] + fn = action["fn"] + + if fn == "export_json": + payload = engine.export_json() + assert payload == expected["payload"], fixture_id + elif fn == "import_json": + payload = action["payload"] + error = expected.get("error") + if error is None: + engine.import_json(payload) + else: + with pytest.raises(ValueError, match=error["message_contains"]): + engine.import_json(payload) + else: + raise AssertionError(f"Unknown state_json action: {fn}") + + assert engine.state == expected["state"], fixture_id + + +def test_checkpoint_fixtures() -> None: + for path in _json_files(_CHECKPOINT_FIXTURES_DIR): + fixture = _load(path) + fixture_id = fixture["id"] + + assert fixture["kind"] == "checkpoint", fixture_id + engine = create_engine(state=fixture["initial_state"]) + _apply_prelude(engine, fixture.get("prelude", [])) + + action = fixture["action"] + expected = fixture["expected"] + fn = action["fn"] + + if fn == "import_checkpoint": + payload = action["payload"] + error = expected.get("error") + if error is None: + engine.import_checkpoint(payload) + else: + with pytest.raises(ValueError, match=error["message_contains"]): + engine.import_checkpoint(payload) + else: + raise AssertionError(f"Unknown checkpoint action: {fn}") + + assert engine.state == expected["state"], fixture_id + + followup = expected.get("followup") + if followup is not None: + decision = engine.step(followup["input"]) + assert decision == followup["decision"], fixture_id + assert engine.state == followup["state"], fixture_id diff --git a/tests/test_preprocessor_conformance.py b/tests/test_preprocessor_conformance.py index 61d1c9e..c9337e0 100644 --- a/tests/test_preprocessor_conformance.py +++ b/tests/test_preprocessor_conformance.py @@ -8,12 +8,8 @@ _PREPROCESSOR_FIXTURES_DIR = Path(__file__).resolve().parent / "fixtures" / "preprocessor" -def _fixture_paths() -> list[Path]: - return sorted( - path - for path in _PREPROCESSOR_FIXTURES_DIR.glob("*.json") - if not path.name.startswith("public-api-") - ) +def _behavior_fixture_paths() -> list[Path]: + return sorted(path for path in _PREPROCESSOR_FIXTURES_DIR.glob("*.json")) def _load_fixture(path: Path) -> dict[str, object]: @@ -40,6 +36,16 @@ def _normalize_result(message: str) -> dict[str, object]: return normalized +def _normalize_validator_result( + raw_output: object, source_input: str | None = None +) -> dict[str, object]: + validated = validate_precompiler_output(raw_output, source_input=source_input) + return { + "classification": validated["classification"], + "output": validated["output"], + } + + def _derived_risky_rewrite_candidates(source_input: str) -> list[str]: normalized = re.sub(r"\s+", " ", source_input.strip().lower()) candidates: list[str] = [] @@ -58,18 +64,37 @@ def _derived_risky_rewrite_candidates(source_input: str) -> list[str]: def test_preprocessor_conformance_fixtures() -> None: - for path in _fixture_paths(): + for path in _behavior_fixture_paths(): fixture = _load_fixture(path) - expected = fixture["expected"] - input_text = fixture["input"] - fixture_name = fixture["name"] + if path.name.startswith("public-api-"): + continue + + expected = fixture.get("expected") + fixture_name = fixture.get("name", path.name) + kind = fixture.get("kind", "heuristic") assert isinstance(expected, dict), fixture_name - assert isinstance(input_text, str), fixture_name + + if kind == "heuristic": + input_text = fixture.get("input") + assert isinstance(input_text, str), fixture_name + + # Deterministic replay check. + first = _normalize_result(input_text) + second = _normalize_result(input_text) + assert first == second, fixture_name + assert first == expected, fixture_name + continue + + assert kind == "validator", fixture_name + assert "raw_output" in fixture, fixture_name + raw_output = fixture["raw_output"] + source_input_obj = fixture.get("source_input") + source_input = source_input_obj if isinstance(source_input_obj, str) else None # Deterministic replay check. - first = _normalize_result(input_text) - second = _normalize_result(input_text) + first = _normalize_validator_result(raw_output, source_input=source_input) + second = _normalize_validator_result(raw_output, source_input=source_input) assert first == second, fixture_name assert first == expected, fixture_name @@ -77,8 +102,12 @@ def test_preprocessor_conformance_fixtures() -> None: def test_engine_owned_near_misses_are_reject_only_for_fallback_rewrites() -> None: # Engine-owned near-misses must not be canonicalized by the preprocessor and # must remain unknown even if fallback proposes a plausible canonical rewrite. - for path in _fixture_paths(): + for path in _behavior_fixture_paths(): fixture = _load_fixture(path) + if path.name.startswith("public-api-"): + continue + if fixture.get("kind", "heuristic") != "heuristic": + continue expected = fixture["expected"] input_text = fixture["input"] fixture_name = fixture["name"] diff --git a/uv.lock b/uv.lock index 7cd06ab..f57caed 100644 --- a/uv.lock +++ b/uv.lock @@ -468,7 +468,7 @@ wheels = [ [[package]] name = "context-compiler" -version = "0.6.17" +version = "0.6.18" source = { editable = "." } [package.optional-dependencies]