117 lines
5.3 KiB
Python
117 lines
5.3 KiB
Python
"""Regression tests for KIN-100 — human-readable agent output format.
|
|
|
|
Verifies that reviewer.md and tester.md prompts contain the two-block format
|
|
instructions (## Verdict + ## Details), ensuring agents produce output that
|
|
is both human-readable and machine-parseable.
|
|
"""
|
|
|
|
import os
|
|
|
|
PROMPTS_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "agents", "prompts")
|
|
|
|
|
|
def _read_prompt(name: str) -> str:
|
|
path = os.path.join(PROMPTS_DIR, name)
|
|
with open(path, encoding="utf-8") as f:
|
|
return f.read()
|
|
|
|
|
|
class TestReviewerPromptFormat:
|
|
def test_reviewer_prompt_contains_verdict_section(self):
|
|
content = _read_prompt("reviewer.md")
|
|
assert "## Verdict" in content, "reviewer.md must contain '## Verdict' section"
|
|
|
|
def test_reviewer_prompt_contains_details_section(self):
|
|
content = _read_prompt("reviewer.md")
|
|
assert "## Details" in content, "reviewer.md must contain '## Details' section"
|
|
|
|
def test_reviewer_prompt_verdict_instructs_plain_russian(self):
|
|
"""Verdict section must instruct plain Russian for the project director."""
|
|
content = _read_prompt("reviewer.md")
|
|
assert "Russian" in content or "russian" in content, (
|
|
"reviewer.md must mention Russian language for the Verdict section"
|
|
)
|
|
|
|
def test_reviewer_prompt_details_uses_json_fence(self):
|
|
"""Details section must specify JSON output in a ```json code fence."""
|
|
content = _read_prompt("reviewer.md")
|
|
assert "```json" in content, "reviewer.md Details section must use ```json fence"
|
|
|
|
def test_reviewer_prompt_verdict_forbids_json(self):
|
|
"""Verdict description must explicitly say no JSON/code in that section."""
|
|
content = _read_prompt("reviewer.md")
|
|
# The prompt should say something like "No JSON" near the Verdict section
|
|
assert "No JSON" in content or "no JSON" in content or "no code" in content.lower(), (
|
|
"reviewer.md Verdict section must explicitly say no JSON/code snippets"
|
|
)
|
|
|
|
def test_reviewer_prompt_has_example_verdict(self):
|
|
"""Prompt must contain an example of a plain-language verdict."""
|
|
content = _read_prompt("reviewer.md")
|
|
# The examples in the prompt contain Russian text after ## Verdict
|
|
assert "Реализация" in content or "проверен" in content.lower(), (
|
|
"reviewer.md must contain a Russian-language example verdict"
|
|
)
|
|
|
|
|
|
class TestTesterPromptFormat:
|
|
def test_tester_prompt_contains_verdict_section(self):
|
|
content = _read_prompt("tester.md")
|
|
assert "## Verdict" in content, "tester.md must contain '## Verdict' section"
|
|
|
|
def test_tester_prompt_contains_details_section(self):
|
|
content = _read_prompt("tester.md")
|
|
assert "## Details" in content, "tester.md must contain '## Details' section"
|
|
|
|
def test_tester_prompt_verdict_instructs_plain_russian(self):
|
|
content = _read_prompt("tester.md")
|
|
assert "Russian" in content or "russian" in content, (
|
|
"tester.md must mention Russian language for the Verdict section"
|
|
)
|
|
|
|
def test_tester_prompt_details_uses_json_fence(self):
|
|
content = _read_prompt("tester.md")
|
|
assert "```json" in content, "tester.md Details section must use ```json fence"
|
|
|
|
def test_tester_prompt_verdict_forbids_json(self):
|
|
content = _read_prompt("tester.md")
|
|
assert "No JSON" in content or "no JSON" in content or "no code" in content.lower(), (
|
|
"tester.md Verdict section must explicitly say no JSON/code snippets"
|
|
)
|
|
|
|
def test_tester_prompt_has_example_verdict(self):
|
|
content = _read_prompt("tester.md")
|
|
# The examples contain Russian text
|
|
assert "Написано" in content or "тест" in content.lower(), (
|
|
"tester.md must contain a Russian-language example verdict"
|
|
)
|
|
|
|
|
|
class TestBothPromptsStructure:
|
|
def test_reviewer_verdict_comes_before_details(self):
|
|
"""## Verdict section must appear before ## Details in reviewer.md."""
|
|
content = _read_prompt("reviewer.md")
|
|
verdict_pos = content.find("## Verdict")
|
|
details_pos = content.find("## Details")
|
|
assert verdict_pos != -1, "## Verdict must exist"
|
|
assert details_pos != -1, "## Details must exist"
|
|
assert verdict_pos < details_pos, "## Verdict must come before ## Details"
|
|
|
|
def test_tester_verdict_comes_before_details(self):
|
|
"""## Verdict section must appear before ## Details in tester.md."""
|
|
content = _read_prompt("tester.md")
|
|
verdict_pos = content.find("## Verdict")
|
|
details_pos = content.find("## Details")
|
|
assert verdict_pos != -1
|
|
assert details_pos != -1
|
|
assert verdict_pos < details_pos, "## Verdict must come before ## Details in tester.md"
|
|
|
|
def test_reviewer_details_status_field_documented(self):
|
|
"""Details JSON must document a 'verdict' field."""
|
|
content = _read_prompt("reviewer.md")
|
|
assert '"verdict"' in content, "reviewer.md Details must document 'verdict' field"
|
|
|
|
def test_tester_details_status_field_documented(self):
|
|
"""Details JSON must document a 'status' field."""
|
|
content = _read_prompt("tester.md")
|
|
assert '"status"' in content, "tester.md Details must document 'status' field"
|