260 lines
12 KiB
Python
260 lines
12 KiB
Python
|
"""Regression tests for KIN-DOCS-005 — prompt_engineer role for AI projects.
|
|||
|
|
|
|||
|
|
Acceptance criteria:
|
|||
|
|
1. specialists.yaml парсится без ошибок; роль prompt_engineer содержит все обязательные поля
|
|||
|
|
2. agents/prompts/prompt_engineer.md содержит ровно 5 обязательных секций в правильном порядке (#940)
|
|||
|
|
3. Роль prompt_engineer доступна в research department (departments.research.workers)
|
|||
|
|
4. Регрессионный тест на наличие роли в списке specialists
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
from pathlib import Path
|
|||
|
|
|
|||
|
|
import pytest
|
|||
|
|
import yaml
|
|||
|
|
|
|||
|
|
SPECIALISTS_YAML = Path(__file__).parent.parent / "agents" / "specialists.yaml"
|
|||
|
|
PROMPTS_DIR = Path(__file__).parent.parent / "agents" / "prompts"
|
|||
|
|
|
|||
|
|
REQUIRED_SECTIONS = [
|
|||
|
|
"## Working Mode",
|
|||
|
|
"## Focus On",
|
|||
|
|
"## Quality Checks",
|
|||
|
|
"## Return Format",
|
|||
|
|
"## Constraints",
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
OUTPUT_SCHEMA_FIELDS = [
|
|||
|
|
"status",
|
|||
|
|
"prompt_design",
|
|||
|
|
"quality_evaluation",
|
|||
|
|
"model_recommendation",
|
|||
|
|
"notes",
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _load_yaml():
|
|||
|
|
return yaml.safe_load(SPECIALISTS_YAML.read_text(encoding="utf-8"))
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ===========================================================================
|
|||
|
|
# 1. Структурный тест agents/specialists.yaml — роль prompt_engineer
|
|||
|
|
# ===========================================================================
|
|||
|
|
|
|||
|
|
class TestPromptEngineerSpecialists:
|
|||
|
|
"""Тесты регистрации prompt_engineer в agents/specialists.yaml."""
|
|||
|
|
|
|||
|
|
def test_role_exists_in_specialists(self):
|
|||
|
|
"""specialists.yaml содержит роль prompt_engineer."""
|
|||
|
|
data = _load_yaml()
|
|||
|
|
assert "prompt_engineer" in data.get("specialists", {}), (
|
|||
|
|
"prompt_engineer отсутствует в specialists.yaml"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_role_model_is_sonnet(self):
|
|||
|
|
"""prompt_engineer использует модель sonnet."""
|
|||
|
|
data = _load_yaml()
|
|||
|
|
role = data["specialists"]["prompt_engineer"]
|
|||
|
|
assert role.get("model") == "sonnet", (
|
|||
|
|
f"Ожидался model=sonnet, получили: {role.get('model')}"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_role_tools_include_read_grep_glob(self):
|
|||
|
|
"""prompt_engineer имеет инструменты Read, Grep, Glob."""
|
|||
|
|
data = _load_yaml()
|
|||
|
|
tools = data["specialists"]["prompt_engineer"].get("tools", [])
|
|||
|
|
for required_tool in ("Read", "Grep", "Glob"):
|
|||
|
|
assert required_tool in tools, (
|
|||
|
|
f"prompt_engineer должен иметь инструмент {required_tool!r}"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_role_has_no_write_tools(self):
|
|||
|
|
"""prompt_engineer НЕ имеет write-инструментов (read-only роль)."""
|
|||
|
|
data = _load_yaml()
|
|||
|
|
tools = set(data["specialists"]["prompt_engineer"].get("tools", []))
|
|||
|
|
write_tools = {"Write", "Edit", "Bash"}
|
|||
|
|
unexpected = write_tools & tools
|
|||
|
|
assert not unexpected, (
|
|||
|
|
f"prompt_engineer не должен иметь write-инструменты: {unexpected}"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_role_permissions_is_read_only(self):
|
|||
|
|
"""prompt_engineer имеет permissions=read_only."""
|
|||
|
|
data = _load_yaml()
|
|||
|
|
role = data["specialists"]["prompt_engineer"]
|
|||
|
|
assert role.get("permissions") == "read_only", (
|
|||
|
|
f"Ожидался permissions=read_only, получили: {role.get('permissions')}"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_role_has_output_schema(self):
|
|||
|
|
"""prompt_engineer имеет поле output_schema."""
|
|||
|
|
data = _load_yaml()
|
|||
|
|
role = data["specialists"]["prompt_engineer"]
|
|||
|
|
assert "output_schema" in role, (
|
|||
|
|
"prompt_engineer должен иметь output_schema"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
@pytest.mark.parametrize("field", OUTPUT_SCHEMA_FIELDS)
|
|||
|
|
def test_output_schema_has_required_field(self, field):
|
|||
|
|
"""output_schema содержит каждое из обязательных полей."""
|
|||
|
|
data = _load_yaml()
|
|||
|
|
schema = data["specialists"]["prompt_engineer"]["output_schema"]
|
|||
|
|
assert field in schema, (
|
|||
|
|
f"output_schema prompt_engineer не содержит обязательного поля {field!r}"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_yaml_parses_without_error(self):
|
|||
|
|
"""specialists.yaml парсится без ошибок (yaml.safe_load не бросает исключений)."""
|
|||
|
|
data = _load_yaml()
|
|||
|
|
assert isinstance(data, dict), "specialists.yaml не вернул dict при парсинге"
|
|||
|
|
assert "specialists" in data, "specialists.yaml не содержит секцию 'specialists'"
|
|||
|
|
|
|||
|
|
def test_role_context_rules_decisions_all(self):
|
|||
|
|
"""prompt_engineer получает все decisions (context_rules.decisions=all)."""
|
|||
|
|
data = _load_yaml()
|
|||
|
|
role = data["specialists"]["prompt_engineer"]
|
|||
|
|
decisions = role.get("context_rules", {}).get("decisions")
|
|||
|
|
assert decisions == "all", (
|
|||
|
|
f"Ожидался context_rules.decisions=all, получили: {decisions}"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ===========================================================================
|
|||
|
|
# 2. Структурный тест agents/prompts/prompt_engineer.md
|
|||
|
|
# ===========================================================================
|
|||
|
|
|
|||
|
|
class TestPromptEngineerPrompt:
|
|||
|
|
"""Структурный тест agents/prompts/prompt_engineer.md (#940)."""
|
|||
|
|
|
|||
|
|
def test_prompt_file_exists(self):
|
|||
|
|
"""Файл agents/prompts/prompt_engineer.md существует."""
|
|||
|
|
path = PROMPTS_DIR / "prompt_engineer.md"
|
|||
|
|
assert path.exists(), "prompt_engineer.md не найден в agents/prompts/"
|
|||
|
|
|
|||
|
|
@pytest.mark.parametrize("section", REQUIRED_SECTIONS)
|
|||
|
|
def test_prompt_has_required_section(self, section):
|
|||
|
|
"""Промпт содержит все 5 обязательных секций (REQUIRED_SECTIONS)."""
|
|||
|
|
content = (PROMPTS_DIR / "prompt_engineer.md").read_text(encoding="utf-8")
|
|||
|
|
assert section in content, (
|
|||
|
|
f"prompt_engineer.md не содержит обязательную секцию {section!r}"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_prompt_sections_in_correct_order(self):
|
|||
|
|
"""5 обязательных секций расположены в правильном порядке в prompt_engineer.md."""
|
|||
|
|
content = (PROMPTS_DIR / "prompt_engineer.md").read_text(encoding="utf-8")
|
|||
|
|
positions = [content.find(sec) for sec in REQUIRED_SECTIONS]
|
|||
|
|
assert all(p != -1 for p in positions), (
|
|||
|
|
"Не все 5 секций найдены в prompt_engineer.md"
|
|||
|
|
)
|
|||
|
|
assert positions == sorted(positions), (
|
|||
|
|
f"Секции расположены не по порядку. Позиции: "
|
|||
|
|
f"{dict(zip(REQUIRED_SECTIONS, positions))}"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_prompt_has_input_section(self):
|
|||
|
|
"""Промпт содержит секцию ## Input — агент-специфичная секция."""
|
|||
|
|
content = (PROMPTS_DIR / "prompt_engineer.md").read_text(encoding="utf-8")
|
|||
|
|
assert "## Input" in content, (
|
|||
|
|
"prompt_engineer.md не содержит секцию '## Input'"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_prompt_contains_blocked_protocol(self):
|
|||
|
|
"""Промпт содержит Blocked Protocol с инструкцией blocked_reason."""
|
|||
|
|
content = (PROMPTS_DIR / "prompt_engineer.md").read_text(encoding="utf-8")
|
|||
|
|
assert "blocked_reason" in content, (
|
|||
|
|
"prompt_engineer.md не содержит 'blocked_reason' — Blocked Protocol обязателен"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_prompt_no_legacy_output_format_header(self):
|
|||
|
|
"""Промпт НЕ содержит устаревшей секции '## Output format'."""
|
|||
|
|
content = (PROMPTS_DIR / "prompt_engineer.md").read_text(encoding="utf-8")
|
|||
|
|
assert "## Output format" not in content, (
|
|||
|
|
"prompt_engineer.md содержит устаревшую секцию '## Output format'"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_prompt_contains_prompt_design_field(self):
|
|||
|
|
"""Промпт упоминает поле prompt_design в Return Format."""
|
|||
|
|
content = (PROMPTS_DIR / "prompt_engineer.md").read_text(encoding="utf-8")
|
|||
|
|
assert "prompt_design" in content, (
|
|||
|
|
"prompt_engineer.md не содержит поля 'prompt_design'"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_prompt_contains_quality_evaluation_field(self):
|
|||
|
|
"""Промпт упоминает поле quality_evaluation в Return Format."""
|
|||
|
|
content = (PROMPTS_DIR / "prompt_engineer.md").read_text(encoding="utf-8")
|
|||
|
|
assert "quality_evaluation" in content, (
|
|||
|
|
"prompt_engineer.md не содержит поля 'quality_evaluation'"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_prompt_contains_model_recommendation_field(self):
|
|||
|
|
"""Промпт упоминает поле model_recommendation в Return Format."""
|
|||
|
|
content = (PROMPTS_DIR / "prompt_engineer.md").read_text(encoding="utf-8")
|
|||
|
|
assert "model_recommendation" in content, (
|
|||
|
|
"prompt_engineer.md не содержит поля 'model_recommendation'"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ===========================================================================
|
|||
|
|
# 3. Роль доступна в research department
|
|||
|
|
# ===========================================================================
|
|||
|
|
|
|||
|
|
class TestPromptEngineerInResearchDepartment:
|
|||
|
|
"""Тесты доступности prompt_engineer в departments.research."""
|
|||
|
|
|
|||
|
|
def test_research_department_exists(self):
|
|||
|
|
"""departments.research существует в specialists.yaml."""
|
|||
|
|
data = _load_yaml()
|
|||
|
|
assert "research" in data.get("departments", {}), (
|
|||
|
|
"departments.research отсутствует в specialists.yaml"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_prompt_engineer_in_research_workers(self):
|
|||
|
|
"""prompt_engineer присутствует в departments.research.workers."""
|
|||
|
|
data = _load_yaml()
|
|||
|
|
workers = data["departments"]["research"].get("workers", [])
|
|||
|
|
assert "prompt_engineer" in workers, (
|
|||
|
|
f"prompt_engineer должен быть в departments.research.workers. "
|
|||
|
|
f"Текущие workers: {workers}"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_research_head_describes_prompt_engineer(self):
|
|||
|
|
"""research_head description упоминает prompt_engineer."""
|
|||
|
|
data = _load_yaml()
|
|||
|
|
description = data["specialists"]["research_head"].get("description", "")
|
|||
|
|
assert "prompt_engineer" in description, (
|
|||
|
|
"research_head description должен упоминать prompt_engineer"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_research_workers_include_tech_researcher_and_architect(self):
|
|||
|
|
"""departments.research.workers по-прежнему содержит tech_researcher и architect (регрессия)."""
|
|||
|
|
data = _load_yaml()
|
|||
|
|
workers = data["departments"]["research"].get("workers", [])
|
|||
|
|
for existing_role in ("tech_researcher", "architect"):
|
|||
|
|
assert existing_role in workers, (
|
|||
|
|
f"Регрессия: {existing_role!r} пропал из departments.research.workers"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ===========================================================================
|
|||
|
|
# 4. Регрессионный тест: наличие роли в списке specialists
|
|||
|
|
# ===========================================================================
|
|||
|
|
|
|||
|
|
class TestPromptEngineerRoleRegistration:
|
|||
|
|
"""Регрессионный тест: prompt_engineer зарегистрирован в specialists."""
|
|||
|
|
|
|||
|
|
def test_prompt_engineer_in_specialists_list(self):
|
|||
|
|
"""prompt_engineer присутствует в секции specialists файла specialists.yaml."""
|
|||
|
|
data = _load_yaml()
|
|||
|
|
specialist_roles = list(data.get("specialists", {}).keys())
|
|||
|
|
assert "prompt_engineer" in specialist_roles, (
|
|||
|
|
f"prompt_engineer отсутствует в списке specialists. "
|
|||
|
|
f"Текущие роли: {specialist_roles}"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_prompt_engineer_not_in_exclusion_list(self):
|
|||
|
|
"""prompt_engineer.md не включён в EXCLUDED_FROM_STRUCTURE_CHECK."""
|
|||
|
|
from tests.test_kin_docs_002_regression import EXCLUDED_FROM_STRUCTURE_CHECK
|
|||
|
|
assert "prompt_engineer.md" not in EXCLUDED_FROM_STRUCTURE_CHECK, (
|
|||
|
|
"prompt_engineer.md не должен быть в EXCLUDED_FROM_STRUCTURE_CHECK — "
|
|||
|
|
"роль должна проходить все стандартные структурные проверки"
|
|||
|
|
)
|