309 lines
12 KiB
Python
309 lines
12 KiB
Python
|
"""Regression tests for KIN-102:
|
||
|
|
Legacy test_command='make test' blocks autocommit in projects without Makefile.
|
||
|
|
|
||
|
|
Root cause: projects with legacy test_command='make test' (old schema default) fail the
|
||
|
|
auto-test runner because the project has no Makefile. This triggers an auto-fix loop
|
||
|
|
that eventually marks the pipeline as failed + task as blocked, causing an early return
|
||
|
|
in run_pipeline BEFORE _run_autocommit is reached.
|
||
|
|
|
||
|
|
Fix: migration in core/db.py resets test_command='make test' → NULL for all projects,
|
||
|
|
allowing auto-detection to find the real test framework or skip tests gracefully.
|
||
|
|
|
||
|
|
Coverage:
|
||
|
|
(1) Migration: test_command='make test' → NULL (KIN-102 fix)
|
||
|
|
(2) Migration: other test_command values are NOT reset
|
||
|
|
(3) Bug scenario: legacy 'make test' → pipeline blocked, autocommit skipped
|
||
|
|
(4) Post-fix: NULL test_command + no framework → pipeline succeeds + autocommit called
|
||
|
|
"""
|
||
|
|
|
||
|
|
import json
|
||
|
|
import os
|
||
|
|
import pytest
|
||
|
|
from unittest.mock import patch, MagicMock
|
||
|
|
|
||
|
|
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
# Fixtures
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
def conn():
|
||
|
|
from core.db import init_db
|
||
|
|
from core import models
|
||
|
|
c = init_db(":memory:")
|
||
|
|
models.create_project(
|
||
|
|
c, "sharedbox", "SharedBox", "~/projects/sharedbox",
|
||
|
|
tech_stack=["typescript", "node"],
|
||
|
|
)
|
||
|
|
models.create_task(
|
||
|
|
c, "SHAREDBOX-003", "sharedbox", "Setup DB",
|
||
|
|
brief={"route_type": "backend_dev"},
|
||
|
|
)
|
||
|
|
yield c
|
||
|
|
c.close()
|
||
|
|
|
||
|
|
|
||
|
|
def _mock_agent_success():
|
||
|
|
m = MagicMock()
|
||
|
|
m.returncode = 0
|
||
|
|
m.stdout = json.dumps({"status": "done", "changes": [], "notes": ""})
|
||
|
|
m.stderr = ""
|
||
|
|
return m
|
||
|
|
|
||
|
|
|
||
|
|
def _mock_make_test_fail():
|
||
|
|
return {
|
||
|
|
"success": False,
|
||
|
|
"output": "make: No rule to make target `test'. Stop.",
|
||
|
|
"returncode": 2,
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
# (1) Migration: test_command='make test' → NULL
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
class TestKin102Migration:
|
||
|
|
def test_legacy_make_test_reset_to_null(self):
|
||
|
|
"""Migration resets test_command='make test' → NULL (KIN-102 fix)."""
|
||
|
|
from core.db import init_db, _migrate
|
||
|
|
from core import models
|
||
|
|
|
||
|
|
c = init_db(":memory:")
|
||
|
|
models.create_project(c, "sbx", "SharedBox", "~/projects/sharedbox",
|
||
|
|
tech_stack=["typescript"])
|
||
|
|
c.execute("UPDATE projects SET test_command='make test', auto_test_enabled=1 WHERE id='sbx'")
|
||
|
|
c.commit()
|
||
|
|
|
||
|
|
# Verify pre-condition: legacy value is set
|
||
|
|
row = c.execute("SELECT test_command FROM projects WHERE id='sbx'").fetchone()
|
||
|
|
assert row[0] == "make test", "Pre-condition: test_command must be 'make test' before migration"
|
||
|
|
|
||
|
|
# Run migration
|
||
|
|
_migrate(c)
|
||
|
|
|
||
|
|
# After migration: test_command must be NULL
|
||
|
|
row = c.execute("SELECT test_command FROM projects WHERE id='sbx'").fetchone()
|
||
|
|
assert row[0] is None, f"Expected NULL after migration, got {row[0]!r}"
|
||
|
|
c.close()
|
||
|
|
|
||
|
|
def test_npm_test_not_reset_by_migration(self):
|
||
|
|
"""Migration must NOT reset test_command='npm test' (explicitly set by user)."""
|
||
|
|
from core.db import init_db, _migrate
|
||
|
|
from core import models
|
||
|
|
|
||
|
|
c = init_db(":memory:")
|
||
|
|
models.create_project(c, "sbx", "SharedBox", "~/projects/sharedbox",
|
||
|
|
tech_stack=["typescript"])
|
||
|
|
c.execute("UPDATE projects SET test_command='npm test' WHERE id='sbx'")
|
||
|
|
c.commit()
|
||
|
|
|
||
|
|
_migrate(c)
|
||
|
|
|
||
|
|
row = c.execute("SELECT test_command FROM projects WHERE id='sbx'").fetchone()
|
||
|
|
assert row[0] == "npm test", "Migration must not change test_command='npm test'"
|
||
|
|
c.close()
|
||
|
|
|
||
|
|
def test_pytest_not_reset_by_migration(self):
|
||
|
|
"""Migration must NOT reset test_command='pytest' (explicitly set by user)."""
|
||
|
|
from core.db import init_db, _migrate
|
||
|
|
from core import models
|
||
|
|
|
||
|
|
c = init_db(":memory:")
|
||
|
|
models.create_project(c, "sbx", "SharedBox", "~/projects/sharedbox",
|
||
|
|
tech_stack=["python"])
|
||
|
|
c.execute("UPDATE projects SET test_command='pytest' WHERE id='sbx'")
|
||
|
|
c.commit()
|
||
|
|
|
||
|
|
_migrate(c)
|
||
|
|
|
||
|
|
row = c.execute("SELECT test_command FROM projects WHERE id='sbx'").fetchone()
|
||
|
|
assert row[0] == "pytest", "Migration must not change test_command='pytest'"
|
||
|
|
c.close()
|
||
|
|
|
||
|
|
def test_null_test_command_stays_null_after_migration(self):
|
||
|
|
"""Migration must NOT affect projects that already have test_command=NULL."""
|
||
|
|
from core.db import init_db, _migrate
|
||
|
|
from core import models
|
||
|
|
|
||
|
|
c = init_db(":memory:")
|
||
|
|
models.create_project(c, "sbx", "SharedBox", "~/projects/sharedbox",
|
||
|
|
tech_stack=["typescript"])
|
||
|
|
# Default test_command is NULL — don't change it
|
||
|
|
|
||
|
|
_migrate(c)
|
||
|
|
|
||
|
|
row = c.execute("SELECT test_command FROM projects WHERE id='sbx'").fetchone()
|
||
|
|
assert row[0] is None, "Migration must not touch NULL test_command"
|
||
|
|
c.close()
|
||
|
|
|
||
|
|
def test_migration_is_idempotent(self):
|
||
|
|
"""Running migration twice must not cause errors or data corruption."""
|
||
|
|
from core.db import init_db, _migrate
|
||
|
|
from core import models
|
||
|
|
|
||
|
|
c = init_db(":memory:")
|
||
|
|
models.create_project(c, "sbx", "SharedBox", "~/projects/sharedbox",
|
||
|
|
tech_stack=["typescript"])
|
||
|
|
c.execute("UPDATE projects SET test_command='make test' WHERE id='sbx'")
|
||
|
|
c.commit()
|
||
|
|
|
||
|
|
_migrate(c)
|
||
|
|
_migrate(c) # second run must be safe
|
||
|
|
|
||
|
|
row = c.execute("SELECT test_command FROM projects WHERE id='sbx'").fetchone()
|
||
|
|
assert row[0] is None
|
||
|
|
c.close()
|
||
|
|
|
||
|
|
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
# (2) Bug scenario: legacy 'make test' → pipeline blocked, autocommit not called
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
class TestKin102BugScenario:
|
||
|
|
@patch("agents.runner._run_autocommit")
|
||
|
|
@patch("agents.runner._run_project_tests")
|
||
|
|
@patch("agents.runner.subprocess.run")
|
||
|
|
def test_legacy_make_test_blocks_task(
|
||
|
|
self, mock_run, mock_tests, mock_autocommit, conn
|
||
|
|
):
|
||
|
|
"""Bug: legacy test_command='make test' on project without Makefile → task blocked."""
|
||
|
|
from agents.runner import run_pipeline
|
||
|
|
from core import models
|
||
|
|
|
||
|
|
models.update_project(conn, "sharedbox",
|
||
|
|
auto_test_enabled=True,
|
||
|
|
test_command="make test")
|
||
|
|
mock_tests.return_value = _mock_make_test_fail()
|
||
|
|
mock_run.return_value = _mock_agent_success()
|
||
|
|
|
||
|
|
with patch.dict(os.environ, {"KIN_AUTO_TEST_MAX_ATTEMPTS": "0"}):
|
||
|
|
result = run_pipeline(conn, "SHAREDBOX-003",
|
||
|
|
[{"role": "backend_dev", "brief": "setup db"}])
|
||
|
|
|
||
|
|
assert result["success"] is False
|
||
|
|
task = models.get_task(conn, "SHAREDBOX-003")
|
||
|
|
assert task["status"] == "blocked"
|
||
|
|
|
||
|
|
@patch("agents.runner._run_autocommit")
|
||
|
|
@patch("agents.runner._run_project_tests")
|
||
|
|
@patch("agents.runner.subprocess.run")
|
||
|
|
def test_legacy_make_test_skips_autocommit(
|
||
|
|
self, mock_run, mock_tests, mock_autocommit, conn
|
||
|
|
):
|
||
|
|
"""Bug: when pipeline fails due to legacy 'make test', _run_autocommit is never called."""
|
||
|
|
from agents.runner import run_pipeline
|
||
|
|
from core import models
|
||
|
|
|
||
|
|
models.update_project(conn, "sharedbox",
|
||
|
|
auto_test_enabled=True,
|
||
|
|
test_command="make test")
|
||
|
|
mock_tests.return_value = _mock_make_test_fail()
|
||
|
|
mock_run.return_value = _mock_agent_success()
|
||
|
|
|
||
|
|
with patch.dict(os.environ, {"KIN_AUTO_TEST_MAX_ATTEMPTS": "0"}):
|
||
|
|
run_pipeline(conn, "SHAREDBOX-003",
|
||
|
|
[{"role": "backend_dev", "brief": "setup db"}])
|
||
|
|
|
||
|
|
# Early return before _run_autocommit — never reached
|
||
|
|
mock_autocommit.assert_not_called()
|
||
|
|
|
||
|
|
@patch("agents.runner._run_autocommit")
|
||
|
|
@patch("agents.runner._run_project_tests")
|
||
|
|
@patch("agents.runner.subprocess.run")
|
||
|
|
def test_blocked_reason_mentions_make_test(
|
||
|
|
self, mock_run, mock_tests, mock_autocommit, conn
|
||
|
|
):
|
||
|
|
"""Blocked reason must mention 'make test' for diagnosability."""
|
||
|
|
from agents.runner import run_pipeline
|
||
|
|
from core import models
|
||
|
|
|
||
|
|
models.update_project(conn, "sharedbox",
|
||
|
|
auto_test_enabled=True,
|
||
|
|
test_command="make test")
|
||
|
|
mock_tests.return_value = _mock_make_test_fail()
|
||
|
|
mock_run.return_value = _mock_agent_success()
|
||
|
|
|
||
|
|
with patch.dict(os.environ, {"KIN_AUTO_TEST_MAX_ATTEMPTS": "0"}):
|
||
|
|
result = run_pipeline(conn, "SHAREDBOX-003",
|
||
|
|
[{"role": "backend_dev", "brief": "setup db"}])
|
||
|
|
|
||
|
|
assert "make test" in result.get("error", ""), (
|
||
|
|
f"Blocked reason must contain 'make test', got: {result.get('error')!r}"
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
# (3) Post-fix: NULL test_command + no framework → pipeline succeeds + autocommit called
|
||
|
|
# ---------------------------------------------------------------------------
|
||
|
|
|
||
|
|
class TestKin102PostFix:
|
||
|
|
@patch("agents.runner._run_autocommit")
|
||
|
|
@patch("agents.runner._detect_test_command")
|
||
|
|
@patch("agents.runner._run_project_tests")
|
||
|
|
@patch("agents.runner.subprocess.run")
|
||
|
|
def test_null_test_command_no_framework_pipeline_succeeds(
|
||
|
|
self, mock_run, mock_tests, mock_detect, mock_autocommit, conn
|
||
|
|
):
|
||
|
|
"""After fix: test_command=NULL, no framework detected → pipeline succeeds."""
|
||
|
|
from agents.runner import run_pipeline
|
||
|
|
from core import models
|
||
|
|
|
||
|
|
# Simulate post-migration state: test_command reset to NULL
|
||
|
|
models.update_project(conn, "sharedbox",
|
||
|
|
auto_test_enabled=True,
|
||
|
|
test_command=None)
|
||
|
|
mock_run.return_value = _mock_agent_success()
|
||
|
|
mock_detect.return_value = None # No test framework found (no Makefile/package.json)
|
||
|
|
|
||
|
|
result = run_pipeline(conn, "SHAREDBOX-003",
|
||
|
|
[{"role": "backend_dev", "brief": "setup db"}])
|
||
|
|
|
||
|
|
assert result["success"] is True
|
||
|
|
mock_tests.assert_not_called()
|
||
|
|
|
||
|
|
@patch("agents.runner._run_autocommit")
|
||
|
|
@patch("agents.runner._detect_test_command")
|
||
|
|
@patch("agents.runner._run_project_tests")
|
||
|
|
@patch("agents.runner.subprocess.run")
|
||
|
|
def test_null_test_command_no_framework_autocommit_called(
|
||
|
|
self, mock_run, mock_tests, mock_detect, mock_autocommit, conn
|
||
|
|
):
|
||
|
|
"""After fix: test_command=NULL + no framework → _run_autocommit IS called."""
|
||
|
|
from agents.runner import run_pipeline
|
||
|
|
from core import models
|
||
|
|
|
||
|
|
models.update_project(conn, "sharedbox",
|
||
|
|
auto_test_enabled=True,
|
||
|
|
test_command=None)
|
||
|
|
mock_run.return_value = _mock_agent_success()
|
||
|
|
mock_detect.return_value = None
|
||
|
|
|
||
|
|
run_pipeline(conn, "SHAREDBOX-003",
|
||
|
|
[{"role": "backend_dev", "brief": "setup db"}])
|
||
|
|
|
||
|
|
# After fix: no early return → autocommit is reached
|
||
|
|
mock_autocommit.assert_called_once()
|
||
|
|
|
||
|
|
@patch("agents.runner._run_autocommit")
|
||
|
|
@patch("agents.runner._detect_test_command")
|
||
|
|
@patch("agents.runner._run_project_tests")
|
||
|
|
@patch("agents.runner.subprocess.run")
|
||
|
|
def test_null_test_command_triggers_autodetect(
|
||
|
|
self, mock_run, mock_tests, mock_detect, mock_autocommit, conn
|
||
|
|
):
|
||
|
|
"""After fix: test_command=NULL → _detect_test_command is called for auto-detection."""
|
||
|
|
from agents.runner import run_pipeline
|
||
|
|
from core import models
|
||
|
|
|
||
|
|
models.update_project(conn, "sharedbox",
|
||
|
|
auto_test_enabled=True,
|
||
|
|
test_command=None)
|
||
|
|
mock_run.return_value = _mock_agent_success()
|
||
|
|
mock_detect.return_value = None
|
||
|
|
|
||
|
|
run_pipeline(conn, "SHAREDBOX-003",
|
||
|
|
[{"role": "backend_dev", "brief": "setup db"}])
|
||
|
|
|
||
|
|
mock_detect.assert_called_once()
|