|
7 | 7 | from mneme.cli import main |
8 | 8 | from mneme.decision_retriever import DecisionRetriever, ScoredDecision |
9 | 9 | from mneme.enforcer import EnforcementResult, Severity, Violation, check_prompt |
| 10 | +from mneme.memory_store import MemoryStore |
10 | 11 | from mneme.schemas import Decision |
11 | 12 |
|
| 13 | +EXAMPLE_MEMORY = Path(__file__).parent.parent / "examples" / "project_memory.json" |
| 14 | + |
12 | 15 |
|
13 | 16 | # ── shared fixtures ─────────────────────────────────────────────────────────── |
14 | 17 |
|
@@ -224,6 +227,165 @@ def test_severity_enum_values(): |
224 | 227 | assert Severity.FAIL == "FAIL" |
225 | 228 |
|
226 | 229 |
|
| 230 | +# ── H1 regression: migrated anti-pattern enforcement behavior ───────────────── |
| 231 | +# |
| 232 | +# Stage 1 (PR #21, squash 096b8be) moved legacy `anti_pattern.content` into |
| 233 | +# `Decision.constraints` instead of `Decision.rationale`. Side effect (H1): when |
| 234 | +# the migrated content begins with "No ..." it now matches the enforcer's |
| 235 | +# `^no\s+(.+)$` constraint regex and produces WARN-severity violations on input |
| 236 | +# tokens drawn from the content body. |
| 237 | +# |
| 238 | +# This is real, intended system behavior — constraints constrain. These tests |
| 239 | +# pin it so a future enforcer regex change, stopword tweak, or migration |
| 240 | +# refactor can't silently revert it without a failing test. |
| 241 | + |
| 242 | + |
| 243 | +def test_migrated_anti_pattern_with_no_prefix_content_warns_via_constraint( |
| 244 | + tmp_path, |
| 245 | +): |
| 246 | + """A legacy anti_pattern whose content starts with "No ..." migrates into |
| 247 | + `constraints` and triggers WARN-severity enforcement on terms drawn from |
| 248 | + that content body — via the enforcer's `^no\\s+` constraint pathway. |
| 249 | +
|
| 250 | + Pre-Stage-1 (content -> rationale), this WARN was impossible: the enforcer |
| 251 | + only inspects constraints + anti_patterns, never rationale. |
| 252 | + """ |
| 253 | + memory_file = tmp_path / "memory.json" |
| 254 | + memory_file.write_text(json.dumps({ |
| 255 | + "meta": {"name": "h1-test", "description": "h1 fixture"}, |
| 256 | + "items": [ |
| 257 | + { |
| 258 | + "id": "anti-h1", |
| 259 | + "type": "anti_pattern", |
| 260 | + "title": "Do not add background workers", |
| 261 | + "content": "No background workers, message queues, or daemon processes.", |
| 262 | + "tags": ["forbidden"], |
| 263 | + "priority": "high", |
| 264 | + }, |
| 265 | + ], |
| 266 | + "examples": [], |
| 267 | + })) |
| 268 | + store = MemoryStore(memory_file); store.load() |
| 269 | + decision = next(d for d in store.decisions() if d.id == "anti-h1") |
| 270 | + |
| 271 | + # Sanity: the migration shape Stage 1 fixed. |
| 272 | + assert decision.constraints == [ |
| 273 | + "No background workers, message queues, or daemon processes." |
| 274 | + ] |
| 275 | + assert decision.rationale == "" |
| 276 | + |
| 277 | + # Input uses tokens unique to the constraint body — none appear in the |
| 278 | + # title-derived anti_patterns entry "Do not add background workers" |
| 279 | + # (which tokenizes to {background, workers}). This isolates the WARN |
| 280 | + # constraint pathway from the FAIL anti_patterns pathway so the verdict |
| 281 | + # is purely WARN. |
| 282 | + scored = [_scored(decision)] |
| 283 | + result = check_prompt( |
| 284 | + "Add message queues and daemon processes for async work.", |
| 285 | + scored, |
| 286 | + ) |
| 287 | + assert result.verdict == Severity.WARN |
| 288 | + warn = next(v for v in result.violations if v.severity == Severity.WARN) |
| 289 | + assert warn.decision_id == "anti-h1" |
| 290 | + assert warn.rule.startswith("No background workers"), ( |
| 291 | + f"WARN must cite the migrated constraint; got rule={warn.rule!r}" |
| 292 | + ) |
| 293 | + assert warn.trigger in {"message", "queues", "daemon", "processes"} |
| 294 | + |
| 295 | + |
| 296 | +def test_migrated_anti_pattern_without_no_prefix_does_not_warn_via_constraint( |
| 297 | + tmp_path, |
| 298 | +): |
| 299 | + """Inverse boundary: when migrated content does NOT begin with "No ...", |
| 300 | + the enforcer's `^no\\s+` constraint pathway is correctly inert. The |
| 301 | + title-as-anti_pattern FAIL pathway is unaffected — this test isolates the |
| 302 | + constraint side specifically.""" |
| 303 | + memory_file = tmp_path / "memory.json" |
| 304 | + memory_file.write_text(json.dumps({ |
| 305 | + "meta": {"name": "h1-inverse-test", "description": "h1 inverse fixture"}, |
| 306 | + "items": [ |
| 307 | + { |
| 308 | + "id": "anti-h1-inv", |
| 309 | + "type": "anti_pattern", |
| 310 | + "title": "Some forbidden thing", |
| 311 | + # Content deliberately does NOT begin with "No ". |
| 312 | + "content": "Background workers and message queues add operational weight.", |
| 313 | + "tags": ["forbidden"], |
| 314 | + "priority": "high", |
| 315 | + }, |
| 316 | + ], |
| 317 | + "examples": [], |
| 318 | + })) |
| 319 | + store = MemoryStore(memory_file); store.load() |
| 320 | + decision = next(d for d in store.decisions() if d.id == "anti-h1-inv") |
| 321 | + assert decision.constraints == [ |
| 322 | + "Background workers and message queues add operational weight." |
| 323 | + ] |
| 324 | + |
| 325 | + scored = [_scored(decision)] |
| 326 | + result = check_prompt( |
| 327 | + "We could add a background worker queue to handle this.", |
| 328 | + scored, |
| 329 | + ) |
| 330 | + # No constraint-driven WARN: the only constraint doesn't begin with "No ". |
| 331 | + constraint_warns = [ |
| 332 | + v for v in result.violations |
| 333 | + if v.severity == Severity.WARN and v.decision_id == "anti-h1-inv" |
| 334 | + ] |
| 335 | + assert constraint_warns == [], ( |
| 336 | + f"Constraint without `^no\\s+` prefix must not produce WARN; " |
| 337 | + f"got {[v.rule for v in constraint_warns]}" |
| 338 | + ) |
| 339 | + |
| 340 | + |
| 341 | +def test_shipped_anti_002_warns_via_migrated_constraint_on_anchor_term(): |
| 342 | + """Pin H1 on the live shipped fixture: anti-002 ("Do not add agentic loops |
| 343 | + in v1") migrates into a constraint beginning with "No tool-use, function |
| 344 | + calling, or multi-turn agent loops…", which produces WARN enforcement on |
| 345 | + inputs containing anchor terms like "function calling" or "tool-use". |
| 346 | +
|
| 347 | + This is the real, observable system behavior introduced by Stage 1 |
| 348 | + (PR #21, squash 096b8be) and visible only via direct `check_prompt` |
| 349 | + (the shipped benchmark suite uses the structured Layer 2 path).""" |
| 350 | + store = MemoryStore(EXAMPLE_MEMORY); store.load() |
| 351 | + retriever = DecisionRetriever(store.decisions()) |
| 352 | + scored = retriever.retrieve( |
| 353 | + "Should we add multi-agent support to Mneme so it can coordinate between agents?" |
| 354 | + ) |
| 355 | + # anti-002 must be in the top-3 retrieval (Stage 1 locks rank 1 here via |
| 356 | + # tests/test_benchmark.py::test_feature_boundary_violation_retrieves_anti_002_at_rank_1). |
| 357 | + top3_ids = [s.decision.id for s in scored if s.score > 0][:3] |
| 358 | + assert "anti-002" in top3_ids, ( |
| 359 | + f"Test premise broken: anti-002 not in top-3; got {top3_ids}" |
| 360 | + ) |
| 361 | + |
| 362 | + # Anchor term "function calling" maps to constraint terms "function" and |
| 363 | + # "calling" via the enforcer's `_rule_terms` tokenizer. |
| 364 | + result = check_prompt( |
| 365 | + "Use function calling and a tool-use loop to coordinate workers.", |
| 366 | + scored, |
| 367 | + top=3, |
| 368 | + ) |
| 369 | + anti_002_warns = [ |
| 370 | + v for v in result.violations |
| 371 | + if v.severity == Severity.WARN and v.decision_id == "anti-002" |
| 372 | + ] |
| 373 | + assert anti_002_warns, ( |
| 374 | + f"Expected WARN from anti-002's migrated constraint; got violations=" |
| 375 | + f"{[(v.severity, v.decision_id, v.trigger) for v in result.violations]}" |
| 376 | + ) |
| 377 | + rule = anti_002_warns[0].rule |
| 378 | + assert rule.lower().startswith("no tool-use"), ( |
| 379 | + f"WARN must cite the migrated constraint beginning 'No tool-use…'; " |
| 380 | + f"got rule={rule!r}" |
| 381 | + ) |
| 382 | + assert anti_002_warns[0].trigger in { |
| 383 | + "tool", "function", "calling", "multi", "turn", "agent", "loops", |
| 384 | + "module", "mneme", "single", "call", "response", "pipeline", |
| 385 | + "agentic", "behaviour", "separate", "concern", "product", "layer", |
| 386 | + } |
| 387 | + |
| 388 | + |
227 | 389 | # ── CLI integration tests ───────────────────────────────────────────────────── |
228 | 390 |
|
229 | 391 | def test_check_cmd_pass_exits_zero(tmp_path): |
|
0 commit comments