refactor: consolidate skill docs, add anti-drift tests, and apply audit fixes
DLO-13: Restructure vlm-library-workflow skill as safety contract layer. - Rewrite SKILL.md (69 lines): safety contract, execution threshold semantics, six-step high-risk loop, decision rules, phase skeleton - Delete redundant references (cli-reference, workflow, command-recipes, dev-guide) - Add triage.md (failure mapping + preflight) and dev-map.md (module→test mapping) - Add tests/test_docs_consistency.py: 78 parametrized tests verifying documented vlm commands exist in CLI registry - Add CSV path mismatch test to test_plan_review.py (4th safety gate path) - Delete vlm-expert.skill (Gemini package, 7 months stale) and README Gemini section DLO-2 audit fixes: rate limiter injection, symmetric quarantine categories, review-plan safety gates, parser improvements, planner validation. CLI modularization: commands/ directory with one module per command group.
This commit is contained in:
@@ -0,0 +1,82 @@
|
||||
"""Verify vlm commands referenced in documentation exist in the CLI registry."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from click.testing import CliRunner
|
||||
|
||||
from vlm.cli import main
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
DOC_GLOBS = [
|
||||
"README.md",
|
||||
"CLAUDE.md",
|
||||
"AGENTS.md",
|
||||
"skills/**/*.md",
|
||||
]
|
||||
|
||||
COMMAND_PATTERN = re.compile(
|
||||
r"^(?:uv run )?(?:\$ )?vlm\s+(.+)$", re.MULTILINE
|
||||
)
|
||||
|
||||
|
||||
def _extract_doc_commands() -> list[str]:
|
||||
commands: list[str] = []
|
||||
for pattern in DOC_GLOBS:
|
||||
for path in REPO_ROOT.glob(pattern):
|
||||
text = path.read_text(encoding="utf-8")
|
||||
for match in COMMAND_PATTERN.finditer(text):
|
||||
raw = match.group(1).strip()
|
||||
raw = raw.rstrip("\\").strip()
|
||||
if raw.startswith("#") or not raw:
|
||||
continue
|
||||
commands.append(raw)
|
||||
return commands
|
||||
|
||||
|
||||
def _cli_command_names() -> dict[str, list[str]]:
|
||||
result: dict[str, list[str]] = {}
|
||||
for name, cmd in main.commands.items():
|
||||
result[name] = []
|
||||
if hasattr(cmd, "commands"):
|
||||
result[name] = list(cmd.commands.keys())
|
||||
return result
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw_cmd",
|
||||
sorted(set(_extract_doc_commands())),
|
||||
ids=lambda c: c[:60],
|
||||
)
|
||||
def test_documented_command_exists(raw_cmd: str):
|
||||
parts = raw_cmd.split()
|
||||
if not parts:
|
||||
pytest.skip("empty command")
|
||||
|
||||
first = parts[0]
|
||||
if first in ("--help", "-h"):
|
||||
return
|
||||
|
||||
registry = _cli_command_names()
|
||||
|
||||
if first not in registry:
|
||||
pytest.fail(f"Documented command 'vlm {first}' not in CLI registry")
|
||||
|
||||
if len(parts) > 1:
|
||||
sub = parts[1]
|
||||
if sub.startswith("-"):
|
||||
return
|
||||
subcommands = registry.get(first, [])
|
||||
if subcommands and sub not in subcommands:
|
||||
pytest.fail(
|
||||
f"Documented subcommand 'vlm {first} {sub}' "
|
||||
f"not in CLI registry (available: {subcommands})"
|
||||
)
|
||||
|
||||
|
||||
def test_fake_command_is_caught():
|
||||
registry = _cli_command_names()
|
||||
assert "nonexistent-command-xyz" not in registry
|
||||
Reference in New Issue
Block a user