Don't scroll to bottm when sending message on mobile

web/.agents/skills/caveman-compress/scripts/__init__.py

@@ -0,0 +1,9 @@
+"""Caveman compress scripts.
+
+This package provides tools to compress natural language markdown files
+into caveman format to save input tokens.
+"""
+
+__all__ = ["cli", "compress", "detect", "validate"]
+
+__version__ = "1.0.0"

web/.agents/skills/caveman-compress/scripts/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+main()

web/.agents/skills/caveman-compress/scripts/benchmark.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+from pathlib import Path
+import sys
+
+# Support both direct execution and module import
+try:
+    from .validate import validate
+except ImportError:
+    sys.path.insert(0, str(Path(__file__).parent))
+    from validate import validate
+
+try:
+    import tiktoken
+    _enc = tiktoken.get_encoding("o200k_base")
+except ImportError:
+    _enc = None
+
+
+def count_tokens(text):
+    if _enc is None:
+        return len(text.split())  # fallback: word count
+    return len(_enc.encode(text))
+
+
+def benchmark_pair(orig_path: Path, comp_path: Path):
+    orig_text = orig_path.read_text(encoding="utf-8", errors="ignore")
+    comp_text = comp_path.read_text(encoding="utf-8", errors="ignore")
+
+    orig_tokens = count_tokens(orig_text)
+    comp_tokens = count_tokens(comp_text)
+    saved = 100 * (orig_tokens - comp_tokens) / orig_tokens if orig_tokens > 0 else 0.0
+    result = validate(orig_path, comp_path)
+
+    return (comp_path.name, orig_tokens, comp_tokens, saved, result.is_valid)
+
+
+def print_table(rows):
+    print("\n| File | Original | Compressed | Saved % | Valid |")
+    print("|------|----------|------------|---------|-------|")
+    for r in rows:
+        print(f"| {r[0]} | {r[1]} | {r[2]} | {r[3]:.1f}% | {'✅' if r[4] else '❌'} |")
+
+
+def main():
+    # Direct file pair: python3 benchmark.py original.md compressed.md
+    if len(sys.argv) == 3:
+        orig = Path(sys.argv[1]).resolve()
+        comp = Path(sys.argv[2]).resolve()
+        if not orig.exists():
+            print(f"❌ Not found: {orig}")
+            sys.exit(1)
+        if not comp.exists():
+            print(f"❌ Not found: {comp}")
+            sys.exit(1)
+        print_table([benchmark_pair(orig, comp)])
+        return
+
+    # Glob mode: repo_root/tests/caveman-compress/
+    # __file__ lives at <repo_root>/skills/caveman-compress/scripts/benchmark.py
+    # Walk up four dirs: scripts → caveman-compress → skills → repo_root.
+    tests_dir = Path(__file__).resolve().parents[3] / "tests" / "caveman-compress"
+    if not tests_dir.exists():
+        print(f"❌ Tests dir not found: {tests_dir}")
+        sys.exit(1)
+
+    rows = []
+    for orig in sorted(tests_dir.glob("*.original.md")):
+        comp = orig.with_name(orig.stem.removesuffix(".original") + ".md")
+        if comp.exists():
+            rows.append(benchmark_pair(orig, comp))
+
+    if not rows:
+        print("No compressed file pairs found.")
+        return
+
+    print_table(rows)
+
+
+if __name__ == "__main__":
+    main()

web/.agents/skills/caveman-compress/scripts/cli.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+"""
+Caveman Compress CLI
+
+Usage:
+    caveman <filepath>
+"""
+
+import sys
+
+# Force UTF-8 on stdout/stderr before any code can print. Windows consoles
+# default to cp1252 and crash on the ❌ glyphs in error/validation branches,
+# masking the real error and leaving the user with a half-compressed file.
+for _stream in (sys.stdout, sys.stderr):
+    reconfigure = getattr(_stream, "reconfigure", None)
+    if callable(reconfigure):
+        try:
+            reconfigure(encoding="utf-8", errors="replace")
+        except Exception:
+            pass
+
+from pathlib import Path
+
+from .compress import backup_dir_for, compress_file
+from .detect import detect_file_type, should_compress
+
+
+def print_usage():
+    print("Usage: caveman <filepath>")
+
+
+def main():
+    if len(sys.argv) != 2:
+        print_usage()
+        sys.exit(1)
+
+    filepath = Path(sys.argv[1])
+
+    # Check file exists
+    if not filepath.exists():
+        print(f"❌ File not found: {filepath}")
+        sys.exit(1)
+
+    if not filepath.is_file():
+        print(f"❌ Not a file: {filepath}")
+        sys.exit(1)
+
+    filepath = filepath.resolve()
+
+    # Detect file type
+    file_type = detect_file_type(filepath)
+
+    print(f"Detected: {file_type}")
+
+    # Check if compressible
+    if not should_compress(filepath):
+        print("Skipping: file is not natural language (code/config)")
+        sys.exit(0)
+
+    print("Starting caveman compression...\n")
+
+    try:
+        success = compress_file(filepath)
+
+        if success:
+            print("\nCompression completed successfully")
+            backup_path = backup_dir_for(filepath) / (filepath.stem + ".original.md")
+            print(f"Compressed: {filepath}")
+            print(f"Original:   {backup_path}")
+            sys.exit(0)
+        else:
+            print("\n❌ Compression failed after retries")
+            sys.exit(2)
+
+    except KeyboardInterrupt:
+        print("\nInterrupted by user")
+        sys.exit(130)
+
+    except Exception as e:
+        print(f"\n❌ Error: {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

web/.agents/skills/caveman-compress/scripts/compress.py

@@ -0,0 +1,414 @@
+#!/usr/bin/env python3
+"""
+Caveman Memory Compression Orchestrator
+
+Usage:
+    python scripts/compress.py <filepath>
+"""
+
+import os
+import re
+import shutil
+import stat
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+from typing import List
+
+OUTER_FENCE_REGEX = re.compile(
+    r"\A\s*(`{3,}|~{3,})[^\n]*\n(.*)\n\1\s*\Z", re.DOTALL
+)
+
+# YAML frontmatter: starts at file start with --- on its own line, ends with --- on its own line.
+# Captures the entire block (including delimiters and trailing newline) and the body after.
+FRONTMATTER_REGEX = re.compile(
+    r"\A(---\r?\n.*?\r?\n---\r?\n)(.*)", re.DOTALL
+)
+
+
+def split_frontmatter(text: str):
+    """Split YAML frontmatter from body. Returns (frontmatter, body).
+
+    Memory files (and many other markdown docs) start with a YAML frontmatter
+    block delimited by `---` lines. The compression LLM has a habit of stripping
+    or rewriting these despite preserve-structure rules in the prompt — so we
+    surgically remove the frontmatter before compression and prepend it back
+    verbatim to the output. Files without frontmatter pass through unchanged.
+    """
+    m = FRONTMATTER_REGEX.match(text)
+    if m:
+        return m.group(1), m.group(2)
+    return "", text
+
+# Filenames and paths that almost certainly hold secrets or PII. Compressing
+# them ships raw bytes to the Anthropic API — a third-party data boundary that
+# developers on sensitive codebases cannot cross. detect.py already skips .env
+# by extension, but credentials.md / secrets.txt / ~/.aws/credentials would
+# slip through the natural-language filter. This is a hard refuse before read.
+SENSITIVE_BASENAME_REGEX = re.compile(
+    r"(?ix)^("
+    r"\.env(\..+)?"
+    r"|\.netrc"
+    r"|credentials(\..+)?"
+    r"|secrets?(\..+)?"
+    r"|passwords?(\..+)?"
+    r"|id_(rsa|dsa|ecdsa|ed25519)(\.pub)?"
+    r"|authorized_keys"
+    r"|known_hosts"
+    r"|.*\.(pem|key|p12|pfx|crt|cer|jks|keystore|asc|gpg)"
+    r")$"
+)
+
+SENSITIVE_PATH_COMPONENTS = frozenset({".ssh", ".aws", ".gnupg", ".kube", ".docker"})
+
+SENSITIVE_NAME_TOKENS = (
+    "secret", "credential", "password", "passwd",
+    "apikey", "accesskey", "token", "privatekey",
+)
+
+
+def backup_dir_for(filepath: Path) -> Path:
+    """Resolve the out-of-tree backup directory for a given source file.
+
+    Backups must live OUTSIDE the source directory so skill auto-loaders
+    (Claude Code rules/, opencode instructions/, etc.) stop re-ingesting the
+    `.original.md` copies as live files. Base dir is platform-aware:
+      - Windows: %LOCALAPPDATA%\\caveman-compress\\backups
+      - else:    $XDG_DATA_HOME/caveman-compress/backups if set,
+                 else ~/.local/share/caveman-compress/backups
+
+    The source file's parent-dir name is mirrored under the base to reduce
+    cross-project collisions (e.g. two `task.md` files in different repos).
+    """
+    if os.name == "nt" or sys.platform == "win32":
+        local_appdata = os.environ.get("LOCALAPPDATA")
+        base = Path(local_appdata) if local_appdata else Path.home() / "AppData" / "Local"
+        base = base / "caveman-compress" / "backups"
+    else:
+        xdg = os.environ.get("XDG_DATA_HOME")
+        base = Path(xdg) if xdg else Path.home() / ".local" / "share"
+        base = base / "caveman-compress" / "backups"
+    return base / filepath.parent.name
+
+
+def is_sensitive_path(filepath: Path) -> bool:
+    """Heuristic denylist for files that must never be shipped to a third-party API."""
+    name = filepath.name
+    if SENSITIVE_BASENAME_REGEX.match(name):
+        return True
+    lowered_parts = {p.lower() for p in filepath.parts}
+    if lowered_parts & SENSITIVE_PATH_COMPONENTS:
+        return True
+    # Normalize separators so "api-key" and "api_key" both match "apikey".
+    lower = re.sub(r"[_\-\s.]", "", name.lower())
+    return any(tok in lower for tok in SENSITIVE_NAME_TOKENS)
+
+
+def strip_llm_wrapper(text: str) -> str:
+    """Strip outer ```markdown ... ``` fence when it wraps the entire output."""
+    m = OUTER_FENCE_REGEX.match(text)
+    if m:
+        return m.group(2)
+    return text
+
+
+def write_text_atomic(path: Path, text: str) -> None:
+    """Write ``text`` to ``path`` atomically as UTF-8.
+
+    Path.write_text() truncates the destination before encoding the string —
+    a UnicodeEncodeError (or any other failure) partway through leaves a
+    0-byte file, destroying whatever was there before (issue #655). Encode
+    first, write the bytes to a sibling temp file, fsync, then os.replace()
+    so the destination only ever moves from one complete, valid file to
+    another. Preserves the original file's permission bits across the swap.
+    """
+    data = text.encode("utf-8")
+    fd, tmp_name = tempfile.mkstemp(
+        dir=str(path.parent), prefix=path.name + ".", suffix=".tmp"
+    )
+    tmp_path = Path(tmp_name)
+    try:
+        with os.fdopen(fd, "wb") as f:
+            f.write(data)
+            f.flush()
+            os.fsync(f.fileno())
+        if path.exists():
+            os.chmod(tmp_path, stat.S_IMODE(path.stat().st_mode))
+        os.replace(tmp_path, path)
+    except Exception:
+        try:
+            tmp_path.unlink()
+        except OSError:
+            pass
+        raise
+
+
+def first_nonblank_line(text: str) -> str:
+    """Return the first non-blank line, stripped — used to detect a prose
+    preamble smuggled in ahead of the real content (issue #588)."""
+    for line in text.splitlines():
+        if line.strip():
+            return line.strip()
+    return ""
+
+
+def _write_target(filepath: Path, text: str, backup_path: Path) -> None:
+    """Write to the target file, surfacing the backup location if the write
+    itself fails. write_text_atomic already leaves the target untouched on
+    failure, but the caller still needs to know where the pre-compression
+    original lives instead of being left to guess (issue #652)."""
+    try:
+        write_text_atomic(filepath, text)
+    except Exception:
+        print(f"❌ Write to {filepath} failed. Original preserved at backup: {backup_path}")
+        raise
+
+
+from .detect import should_compress
+from .validate import validate
+
+MAX_RETRIES = 2
+
+
+# ---------- Claude Calls ----------
+
+
+def call_claude(prompt: str) -> str:
+    """Send a prompt to Claude.
+
+    Prefers the Anthropic SDK when ANTHROPIC_API_KEY is set; otherwise falls
+    back to the ``claude --print`` CLI (which handles desktop auth).
+
+    On Windows the CLI subprocess decoding defaults to the system codepage
+    (cp1251 / cp1252) and crashes on UTF-8 output — see issue #152. Pinning
+    ``encoding="utf-8"`` with ``errors="replace"`` matches the CLI's actual
+    native I/O and prevents the UnicodeDecodeError before validation can
+    report. Windows users with non-ASCII content can also set
+    ``ANTHROPIC_API_KEY`` to route through the SDK and skip the subprocess.
+    """
+    api_key = os.environ.get("ANTHROPIC_API_KEY")
+    if api_key:
+        try:
+            import anthropic
+
+            client = anthropic.Anthropic(api_key=api_key)
+            msg = client.messages.create(
+                model=os.environ.get("CAVEMAN_MODEL", "claude-sonnet-4-5"),
+                max_tokens=8192,
+                messages=[{"role": "user", "content": prompt}],
+            )
+            return strip_llm_wrapper(msg.content[0].text.strip())
+        except ImportError:
+            pass  # anthropic not installed, fall back to CLI
+    # Fallback: use claude CLI (handles desktop auth).
+    # Resolve binary via shutil.which so Windows .cmd/.bat shims (e.g.
+    # %APPDATA%\npm\claude.CMD) work without shell=True. On POSIX,
+    # shutil.which returns the same absolute path as the implicit lookup,
+    # so this is a no-op there. Falls back to bare "claude" if not found
+    # on PATH so subprocess raises a clear FileNotFoundError.
+    claude_bin = shutil.which("claude") or "claude"
+    try:
+        result = subprocess.run(
+            [claude_bin, "--print"],
+            input=prompt,
+            text=True,
+            capture_output=True,
+            check=True,
+            encoding="utf-8",
+            errors="replace",
+        )
+        return strip_llm_wrapper(result.stdout.strip())
+    except subprocess.CalledProcessError as e:
+        raise RuntimeError(f"Claude call failed:\n{e.stderr}")
+
+
+def build_compress_prompt(original: str) -> str:
+    return f"""
+Compress this markdown into caveman format.
+
+STRICT RULES:
+- Do NOT modify anything inside ``` code blocks
+- Do NOT modify anything inside inline backticks
+- Preserve ALL URLs exactly
+- Preserve ALL headings exactly
+- Preserve file paths and commands
+- Return ONLY the compressed markdown body — do NOT wrap the entire output in a ```markdown fence or any other fence. Inner code blocks from the original stay as-is; do not add a new outer fence around the whole file.
+
+Only compress natural language.
+
+TEXT:
+{original}
+"""
+
+
+def build_fix_prompt(original: str, compressed: str, errors: List[str]) -> str:
+    errors_str = "\n".join(f"- {e}" for e in errors)
+    return f"""You are fixing a caveman-compressed markdown file. Specific validation errors were found.
+
+CRITICAL RULES:
+- DO NOT recompress or rephrase the file
+- ONLY fix the listed errors — leave everything else exactly as-is
+- The ORIGINAL is provided as reference only (to restore missing content)
+- Preserve caveman style in all untouched sections
+
+ERRORS TO FIX:
+{errors_str}
+
+HOW TO FIX:
+- Missing URL: find it in ORIGINAL, restore it exactly where it belongs in COMPRESSED
+- Code block mismatch: find the exact code block in ORIGINAL, restore it in COMPRESSED
+- Heading mismatch: restore the exact heading text from ORIGINAL into COMPRESSED
+- Do not touch any section not mentioned in the errors
+
+ORIGINAL (reference only):
+{original}
+
+COMPRESSED (fix this):
+{compressed}
+
+Return ONLY the fixed compressed file. No explanation.
+"""
+
+
+# ---------- Core Logic ----------
+
+
+def compress_file(filepath: Path) -> bool:
+    # Resolve and validate path
+    filepath = filepath.resolve()
+    MAX_FILE_SIZE = 500_000  # 500KB
+    if not filepath.exists():
+        raise FileNotFoundError(f"File not found: {filepath}")
+    if filepath.stat().st_size > MAX_FILE_SIZE:
+        raise ValueError(f"File too large to compress safely (max 500KB): {filepath}")
+
+    # Refuse files that look like they contain secrets or PII. Compressing ships
+    # the raw bytes to the Anthropic API — a third-party boundary — so we fail
+    # loudly rather than silently exfiltrate credentials or keys. Override is
+    # intentional: the user must rename the file if the heuristic is wrong.
+    if is_sensitive_path(filepath):
+        raise ValueError(
+            f"Refusing to compress {filepath}: filename looks sensitive "
+            "(credentials, keys, secrets, or known private paths). "
+            "Compression sends file contents to the Anthropic API. "
+            "Rename the file if this is a false positive."
+        )
+
+    print(f"Processing: {filepath}")
+
+    if not should_compress(filepath):
+        print("Skipping (not natural language)")
+        return False
+
+    original_text = filepath.read_text(encoding="utf-8", errors="ignore")
+    # Store backup outside the source directory so skill auto-loaders don't
+    # re-ingest the `.original.md` copy as a live file. Mirror the source's
+    # parent-dir name + stem under a platform-aware base to reduce collisions.
+    backup_dir = backup_dir_for(filepath)
+    backup_path = backup_dir / (filepath.stem + ".original.md")
+
+    if not original_text.strip():
+        print("❌ Refusing to compress: file is empty or whitespace-only.")
+        return False
+
+    # Check if backup already exists to prevent accidental overwriting
+    if backup_path.exists():
+        print(f"⚠️ Backup file already exists: {backup_path}")
+        print("The original backup may contain important content.")
+        print("Aborting to prevent data loss. Please remove or rename the backup file if you want to proceed.")
+        return False
+
+    # Split YAML frontmatter off before compression. Claude tends to strip or
+    # rewrite frontmatter despite preserve-structure rules; we keep it verbatim
+    # by removing it from the input and re-prepending it to the output.
+    frontmatter, body = split_frontmatter(original_text)
+    if frontmatter:
+        print(f"Detected YAML frontmatter ({len(frontmatter)} chars) — preserving verbatim")
+
+    if not body.strip():
+        print("❌ Refusing to compress: body is empty after frontmatter removal.")
+        return False
+
+    # Step 1: Compress (body only, frontmatter excluded)
+    print("Compressing with Claude...")
+    compressed_body = call_claude(build_compress_prompt(body))
+
+    if compressed_body is None or not compressed_body.strip():
+        print("❌ Compression aborted: Claude returned an empty response.")
+        print("   Original file is untouched (no backup created).")
+        return False
+
+    # Compare the BODY (not the whole file) — frontmatter is preserved verbatim
+    # and would never change, so identity must be judged on the compressible part.
+    if compressed_body.strip() == body.strip():
+        print("❌ Compression aborted: output is identical to input.")
+        print("   Likely causes: Claude refused, returned the prompt verbatim, or the file is")
+        print("   already in caveman form. Original file is untouched (no backup created).")
+        return False
+
+    # Reassemble: frontmatter (verbatim) + compressed body
+    compressed = frontmatter + compressed_body
+
+    # Save original as backup, then verify the backup readback before
+    # touching the input file. If the filesystem dropped bytes (encoding,
+    # antivirus, disk full), unlink the bad backup and abort instead of
+    # leaving the user with a corrupt backup + compressed primary.
+    backup_dir.mkdir(parents=True, exist_ok=True)
+    write_text_atomic(backup_path, original_text)
+    backup_readback = backup_path.read_text(encoding="utf-8", errors="ignore")
+    if backup_readback != original_text:
+        print(f"❌ Backup write verification failed: {backup_path}")
+        print("   In-memory original differs from on-disk backup. Aborting before touching the input file.")
+        try:
+            backup_path.unlink()
+        except OSError:
+            pass
+        return False
+    _write_target(filepath, compressed, backup_path)
+
+    # Step 2: Validate + Retry
+    for attempt in range(MAX_RETRIES):
+        print(f"\nValidation attempt {attempt + 1}")
+
+        result = validate(backup_path, filepath)
+
+        if result.is_valid:
+            print("Validation passed")
+            break
+
+        print("❌ Validation failed:")
+        for err in result.errors:
+            print(f"   - {err}")
+
+        if attempt == MAX_RETRIES - 1:
+            # Restore original on failure
+            _write_target(filepath, original_text, backup_path)
+            backup_path.unlink(missing_ok=True)
+            print("❌ Failed after retries — original restored")
+            return False
+
+        print("Fixing with Claude...")
+        compressed = call_claude(
+            build_fix_prompt(original_text, compressed, result.errors)
+        )
+
+        if compressed is None or not compressed.strip():
+            print("❌ Fix attempt aborted: Claude returned an empty response.")
+            print("   Skipping this attempt.")
+            continue
+
+        # Guard against a prose preamble smuggled in ahead of the real fixed
+        # content (issue #588). Only enforced when the original starts with a
+        # structural anchor (frontmatter `---` or a heading) — plain-prose
+        # first lines get legitimately rewritten by compression, and requiring
+        # them verbatim would reject every valid fix.
+        anchor = first_nonblank_line(original_text)
+        if anchor.startswith(("---", "#")) and first_nonblank_line(compressed) != anchor:
+            print("❌ Fix attempt aborted: output does not start with the original's first line.")
+            print("   Possible preamble leak. Skipping this attempt.")
+            continue
+
+        _write_target(filepath, compressed, backup_path)
+
+    return True

web/.agents/skills/caveman-compress/scripts/detect.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+"""Detect whether a file is natural language (compressible) or code/config (skip)."""
+
+import json
+import re
+from pathlib import Path
+
+# Extensions that are natural language and compressible
+COMPRESSIBLE_EXTENSIONS = {".md", ".txt", ".markdown", ".rst", ".typ", ".typst", ".tex"}
+
+# Extensions that are code/config and should be skipped
+SKIP_EXTENSIONS = {
+    ".py", ".js", ".ts", ".tsx", ".jsx", ".json", ".yaml", ".yml",
+    ".toml", ".env", ".lock", ".css", ".scss", ".html", ".xml",
+    ".sql", ".sh", ".bash", ".zsh", ".go", ".rs", ".java", ".c",
+    ".cpp", ".h", ".hpp", ".rb", ".php", ".swift", ".kt", ".lua",
+    ".dockerfile", ".makefile", ".csv", ".ini", ".cfg",
+}
+
+# Well-known build/config files that carry no (or a misleading) extension —
+# `Dockerfile` has no suffix so `.dockerfile` above never matches it, and
+# `CMakeLists.txt` would ride the compressible `.txt` rule. Checked by
+# basename before any extension rule.
+KNOWN_CODE_FILENAMES = {
+    "dockerfile", "makefile", "gnumakefile", "jenkinsfile", "vagrantfile",
+    "rakefile", "gemfile", "justfile", "procfile", "brewfile",
+    "cmakelists.txt",
+}
+
+# Patterns that indicate a line is code
+CODE_PATTERNS = [
+    re.compile(r"^\s*(import |from .+ import |require\(|const |let |var )"),
+    re.compile(r"^\s*(def |class |function |async function |export )"),
+    re.compile(r"^\s*(if\s*\(|for\s*\(|while\s*\(|switch\s*\(|try\s*\{)"),
+    re.compile(r"^\s*[\}\]\);]+\s*$"),  # closing braces/brackets
+    re.compile(r"^\s*@\w+"),  # decorators/annotations
+    re.compile(r'^\s*"[^"]+"\s*:\s*'),  # JSON-like key-value
+    re.compile(r"^\s*\w+\s*=\s*[{\[\(\"']"),  # assignment with literal
+]
+
+
+def _is_code_line(line: str) -> bool:
+    """Check if a line looks like code."""
+    return any(p.match(line) for p in CODE_PATTERNS)
+
+
+def _is_json_content(text: str) -> bool:
+    """Check if content is valid JSON."""
+    try:
+        json.loads(text)
+        return True
+    except (json.JSONDecodeError, ValueError):
+        return False
+
+
+def _is_yaml_content(lines: list[str]) -> bool:
+    """Heuristic: check if content looks like YAML."""
+    yaml_indicators = 0
+    for line in lines[:30]:
+        stripped = line.strip()
+        if stripped.startswith("---"):
+            yaml_indicators += 1
+        elif re.match(r"^\w[\w\s]*:\s", stripped):
+            yaml_indicators += 1
+        elif stripped.startswith("- ") and ":" in stripped:
+            yaml_indicators += 1
+    # If most non-empty lines look like YAML
+    non_empty = sum(1 for l in lines[:30] if l.strip())
+    return non_empty > 0 and yaml_indicators / non_empty > 0.6
+
+
+def detect_file_type(filepath: Path) -> str:
+    """Classify a file as 'natural_language', 'code', 'config', or 'unknown'.
+
+    Returns:
+        One of: 'natural_language', 'code', 'config', 'unknown'
+    """
+    ext = filepath.suffix.lower()
+
+    # Known code filenames win over any extension rule
+    if filepath.name.lower() in KNOWN_CODE_FILENAMES:
+        return "code"
+
+    # Extension-based classification
+    if ext in COMPRESSIBLE_EXTENSIONS:
+        return "natural_language"
+    if ext in SKIP_EXTENSIONS:
+        return "code" if ext not in {".json", ".yaml", ".yml", ".toml", ".ini", ".cfg", ".env"} else "config"
+
+    # Extensionless files (like CLAUDE.md, TODO) — check content
+    if not ext:
+        try:
+            text = filepath.read_text(encoding="utf-8", errors="ignore")
+        except (OSError, PermissionError):
+            return "unknown"
+
+        lines = text.splitlines()[:50]
+
+        # Shebang means executable script, never prose
+        if text.startswith("#!"):
+            return "code"
+
+        if _is_json_content(text[:10000]):
+            return "config"
+        if _is_yaml_content(lines):
+            return "config"
+
+        code_lines = sum(1 for l in lines if l.strip() and _is_code_line(l))
+        non_empty = sum(1 for l in lines if l.strip())
+        if non_empty > 0 and code_lines / non_empty > 0.4:
+            return "code"
+
+        return "natural_language"
+
+    return "unknown"
+
+
+def should_compress(filepath: Path) -> bool:
+    """Return True if the file is natural language and should be compressed."""
+    if not filepath.is_file():
+        return False
+    # Skip backup files
+    if filepath.name.endswith(".original.md"):
+        return False
+    return detect_file_type(filepath) == "natural_language"
+
+
+if __name__ == "__main__":
+    import sys
+
+    if len(sys.argv) < 2:
+        print("Usage: python detect.py <file1> [file2] ...")
+        sys.exit(1)
+
+    for path_str in sys.argv[1:]:
+        p = Path(path_str).resolve()
+        file_type = detect_file_type(p)
+        compress = should_compress(p)
+        print(f"  {p.name:30s} type={file_type:20s} compress={compress}")

web/.agents/skills/caveman-compress/scripts/validate.py

@@ -0,0 +1,272 @@
+#!/usr/bin/env python3
+import re
+from collections import Counter
+from pathlib import Path
+
+URL_REGEX = re.compile(r"https?://[^\s)]+")
+FENCE_OPEN_REGEX = re.compile(r"^(\s{0,3})(`{3,}|~{3,})(.*)$")
+
+# A line that is nothing but a fence marker plus an optional info string, at ANY
+# indentation. Used ONLY to scrub leaked markers before inline-code pairing (see
+# extract_inline_codes) — never for block extraction.
+#
+# Widening FENCE_OPEN_REGEX itself to `\s*` looks like the obvious fix for #820
+# and is a net regression: a lone indented ``` (the natural way to SHOW a fence
+# inside prose) then opens a block that runs to EOF, swallowing real code blocks
+# and silently removing their inline spans from validation. That turns a
+# false-failure bug into a false-PASS bug, and a false PASS overwrites the
+# user's file with unvalidated output.
+FENCE_MARKER_LINE_REGEX = re.compile(r"^\s*(?:`{3,}|~{3,})[^`~]*$")
+
+# Cap on how much of a lost/added span is echoed in an error message. Unpaired
+# backticks can make a "span" hundreds of characters of prose; printing it whole
+# is what made #820's failures undiagnosable.
+MAX_REPORTED_SPAN = 60
+HEADING_REGEX = re.compile(r"^(#{1,6})\s+(.*)", re.MULTILINE)
+BULLET_REGEX = re.compile(r"^\s*[-*+]\s+", re.MULTILINE)
+
+# crude but effective path detection
+# Requires either a path prefix (./ ../ / or drive letter) or a slash/backslash within the match
+PATH_REGEX = re.compile(r"(?:\./|\.\./|/|[A-Za-z]:\\)[\w\-/\\\.]+|[\w\-\.]+[/\\][\w\-/\\\.]+")
+
+
+class ValidationResult:
+    def __init__(self):
+        self.is_valid = True
+        self.errors = []
+        self.warnings = []
+
+    def add_error(self, msg):
+        self.is_valid = False
+        self.errors.append(msg)
+
+    def add_warning(self, msg):
+        self.warnings.append(msg)
+
+
+def read_file(path: Path) -> str:
+    return path.read_text(encoding="utf-8")
+
+
+# ---------- Extractors ----------
+
+
+def extract_headings(text):
+    return [(level, title.strip()) for level, title in HEADING_REGEX.findall(text)]
+
+
+def extract_code_blocks(text):
+    """Line-based fenced code block extractor.
+
+    Handles ``` and ~~~ fences with variable length (CommonMark: closing
+    fence must use same char and be at least as long as opening). Supports
+    nested fences (e.g. an outer 4-backtick block wrapping inner 3-backtick
+    content).
+    """
+    blocks = []
+    lines = text.split("\n")
+    i = 0
+    n = len(lines)
+    while i < n:
+        m = FENCE_OPEN_REGEX.match(lines[i])
+        if not m:
+            i += 1
+            continue
+        fence_char = m.group(2)[0]
+        fence_len = len(m.group(2))
+        open_line = lines[i]
+        block_lines = [open_line]
+        i += 1
+        closed = False
+        while i < n:
+            close_m = FENCE_OPEN_REGEX.match(lines[i])
+            if (
+                close_m
+                and close_m.group(2)[0] == fence_char
+                and len(close_m.group(2)) >= fence_len
+                and close_m.group(3).strip() == ""
+            ):
+                block_lines.append(lines[i])
+                closed = True
+                i += 1
+                break
+            block_lines.append(lines[i])
+            i += 1
+        if closed:
+            blocks.append("\n".join(block_lines))
+        # Unclosed fences are silently skipped — they indicate malformed markdown
+        # and including them would cause false-positive validation failures.
+    return blocks
+
+
+def extract_urls(text):
+    return set(URL_REGEX.findall(text))
+
+
+def extract_paths(text):
+    return set(PATH_REGEX.findall(text))
+
+
+def count_bullets(text):
+    return len(BULLET_REGEX.findall(text))
+
+
+def extract_inline_codes(text):
+    """Backtick-delimited inline spans, with fenced code blocks stripped first.
+
+    Previously used a column-0-anchored regex to strip fences, which misses
+    fences indented 1-3 spaces (valid CommonMark). Reuse extract_code_blocks
+    (FENCE_OPEN_REGEX-based, indentation-aware) instead so an indented fence's
+    body backticks don't leak into inline-code pairing.
+
+    Any fence-marker line that survives that pass is then blanked (#820). A
+    fence indented 4+ spaces — what you get from showing an example inside a
+    bullet — is not matched by FENCE_OPEN_REGEX, so extract_code_blocks does
+    not remove it and its OWN backticks used to leak in and shift the pairing
+    of every following span, making the file permanently uncompressible.
+    Blanking just the marker lines fixes that without removing any prose, and
+    cannot run away the way a widened fence opener does.
+
+    The span pattern deliberately still spans newlines. CommonMark permits a
+    line ending inside a code span and hard-wrapped markdown produces them, so
+    a single-line pattern silently drops real spans — which downgrades a
+    deleted or mutated span from error to PASS. Long/garbled spans are a
+    presentation problem, handled by truncating in the error message instead.
+    """
+    text_without_fences = text
+    for block in extract_code_blocks(text):
+        text_without_fences = text_without_fences.replace(block, "", 1)
+    text_without_fences = "\n".join(
+        "" if FENCE_MARKER_LINE_REGEX.match(line) else line
+        for line in text_without_fences.split("\n")
+    )
+    return re.findall(r"`([^`]+)`", text_without_fences)
+
+
+# ---------- Validators ----------
+
+
+def validate_headings(orig, comp, result):
+    h1 = extract_headings(orig)
+    h2 = extract_headings(comp)
+
+    if len(h1) != len(h2):
+        result.add_error(f"Heading count mismatch: {len(h1)} vs {len(h2)}")
+
+    if h1 != h2:
+        result.add_warning("Heading text/order changed")
+
+
+def validate_code_blocks(orig, comp, result):
+    c1 = extract_code_blocks(orig)
+    c2 = extract_code_blocks(comp)
+
+    if c1 != c2:
+        result.add_error("Code blocks not preserved exactly")
+
+
+def validate_urls(orig, comp, result):
+    u1 = extract_urls(orig)
+    u2 = extract_urls(comp)
+
+    if u1 != u2:
+        result.add_error(f"URL mismatch: lost={u1 - u2}, added={u2 - u1}")
+
+
+def validate_paths(orig, comp, result):
+    p1 = extract_paths(orig)
+    p2 = extract_paths(comp)
+
+    if p1 != p2:
+        result.add_warning(f"Path mismatch: lost={p1 - p2}, added={p2 - p1}")
+
+
+def validate_bullets(orig, comp, result):
+    b1 = count_bullets(orig)
+    b2 = count_bullets(comp)
+
+    if b1 == 0:
+        return
+
+    diff = abs(b1 - b2) / b1
+
+    if diff > 0.15:
+        result.add_warning(f"Bullet count changed too much: {b1} -> {b2}")
+
+
+def validate_inline_codes(orig, comp, result):
+    def _render_spans(spans):
+        """Render spans for an error message, truncated and newline-escaped.
+
+        A span may legitimately contain newlines, and an unpaired backtick can
+        make one hundreds of characters of prose. Printing those whole is what
+        made #820's failures undiagnosable — but the fix belongs here, in
+        presentation, not in what counts as a span.
+        """
+        out = []
+        for span in sorted(spans):
+            flat = span.replace("\n", "\\n")
+            if len(flat) > MAX_REPORTED_SPAN:
+                flat = flat[:MAX_REPORTED_SPAN] + "…"
+            out.append(repr(flat))
+        return "{" + ", ".join(out) + "}"
+
+    c1 = Counter(extract_inline_codes(orig))
+    c2 = Counter(extract_inline_codes(comp))
+
+    if c1 != c2:
+        lost = set(c1.keys()) - set(c2.keys())
+        added = set(c2.keys()) - set(c1.keys())
+        for code, count in c1.items():
+            if code in c2 and c2[code] < count:
+                lost.add(f"{code} (lost {count - c2[code]} of {count} occurrences)")
+        if lost:
+            result.add_error(f"Inline code lost: {_render_spans(lost)}")
+        if added:
+            result.add_warning(f"Inline code added: {_render_spans(added)}")
+
+
+# ---------- Main ----------
+
+
+def validate(original_path: Path, compressed_path: Path) -> ValidationResult:
+    result = ValidationResult()
+
+    orig = read_file(original_path)
+    comp = read_file(compressed_path)
+
+    validate_headings(orig, comp, result)
+    validate_code_blocks(orig, comp, result)
+    validate_urls(orig, comp, result)
+    validate_paths(orig, comp, result)
+    validate_bullets(orig, comp, result)
+    validate_inline_codes(orig, comp, result)
+
+    return result
+
+
+# ---------- CLI ----------
+
+if __name__ == "__main__":
+    import sys
+
+    if len(sys.argv) != 3:
+        print("Usage: python validate.py <original> <compressed>")
+        sys.exit(1)
+
+    orig = Path(sys.argv[1]).resolve()
+    comp = Path(sys.argv[2]).resolve()
+
+    res = validate(orig, comp)
+
+    print(f"\nValid: {res.is_valid}")
+
+    if res.errors:
+        print("\nErrors:")
+        for e in res.errors:
+            print(f"  - {e}")
+
+    if res.warnings:
+        print("\nWarnings:")
+        for w in res.warnings:
+            print(f"  - {w}")