Claude Code Memory Compiler

scripts/compile.py

@@ -0,0 +1,224 @@
+"""
+Compile daily conversation logs into structured knowledge articles.
+
+This is the "LLM compiler" - it reads daily logs (source code) and produces
+organized knowledge articles (the executable).
+
+Usage:
+    uv run python compile.py                    # compile new/changed logs only
+    uv run python compile.py --all              # force recompile everything
+    uv run python compile.py --file daily/2026-04-01.md  # compile a specific log
+    uv run python compile.py --dry-run          # show what would be compiled
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import sys
+from pathlib import Path
+
+from config import AGENTS_FILE, CONCEPTS_DIR, CONNECTIONS_DIR, DAILY_DIR, KNOWLEDGE_DIR, now_iso
+from utils import (
+    file_hash,
+    list_raw_files,
+    list_wiki_articles,
+    load_state,
+    read_wiki_index,
+    save_state,
+)
+
+# ── Paths for the LLM to use ──────────────────────────────────────────
+ROOT_DIR = Path(__file__).resolve().parent.parent
+
+
+async def compile_daily_log(log_path: Path, state: dict) -> float:
+    """Compile a single daily log into knowledge articles.
+
+    Returns the API cost of the compilation.
+    """
+    from claude_agent_sdk import (
+        AssistantMessage,
+        ClaudeAgentOptions,
+        ResultMessage,
+        TextBlock,
+        query,
+    )
+
+    log_content = log_path.read_text(encoding="utf-8")
+    schema = AGENTS_FILE.read_text(encoding="utf-8")
+    wiki_index = read_wiki_index()
+
+    # Read existing articles for context
+    existing_articles_context = ""
+    existing = {}
+    for article_path in list_wiki_articles():
+        rel = article_path.relative_to(KNOWLEDGE_DIR)
+        existing[str(rel)] = article_path.read_text(encoding="utf-8")
+
+    if existing:
+        parts = []
+        for rel_path, content in existing.items():
+            parts.append(f"### {rel_path}\n```markdown\n{content}\n```")
+        existing_articles_context = "\n\n".join(parts)
+
+    timestamp = now_iso()
+
+    prompt = f"""You are a knowledge compiler. Your job is to read a daily conversation log
+and extract knowledge into structured wiki articles.
+
+## Schema (AGENTS.md)
+
+{schema}
+
+## Current Wiki Index
+
+{wiki_index}
+
+## Existing Wiki Articles
+
+{existing_articles_context if existing_articles_context else "(No existing articles yet)"}
+
+## Daily Log to Compile
+
+**File:** {log_path.name}
+
+{log_content}
+
+## Your Task
+
+Read the daily log above and compile it into wiki articles following the schema exactly.
+
+### Rules:
+
+1. **Extract key concepts** - Identify 3-7 distinct concepts worth their own article
+2. **Create concept articles** in `knowledge/concepts/` - One .md file per concept
+   - Use the exact article format from AGENTS.md (YAML frontmatter + sections)
+   - Include `sources:` in frontmatter pointing to the daily log file
+   - Use `[[concepts/slug]]` wikilinks to link to related concepts
+   - Write in encyclopedia style - neutral, comprehensive
+3. **Create connection articles** in `knowledge/connections/` if this log reveals non-obvious
+   relationships between 2+ existing concepts
+4. **Update existing articles** if this log adds new information to concepts already in the wiki
+   - Read the existing article, add the new information, add the source to frontmatter
+5. **Update knowledge/index.md** - Add new entries to the table
+   - Each entry: `| [[path/slug]] | One-line summary | source-file | {timestamp[:10]} |`
+6. **Append to knowledge/log.md** - Add a timestamped entry:
+   ```
+   ## [{timestamp}] compile | {log_path.name}
+   - Source: daily/{log_path.name}
+   - Articles created: [[concepts/x]], [[concepts/y]]
+   - Articles updated: [[concepts/z]] (if any)
+   ```
+
+### File paths:
+- Write concept articles to: {CONCEPTS_DIR}
+- Write connection articles to: {CONNECTIONS_DIR}
+- Update index at: {KNOWLEDGE_DIR / 'index.md'}
+- Append log at: {KNOWLEDGE_DIR / 'log.md'}
+
+### Quality standards:
+- Every article must have complete YAML frontmatter
+- Every article must link to at least 2 other articles via [[wikilinks]]
+- Key Points section should have 3-5 bullet points
+- Details section should have 2+ paragraphs
+- Related Concepts section should have 2+ entries
+- Sources section should cite the daily log with specific claims extracted
+"""
+
+    cost = 0.0
+
+    try:
+        async for message in query(
+            prompt=prompt,
+            options=ClaudeAgentOptions(
+                cwd=str(ROOT_DIR),
+                system_prompt={"type": "preset", "preset": "claude_code"},
+                allowed_tools=["Read", "Write", "Edit", "Glob", "Grep"],
+                permission_mode="acceptEdits",
+                max_turns=30,
+            ),
+        ):
+            if isinstance(message, AssistantMessage):
+                for block in message.content:
+                    if isinstance(block, TextBlock):
+                        pass  # compilation output - LLM writes files directly
+            elif isinstance(message, ResultMessage):
+                cost = message.total_cost_usd or 0.0
+                print(f"  Cost: ${cost:.4f}")
+    except Exception as e:
+        print(f"  Error: {e}")
+        return 0.0
+
+    # Update state
+    rel_path = log_path.name
+    state.setdefault("ingested", {})[rel_path] = {
+        "hash": file_hash(log_path),
+        "compiled_at": now_iso(),
+        "cost_usd": cost,
+    }
+    state["total_cost"] = state.get("total_cost", 0.0) + cost
+    save_state(state)
+
+    return cost
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Compile daily logs into knowledge articles")
+    parser.add_argument("--all", action="store_true", help="Force recompile all logs")
+    parser.add_argument("--file", type=str, help="Compile a specific daily log file")
+    parser.add_argument("--dry-run", action="store_true", help="Show what would be compiled")
+    args = parser.parse_args()
+
+    state = load_state()
+
+    # Determine which files to compile
+    if args.file:
+        target = Path(args.file)
+        if not target.is_absolute():
+            target = DAILY_DIR / target.name
+        if not target.exists():
+            # Try resolving relative to project root
+            target = ROOT_DIR / args.file
+        if not target.exists():
+            print(f"Error: {args.file} not found")
+            sys.exit(1)
+        to_compile = [target]
+    else:
+        all_logs = list_raw_files()
+        if args.all:
+            to_compile = all_logs
+        else:
+            to_compile = []
+            for log_path in all_logs:
+                rel = log_path.name
+                prev = state.get("ingested", {}).get(rel, {})
+                if not prev or prev.get("hash") != file_hash(log_path):
+                    to_compile.append(log_path)
+
+    if not to_compile:
+        print("Nothing to compile - all daily logs are up to date.")
+        return
+
+    print(f"{'[DRY RUN] ' if args.dry_run else ''}Files to compile ({len(to_compile)}):")
+    for f in to_compile:
+        print(f"  - {f.name}")
+
+    if args.dry_run:
+        return
+
+    # Compile each file sequentially
+    total_cost = 0.0
+    for i, log_path in enumerate(to_compile, 1):
+        print(f"\n[{i}/{len(to_compile)}] Compiling {log_path.name}...")
+        cost = asyncio.run(compile_daily_log(log_path, state))
+        total_cost += cost
+        print(f"  Done.")
+
+    articles = list_wiki_articles()
+    print(f"\nCompilation complete. Total cost: ${total_cost:.2f}")
+    print(f"Knowledge base: {len(articles)} articles")
+
+
+if __name__ == "__main__":
+    main()