add reproducible worked examples for httpx, karpathy-repos, and mixed-corpus benchmarks

worked/httpx/README.md

@@ -0,0 +1,42 @@
+# httpx Corpus Benchmark — How to Reproduce
+
+A synthetic 6-file Python codebase modeled after httpx's architecture. Tests graphify
+on a realistic library codebase with clean layering: exceptions → models → auth/transport → client.
+
+## Corpus (6 files)
+
+All input files are in `raw/`:
+
+```
+raw/
+├── exceptions.py   — full HTTPError hierarchy (RequestError, TransportError, HTTPStatusError, etc.)
+├── models.py       — URL, Headers, Cookies, Request, Response with raise_for_status
+├── auth.py         — BasicAuth, BearerAuth, DigestAuth (challenge-response), NetRCAuth
+├── utils.py        — header normalization, query param flattening, content-type parsing
+├── transport.py    — ConnectionPool, HTTPTransport, AsyncHTTPTransport, MockTransport, ProxyTransport
+└── client.py       — Timeout, Limits, BaseClient, Client (sync), AsyncClient
+```
+
+## How to run
+
+```bash
+pip install graphifyy && graphify install
+/graphify ./raw
+```
+
+Or from the CLI directly:
+
+```bash
+pip install graphifyy
+graphify ./raw
+```
+
+## What to expect
+
+- ~95 nodes, ~130 edges
+- 4 communities: Exception Hierarchy, Models & Data, Auth & Transport, Client Layer
+- God nodes: `client.py`, `models.py`, `transport.py`, `exceptions.py`, `BaseClient`, `Response`
+- Surprising connections: `DigestAuth` ↔ `Response` (auth.py reads Response to parse WWW-Authenticate)
+- All edges EXTRACTED — no inference needed, dependency graph is explicit
+
+Full eval with scores and analysis: `review.md`

worked/httpx/raw/auth.py

@@ -0,0 +1,114 @@
+"""
+Authentication handlers.
+Auth objects are callables that modify a request before it is sent.
+DigestAuth is the most interesting: it participates in a full request/response cycle,
+reading the 401 response to build the challenge before re-sending.
+"""
+import hashlib
+import time
+from models import Request, Response
+
+
+class Auth:
+    """Base class for all authentication handlers."""
+
+    def auth_flow(self, request: Request):
+        """Modify the request. May yield to inspect the response."""
+        raise NotImplementedError
+
+
+class BasicAuth(Auth):
+    """HTTP Basic Authentication."""
+
+    def __init__(self, username: str, password: str):
+        self.username = username
+        self.password = password
+
+    def auth_flow(self, request: Request):
+        import base64
+        credentials = f"{self.username}:{self.password}".encode()
+        encoded = base64.b64encode(credentials).decode()
+        request.headers["Authorization"] = f"Basic {encoded}"
+        yield request
+
+
+class BearerAuth(Auth):
+    """Bearer token authentication."""
+
+    def __init__(self, token: str):
+        self.token = token
+
+    def auth_flow(self, request: Request):
+        request.headers["Authorization"] = f"Bearer {self.token}"
+        yield request
+
+
+class DigestAuth(Auth):
+    """
+    HTTP Digest Authentication.
+    Requires a full request/response cycle: sends the initial request,
+    reads the 401 WWW-Authenticate header, then re-sends with credentials.
+    This is the only auth handler that reads from Response.
+    """
+
+    def __init__(self, username: str, password: str):
+        self.username = username
+        self.password = password
+        self._nonce_count = 0
+
+    def auth_flow(self, request: Request):
+        yield request  # first attempt, no credentials
+
+        # This handler must inspect the Response to continue
+        response = yield
+
+        if response.status_code == 401:
+            challenge = self._parse_challenge(response)
+            credentials = self._build_credentials(request, challenge)
+            request.headers["Authorization"] = credentials
+            yield request
+
+    def _parse_challenge(self, response: Response) -> dict:
+        """Extract digest parameters from the WWW-Authenticate header."""
+        header = response.headers.get("www-authenticate", "")
+        params = {}
+        for part in header.replace("Digest ", "").split(","):
+            if "=" in part:
+                key, _, value = part.strip().partition("=")
+                params[key.strip()] = value.strip().strip('"')
+        return params
+
+    def _build_credentials(self, request: Request, challenge: dict) -> str:
+        """Compute the Authorization header value for a digest challenge."""
+        self._nonce_count += 1
+        nc = f"{self._nonce_count:08x}"
+        cnonce = hashlib.md5(str(time.time()).encode()).hexdigest()[:8]
+        realm = challenge.get("realm", "")
+        nonce = challenge.get("nonce", "")
+
+        ha1 = hashlib.md5(f"{self.username}:{realm}:{self.password}".encode()).hexdigest()
+        ha2 = hashlib.md5(f"{request.method}:{request.url.path}".encode()).hexdigest()
+        response_hash = hashlib.md5(f"{ha1}:{nonce}:{nc}:{cnonce}:auth:{ha2}".encode()).hexdigest()
+
+        return (
+            f'Digest username="{self.username}", realm="{realm}", '
+            f'nonce="{nonce}", uri="{request.url.path}", '
+            f'nc={nc}, cnonce="{cnonce}", response="{response_hash}"'
+        )
+
+
+class NetRCAuth(Auth):
+    """Load credentials from ~/.netrc based on the request host."""
+
+    def auth_flow(self, request: Request):
+        import netrc
+        try:
+            credentials = netrc.netrc().authenticators(request.url.host)
+            if credentials:
+                username, _, password = credentials
+                basic = BasicAuth(username, password)
+                yield from basic.auth_flow(request)
+                return
+        except Exception:
+            pass
+        yield request

worked/httpx/raw/client.py

@@ -0,0 +1,161 @@
+"""
+The main Client and AsyncClient classes.
+BaseClient holds all shared logic. Client and AsyncClient extend it for sync/async.
+This is the integration hub of the library - it imports from every other module.
+"""
+from models import Request, Response, URL, Headers, Cookies
+from auth import Auth, BasicAuth
+from transport import BaseTransport, HTTPTransport, AsyncHTTPTransport
+from exceptions import TooManyRedirects, InvalidURL
+from utils import build_url_with_params, obfuscate_sensitive_headers
+
+
+DEFAULT_MAX_REDIRECTS = 20
+
+
+class Timeout:
+    def __init__(self, timeout=5.0, *, connect=None, read=None, write=None, pool=None):
+        self.connect = connect or timeout
+        self.read = read or timeout
+        self.write = write or timeout
+        self.pool = pool or timeout
+
+
+class Limits:
+    def __init__(self, max_connections=100, max_keepalive_connections=20, keepalive_expiry=5.0):
+        self.max_connections = max_connections
+        self.max_keepalive_connections = max_keepalive_connections
+        self.keepalive_expiry = keepalive_expiry
+
+
+class BaseClient:
+    """
+    Shared implementation for Client and AsyncClient.
+    Handles auth, redirects, cookies, and header defaults.
+    """
+
+    def __init__(
+        self,
+        *,
+        auth=None,
+        headers=None,
+        cookies=None,
+        timeout=Timeout(),
+        max_redirects=DEFAULT_MAX_REDIRECTS,
+        base_url="",
+    ):
+        self._auth = auth
+        self._headers = Headers(headers or {})
+        self._cookies = Cookies(cookies or {})
+        self._timeout = timeout
+        self._max_redirects = max_redirects
+        self._base_url = URL(base_url) if base_url else None
+
+    def _build_request(self, method: str, url: str, **kwargs) -> Request:
+        if self._base_url:
+            url = self._base_url.raw.rstrip("/") + "/" + url.lstrip("/")
+        if kwargs.get("params"):
+            url = build_url_with_params(url, kwargs.pop("params"))
+        headers = Headers(kwargs.get("headers", {}))
+        for k, v in self._headers.items():
+            if k not in headers:
+                headers[k] = v
+        return Request(method, url, headers=headers, content=kwargs.get("content"), cookies=self._cookies)
+
+    def _merge_cookies(self, response: Response) -> None:
+        for name, value in response.cookies.items():
+            self._cookies.set(name, value)
+
+
+class Client(BaseClient):
+    """Synchronous HTTP client."""
+
+    def __init__(self, *, transport: BaseTransport = None, **kwargs):
+        super().__init__(**kwargs)
+        self._transport = transport or HTTPTransport()
+
+    def request(self, method: str, url: str, **kwargs) -> Response:
+        request = self._build_request(method, url, **kwargs)
+        auth = kwargs.get("auth") or self._auth
+        if auth:
+            flow = auth.auth_flow(request)
+            request = next(flow)
+        response = self._transport.handle_request(request)
+        self._merge_cookies(response)
+        if auth:
+            try:
+                flow.send(response)
+            except StopIteration:
+                pass
+        return response
+
+    def get(self, url: str, **kwargs) -> Response:
+        return self.request("GET", url, **kwargs)
+
+    def post(self, url: str, **kwargs) -> Response:
+        return self.request("POST", url, **kwargs)
+
+    def put(self, url: str, **kwargs) -> Response:
+        return self.request("PUT", url, **kwargs)
+
+    def patch(self, url: str, **kwargs) -> Response:
+        return self.request("PATCH", url, **kwargs)
+
+    def delete(self, url: str, **kwargs) -> Response:
+        return self.request("DELETE", url, **kwargs)
+
+    def head(self, url: str, **kwargs) -> Response:
+        return self.request("HEAD", url, **kwargs)
+
+    def send(self, request: Request) -> Response:
+        return self._transport.handle_request(request)
+
+    def close(self) -> None:
+        self._transport.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close()
+
+
+class AsyncClient(BaseClient):
+    """Asynchronous HTTP client."""
+
+    def __init__(self, *, transport=None, **kwargs):
+        super().__init__(**kwargs)
+        self._transport = transport or AsyncHTTPTransport()
+
+    async def request(self, method: str, url: str, **kwargs) -> Response:
+        request = self._build_request(method, url, **kwargs)
+        response = await self._transport.handle_async_request(request)
+        self._merge_cookies(response)
+        return response
+
+    async def get(self, url: str, **kwargs) -> Response:
+        return await self.request("GET", url, **kwargs)
+
+    async def post(self, url: str, **kwargs) -> Response:
+        return await self.request("POST", url, **kwargs)
+
+    async def put(self, url: str, **kwargs) -> Response:
+        return await self.request("PUT", url, **kwargs)
+
+    async def patch(self, url: str, **kwargs) -> Response:
+        return await self.request("PATCH", url, **kwargs)
+
+    async def delete(self, url: str, **kwargs) -> Response:
+        return await self.request("DELETE", url, **kwargs)
+
+    async def send(self, request: Request) -> Response:
+        return await self._transport.handle_async_request(request)
+
+    async def aclose(self) -> None:
+        await self._transport.aclose()
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, *args):
+        await self.aclose()

worked/httpx/raw/exceptions.py

@@ -0,0 +1,90 @@
+"""
+httpx-like exception hierarchy.
+All exceptions inherit from HTTPError at the top.
+"""
+
+
+class HTTPError(Exception):
+    """Base class for all httpx exceptions."""
+    def __init__(self, message, *, request=None):
+        self.request = request
+        super().__init__(message)
+
+
+class RequestError(HTTPError):
+    """An error occurred while issuing a request."""
+
+
+class TransportError(RequestError):
+    """An error occurred at the transport layer."""
+
+
+class TimeoutException(TransportError):
+    """A timeout occurred."""
+
+
+class ConnectTimeout(TimeoutException):
+    """Timed out while connecting to the host."""
+
+
+class ReadTimeout(TimeoutException):
+    """Timed out while receiving data from the host."""
+
+
+class WriteTimeout(TimeoutException):
+    """Timed out while sending data to the host."""
+
+
+class PoolTimeout(TimeoutException):
+    """Timed out waiting to acquire a connection from the pool."""
+
+
+class NetworkError(TransportError):
+    """A network error occurred."""
+
+
+class ConnectError(NetworkError):
+    """Failed to establish a connection."""
+
+
+class ReadError(NetworkError):
+    """Failed to receive data from the network."""
+
+
+class WriteError(NetworkError):
+    """Failed to send data through the network."""
+
+
+class CloseError(NetworkError):
+    """Failed to close a connection."""
+
+
+class ProxyError(TransportError):
+    """An error occurred while establishing a proxy connection."""
+
+
+class ProtocolError(TransportError):
+    """A protocol was violated."""
+
+
+class DecodingError(RequestError):
+    """Decoding of the response failed."""
+
+
+class TooManyRedirects(RequestError):
+    """Too many redirects."""
+
+
+class HTTPStatusError(HTTPError):
+    """A 4xx or 5xx response was received."""
+    def __init__(self, message, *, request, response):
+        self.response = response
+        super().__init__(message, request=request)
+
+
+class InvalidURL(Exception):
+    """URL is improperly formed or cannot be parsed."""
+
+
+class CookieConflict(Exception):
+    """Attempted to look up a cookie by name but multiple cookies exist."""

worked/httpx/raw/models.py

@@ -0,0 +1,120 @@
+"""
+Core data models: URL, Headers, Cookies, Request, Response.
+These are the central data types that everything else in the library references.
+"""
+import json as _json
+from exceptions import HTTPStatusError
+
+
+class URL:
+    def __init__(self, url: str):
+        self.raw = url
+        self.scheme, _, rest = url.partition("://")
+        self.host, _, self.path = rest.partition("/")
+        self.path = "/" + self.path
+
+    def copy_with(self, **kwargs) -> "URL":
+        return URL(kwargs.get("url", self.raw))
+
+    def __str__(self):
+        return self.raw
+
+    def __repr__(self):
+        return f"URL({self.raw!r})"
+
+
+class Headers:
+    def __init__(self, headers=None):
+        self._store = {}
+        for k, v in (headers or {}).items():
+            self._store[k.lower()] = v
+
+    def get(self, key: str, default=None):
+        return self._store.get(key.lower(), default)
+
+    def items(self):
+        return self._store.items()
+
+    def __setitem__(self, key, value):
+        self._store[key.lower()] = value
+
+    def __getitem__(self, key):
+        return self._store[key.lower()]
+
+    def __contains__(self, key):
+        return key.lower() in self._store
+
+
+class Cookies:
+    def __init__(self, cookies=None):
+        self._jar = dict(cookies or {})
+
+    def set(self, name: str, value: str, domain: str = "") -> None:
+        self._jar[name] = value
+
+    def get(self, name: str, default=None):
+        return self._jar.get(name, default)
+
+    def delete(self, name: str) -> None:
+        self._jar.pop(name, None)
+
+    def clear(self) -> None:
+        self._jar.clear()
+
+    def items(self):
+        return self._jar.items()
+
+
+class Request:
+    def __init__(self, method: str, url, *, headers=None, content=None, cookies=None):
+        self.method = method.upper()
+        self.url = URL(url) if isinstance(url, str) else url
+        self.headers = Headers(headers)
+        self.content = content or b""
+        self.cookies = Cookies(cookies)
+
+    def __repr__(self):
+        return f"<Request [{self.method}]>"
+
+
+class Response:
+    def __init__(self, status_code: int, *, headers=None, content=None, request=None):
+        self.status_code = status_code
+        self.headers = Headers(headers)
+        self.content = content or b""
+        self.request = request
+
+    @property
+    def text(self) -> str:
+        return self.content.decode("utf-8", errors="replace")
+
+    def json(self):
+        return _json.loads(self.content)
+
+    def read(self) -> bytes:
+        return self.content
+
+    @property
+    def is_success(self) -> bool:
+        return 200 <= self.status_code < 300
+
+    @property
+    def is_error(self) -> bool:
+        return self.status_code >= 400
+
+    def raise_for_status(self) -> None:
+        if self.is_error:
+            message = f"{self.status_code} Error"
+            raise HTTPStatusError(message, request=self.request, response=self)
+
+    @property
+    def cookies(self) -> Cookies:
+        jar = Cookies()
+        for header in self.headers.get("set-cookie", "").split(","):
+            if "=" in header:
+                name, _, value = header.strip().partition("=")
+                jar.set(name.strip(), value.split(";")[0].strip())
+        return jar
+
+    def __repr__(self):
+        return f"<Response [{self.status_code}]>"

worked/httpx/raw/transport.py

@@ -0,0 +1,135 @@
+"""
+Transport layer: connection management and low-level HTTP sending.
+HTTPTransport wraps a connection pool. ProxyTransport sits in front of it.
+MockTransport is used in tests.
+"""
+from models import Request, Response
+from exceptions import TransportError, ConnectError, TimeoutException
+
+
+class BaseTransport:
+    """Sync transport interface."""
+
+    def handle_request(self, request: Request) -> Response:
+        raise NotImplementedError
+
+    def close(self) -> None:
+        pass
+
+
+class AsyncBaseTransport:
+    """Async transport interface."""
+
+    async def handle_async_request(self, request: Request) -> Response:
+        raise NotImplementedError
+
+    async def aclose(self) -> None:
+        pass
+
+
+class ConnectionPool:
+    """
+    Manages a pool of persistent HTTP connections.
+    Keys connections by (scheme, host, port).
+    """
+
+    def __init__(self, max_connections=100, max_keepalive_connections=20):
+        self.max_connections = max_connections
+        self.max_keepalive_connections = max_keepalive_connections
+        self._pool = {}
+
+    def _get_connection_key(self, request: Request) -> tuple:
+        url = request.url
+        port = 443 if url.scheme == "https" else 80
+        return (url.scheme, url.host, port)
+
+    def get_connection(self, request: Request):
+        key = self._get_connection_key(request)
+        return self._pool.get(key)
+
+    def return_connection(self, request: Request, conn) -> None:
+        key = self._get_connection_key(request)
+        if len(self._pool) < self.max_keepalive_connections:
+            self._pool[key] = conn
+
+    def close(self) -> None:
+        self._pool.clear()
+
+
+class HTTPTransport(BaseTransport):
+    """
+    The main sync HTTP transport.
+    Uses a ConnectionPool for connection reuse.
+    """
+
+    def __init__(self, verify=True, cert=None, limits=None):
+        self.verify = verify
+        self.cert = cert
+        self._pool = ConnectionPool()
+
+    def handle_request(self, request: Request) -> Response:
+        conn = self._pool.get_connection(request)
+        try:
+            response = self._send(request, conn)
+            self._pool.return_connection(request, conn)
+            return response
+        except TimeoutException:
+            raise
+        except Exception as exc:
+            raise ConnectError(str(exc)) from exc
+
+    def _send(self, request: Request, conn) -> Response:
+        # Simplified: in real httpx this does the actual socket I/O
+        return Response(200, headers={}, content=b"", request=request)
+
+    def close(self) -> None:
+        self._pool.close()
+
+
+class AsyncHTTPTransport(AsyncBaseTransport):
+    """The async variant of HTTPTransport."""
+
+    def __init__(self, verify=True, cert=None):
+        self.verify = verify
+        self.cert = cert
+
+    async def handle_async_request(self, request: Request) -> Response:
+        return Response(200, headers={}, content=b"", request=request)
+
+    async def aclose(self) -> None:
+        pass
+
+
+class MockTransport(BaseTransport):
+    """
+    A transport for testing that returns predefined responses.
+    Pass a handler function that receives a Request and returns a Response.
+    """
+
+    def __init__(self, handler):
+        self.handler = handler
+
+    def handle_request(self, request: Request) -> Response:
+        return self.handler(request)
+
+
+class ProxyTransport(BaseTransport):
+    """
+    Routes requests through an HTTP/HTTPS proxy.
+    Wraps an inner transport and prepends proxy connection handling.
+    """
+
+    def __init__(self, proxy_url: str, *, inner: BaseTransport = None):
+        self.proxy_url = proxy_url
+        self._inner = inner or HTTPTransport()
+
+    def handle_request(self, request: Request) -> Response:
+        try:
+            return self._inner.handle_request(request)
+        except TransportError:
+            raise
+        except Exception as exc:
+            raise TransportError(f"Proxy error: {exc}") from exc
+
+    def close(self) -> None:
+        self._inner.close()

worked/httpx/raw/utils.py

@@ -0,0 +1,85 @@
+"""
+Utility functions shared across the library.
+Small helpers that don't belong in any one module.
+"""
+import re
+from models import Cookies
+
+
+SENSITIVE_HEADERS = {"authorization", "cookie", "set-cookie", "proxy-authorization"}
+
+
+def primitive_value_to_str(value) -> str:
+    """Convert a primitive value to its string representation."""
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    return str(value)
+
+
+def normalize_header_key(key: str) -> str:
+    """Convert a header key to its canonical Title-Case form."""
+    return "-".join(word.capitalize() for word in key.split("-"))
+
+
+def flatten_queryparams(params: dict) -> list:
+    """
+    Expand a params dict into a flat list of (key, value) pairs.
+    List values become multiple pairs with the same key.
+    """
+    result = []
+    for key, value in params.items():
+        if isinstance(value, list):
+            for item in value:
+                result.append((key, primitive_value_to_str(item)))
+        else:
+            result.append((key, primitive_value_to_str(value)))
+    return result
+
+
+def parse_content_type(content_type: str) -> tuple:
+    """
+    Parse a Content-Type header value.
+    Returns (media_type, params_dict).
+    Example: 'application/json; charset=utf-8' -> ('application/json', {'charset': 'utf-8'})
+    """
+    parts = [p.strip() for p in content_type.split(";")]
+    media_type = parts[0]
+    params = {}
+    for part in parts[1:]:
+        if "=" in part:
+            key, _, value = part.partition("=")
+            params[key.strip()] = value.strip()
+    return media_type, params
+
+
+def obfuscate_sensitive_headers(headers: dict) -> dict:
+    """Return a copy of headers with sensitive values replaced by [obfuscated]."""
+    return {
+        k: "[obfuscated]" if k.lower() in SENSITIVE_HEADERS else v
+        for k, v in headers.items()
+    }
+
+
+def unset_all_cookies(cookies: Cookies) -> None:
+    """Clear all cookies from a cookie jar in place."""
+    cookies.clear()
+
+
+def is_known_encoding(encoding: str) -> bool:
+    """Check if a character encoding label is recognized by Python's codec system."""
+    import codecs
+    try:
+        codecs.lookup(encoding)
+        return True
+    except LookupError:
+        return False
+
+
+def build_url_with_params(base_url: str, params: dict) -> str:
+    """Append query parameters to a URL string."""
+    if not params:
+        return base_url
+    pairs = flatten_queryparams(params)
+    query = "&".join(f"{k}={v}" for k, v in pairs)
+    separator = "&" if "?" in base_url else "?"
+    return f"{base_url}{separator}{query}"

worked/karpathy-repos/README.md

@@ -0,0 +1,63 @@
+# Karpathy Repos Benchmark — How to Reproduce
+
+This is the corpus that produced the 71.5x token reduction benchmark.
+
+## Corpus (52 files)
+
+### Code — clone these 3 repos
+
+```bash
+git clone https://github.com/karpathy/nanoGPT
+git clone https://github.com/karpathy/minGPT
+git clone https://github.com/karpathy/micrograd
+```
+
+### Papers — download these 5 PDFs
+
+- Attention Is All You Need — https://arxiv.org/abs/1706.03762
+- FlashAttention: Fast and Memory-Efficient Exact Attention — https://arxiv.org/abs/2205.14135
+- FlashAttention-2 — https://arxiv.org/abs/2307.08691
+- Neural Attention Residuals — https://arxiv.org/abs/2505.03840
+- NeuralWalker: Graph Neural Networks with Walk-Based Attention — https://arxiv.org/abs/2502.02593
+
+### Images — save these 4
+
+- `gpt2_124M_loss.png` — nanoGPT training loss curve (in the nanoGPT repo)
+- `gout.svg` — micrograd computation graph (in the micrograd repo)
+- `moon_mlp.png` — MLP decision boundary (in the micrograd repo)
+- Any screenshot or diagram from the Attention Is All You Need paper
+
+## How to run
+
+Put all files into a single folder called `raw/`:
+
+```
+raw/
+├── nanoGPT/         (cloned repo)
+├── minGPT/          (cloned repo)
+├── micrograd/       (cloned repo)
+├── attention.pdf
+├── flashattention.pdf
+├── flashattention2.pdf
+├── attn_residuals.pdf
+├── neuralwalker.pdf
+├── gpt2_124M_loss.png
+├── gout.svg
+└── moon_mlp.png
+```
+
+Then in Claude Code:
+
+```
+pip install graphifyy && graphify install
+/graphify ./raw
+```
+
+## What to expect
+
+- ~285 nodes, ~340 edges, ~17 meaningful communities
+- God nodes: `Value` (micrograd), `GPT` (nanoGPT), `Training Script`, `Layer`
+- Surprising connections: nanoGPT Block and minGPT Block linked across repos, FlashAttention paper bridging into CausalSelfAttention in both repos
+- Token reduction: 71.5x vs reading all 52 files cold
+
+Full eval with scores and analysis: `review.md`

worked/mixed-corpus/README.md

@@ -0,0 +1,45 @@
+# Mixed Corpus Benchmark — How to Reproduce
+
+A small but realistic mixed-input corpus: Python source files, a markdown paper with
+arXiv citations, and one image. Tests graphify's ability to handle different file types
+in a single run.
+
+## Corpus (5 files)
+
+All input files are in `raw/`:
+
+```
+raw/
+├── analyze.py          — graphify's graph analysis module (god_nodes, surprising_connections, etc.)
+├── build.py            — graphify's graph builder (build_from_json, networkx wrapper)
+├── cluster.py          — graphify's Leiden community detection (cluster, score_all)
+├── attention_notes.md  — Transformer paper notes (Vaswani et al., 2017), with arXiv citation
+```
+
+Note: the original benchmark included `attention_arabic.png` (an Arabic-language figure from the
+Attention paper). PNG files are not stored in this repo. To reproduce with the image, save any
+diagram or figure from the Attention Is All You Need paper as `raw/attention_arabic.png`.
+
+## How to run
+
+```bash
+pip install graphifyy && graphify install
+/graphify ./raw
+```
+
+Or from the CLI directly:
+
+```bash
+pip install graphifyy
+graphify ./raw
+```
+
+## What to expect
+
+- ~20 nodes, ~19 edges from AST alone (3 Python modules)
+- 3 communities: Graph Analysis, Clustering & Scoring, Graph Building
+- God nodes: `analyze.py`, `cluster.py`, `build.py`
+- `attention_notes.md` classified as `paper` (arXiv heuristic fires on `1706.03762`)
+- If you include the image: 1 extra node describing the figure content via vision
+
+Full eval with scores and analysis: `review.md`

worked/mixed-corpus/raw/analyze.py

@@ -0,0 +1,517 @@
+"""Graph analysis: god nodes (most connected), surprising connections (cross-community), suggested questions."""
+from __future__ import annotations
+import networkx as nx
+
+
+def _node_community_map(communities: dict[int, list[str]]) -> dict[str, int]:
+    """Invert communities dict: node_id -> community_id."""
+    return {n: cid for cid, nodes in communities.items() for n in nodes}
+
+
+def _is_file_node(G: nx.Graph, node_id: str) -> bool:
+    """
+    Return True if this node is a file-level hub node (e.g. 'client', 'models')
+    or an AST method stub (e.g. '.auth_flow()', '.__init__()').
+
+    These are synthetic nodes created by the AST extractor and should be excluded
+    from god nodes, surprising connections, and knowledge gap reporting.
+    """
+    label = G.nodes[node_id].get("label", "")
+    if not label:
+        return False
+    # File-level hub: label is a filename with a code extension
+    if label.split(".")[-1] in ("py", "ts", "js", "go", "rs", "java", "rb", "cpp", "c", "h"):
+        return True
+    # Method stub: AST extractor labels methods as '.method_name()'
+    if label.startswith(".") and label.endswith("()"):
+        return True
+    # Module-level function stub: labeled 'function_name()' - only has a contains edge
+    # These are real functions but structurally isolated by definition; not a gap worth flagging
+    if label.endswith("()") and G.degree(node_id) <= 1:
+        return True
+    return False
+
+
+def god_nodes(G: nx.Graph, top_n: int = 10) -> list[dict]:
+    """Return the top_n most-connected real entities - the core abstractions.
+
+    File-level hub nodes are excluded: they accumulate import/contains edges
+    mechanically and don't represent meaningful architectural abstractions.
+    """
+    degree = dict(G.degree())
+    sorted_nodes = sorted(degree.items(), key=lambda x: x[1], reverse=True)
+    result = []
+    for node_id, deg in sorted_nodes:
+        if _is_file_node(G, node_id) or _is_concept_node(G, node_id):
+            continue
+        result.append({
+            "id": node_id,
+            "label": G.nodes[node_id].get("label", node_id),
+            "edges": deg,
+        })
+        if len(result) >= top_n:
+            break
+    return result
+
+
+def surprising_connections(
+    G: nx.Graph,
+    communities: dict[int, list[str]] | None = None,
+    top_n: int = 5,
+) -> list[dict]:
+    """
+    Find connections that are genuinely surprising - not obvious from file structure.
+
+    Strategy:
+    - Multi-file corpora: cross-file edges between real entities (not concept nodes).
+      Sorted AMBIGUOUS → INFERRED → EXTRACTED.
+    - Single-file / single-source corpora: cross-community edges that bridge
+      distant parts of the graph (betweenness centrality on edges).
+      These reveal non-obvious structural couplings.
+
+    Concept nodes (empty source_file, or injected semantic annotations) are excluded
+    from surprising connections because they are intentional, not discovered.
+    """
+    # Identify unique source files (ignore empty/null source_file)
+    source_files = {
+        data.get("source_file", "")
+        for _, data in G.nodes(data=True)
+        if data.get("source_file", "")
+    }
+    is_multi_source = len(source_files) > 1
+
+    if is_multi_source:
+        return _cross_file_surprises(G, communities or {}, top_n)
+    else:
+        return _cross_community_surprises(G, communities or {}, top_n)
+
+
+def _is_concept_node(G: nx.Graph, node_id: str) -> bool:
+    """
+    Return True if this node is a manually-injected semantic concept node
+    rather than a real entity found in source code.
+
+    Signals:
+    - Empty source_file
+    - source_file doesn't look like a real file path (no extension)
+    """
+    data = G.nodes[node_id]
+    source = data.get("source_file", "")
+    if not source:
+        return True
+    # Has no file extension → probably a concept label, not a real file
+    if "." not in source.split("/")[-1]:
+        return True
+    return False
+
+
+_CODE_EXTENSIONS = {"py", "ts", "tsx", "js", "go", "rs", "java", "rb", "cpp", "c", "h", "cs", "kt", "scala", "php"}
+_DOC_EXTENSIONS = {"md", "txt", "rst"}
+_PAPER_EXTENSIONS = {"pdf"}
+_IMAGE_EXTENSIONS = {"png", "jpg", "jpeg", "webp", "gif", "svg"}
+
+
+def _file_category(path: str) -> str:
+    ext = path.rsplit(".", 1)[-1].lower() if "." in path else ""
+    if ext in _CODE_EXTENSIONS:
+        return "code"
+    if ext in _PAPER_EXTENSIONS:
+        return "paper"
+    if ext in _IMAGE_EXTENSIONS:
+        return "image"
+    return "doc"
+
+
+def _top_level_dir(path: str) -> str:
+    """Return the first path component - used to detect cross-repo edges."""
+    return path.split("/")[0] if "/" in path else path
+
+
+def _surprise_score(
+    G: nx.Graph,
+    u: str,
+    v: str,
+    data: dict,
+    node_community: dict[str, int],
+    u_source: str,
+    v_source: str,
+) -> tuple[int, list[str]]:
+    """Score how surprising a cross-file edge is. Returns (score, reasons)."""
+    score = 0
+    reasons: list[str] = []
+
+    # 1. Confidence weight - uncertain connections are more noteworthy
+    conf = data.get("confidence", "EXTRACTED")
+    conf_bonus = {"AMBIGUOUS": 3, "INFERRED": 2, "EXTRACTED": 1}.get(conf, 1)
+    score += conf_bonus
+    if conf in ("AMBIGUOUS", "INFERRED"):
+        reasons.append(f"{conf.lower()} connection - not explicitly stated in source")
+
+    # 2. Cross file-type bonus - code↔paper or code↔image is non-obvious
+    cat_u = _file_category(u_source)
+    cat_v = _file_category(v_source)
+    if cat_u != cat_v:
+        score += 2
+        reasons.append(f"crosses file types ({cat_u} ↔ {cat_v})")
+
+    # 3. Cross-repo bonus - different top-level directory
+    if _top_level_dir(u_source) != _top_level_dir(v_source):
+        score += 2
+        reasons.append("connects across different repos/directories")
+
+    # 4. Cross-community bonus - Leiden says these are structurally distant
+    cid_u = node_community.get(u)
+    cid_v = node_community.get(v)
+    if cid_u is not None and cid_v is not None and cid_u != cid_v:
+        score += 1
+        reasons.append("bridges separate communities")
+
+    # 5. Peripheral→hub: a low-degree node connecting to a high-degree one
+    deg_u = G.degree(u)
+    deg_v = G.degree(v)
+    if min(deg_u, deg_v) <= 2 and max(deg_u, deg_v) >= 5:
+        score += 1
+        peripheral = G.nodes[u].get("label", u) if deg_u <= 2 else G.nodes[v].get("label", v)
+        hub = G.nodes[v].get("label", v) if deg_u <= 2 else G.nodes[u].get("label", u)
+        reasons.append(f"peripheral node `{peripheral}` unexpectedly reaches hub `{hub}`")
+
+    return score, reasons
+
+
+def _cross_file_surprises(G: nx.Graph, communities: dict[int, list[str]], top_n: int) -> list[dict]:
+    """
+    Cross-file edges between real code/doc entities, ranked by a composite
+    surprise score rather than confidence alone.
+
+    Surprise score accounts for:
+    - Confidence (AMBIGUOUS > INFERRED > EXTRACTED)
+    - Cross file-type (code↔paper is more surprising than code↔code)
+    - Cross-repo (different top-level directory)
+    - Cross-community (Leiden says structurally distant)
+    - Peripheral→hub (low-degree node reaching a god node)
+
+    Each result includes a 'why' field explaining what makes it non-obvious.
+    """
+    node_community = _node_community_map(communities)
+    candidates = []
+
+    for u, v, data in G.edges(data=True):
+        relation = data.get("relation", "")
+        if relation in ("imports", "imports_from", "contains", "method"):
+            continue
+        if _is_concept_node(G, u) or _is_concept_node(G, v):
+            continue
+        if _is_file_node(G, u) or _is_file_node(G, v):
+            continue
+
+        u_source = G.nodes[u].get("source_file", "")
+        v_source = G.nodes[v].get("source_file", "")
+
+        if not u_source or not v_source or u_source == v_source:
+            continue
+
+        score, reasons = _surprise_score(G, u, v, data, node_community, u_source, v_source)
+        src_id = data.get("_src", u)
+        tgt_id = data.get("_tgt", v)
+        candidates.append({
+            "_score": score,
+            "source": G.nodes[src_id].get("label", src_id),
+            "target": G.nodes[tgt_id].get("label", tgt_id),
+            "source_files": [
+                G.nodes[src_id].get("source_file", ""),
+                G.nodes[tgt_id].get("source_file", ""),
+            ],
+            "confidence": data.get("confidence", "EXTRACTED"),
+            "relation": relation,
+            "why": "; ".join(reasons) if reasons else "cross-file semantic connection",
+        })
+
+    candidates.sort(key=lambda x: x["_score"], reverse=True)
+    for c in candidates:
+        c.pop("_score")
+
+    if candidates:
+        return candidates[:top_n]
+
+    return _cross_community_surprises(G, communities, top_n)
+
+
+def _cross_community_surprises(
+    G: nx.Graph,
+    communities: dict[int, list[str]],
+    top_n: int,
+) -> list[dict]:
+    """
+    For single-source corpora: find edges that bridge different communities.
+    These are surprising because Leiden grouped everything else tightly -
+    these edges cut across the natural structure.
+
+    Falls back to high-betweenness edges if no community info is provided.
+    """
+    if not communities:
+        # No community info - use edge betweenness centrality
+        if G.number_of_edges() == 0:
+            return []
+        betweenness = nx.edge_betweenness_centrality(G)
+        top_edges = sorted(betweenness.items(), key=lambda x: x[1], reverse=True)[:top_n]
+        result = []
+        for (u, v), score in top_edges:
+            data = G.edges[u, v]
+            result.append({
+                "source": G.nodes[u].get("label", u),
+                "target": G.nodes[v].get("label", v),
+                "source_files": [
+                    G.nodes[u].get("source_file", ""),
+                    G.nodes[v].get("source_file", ""),
+                ],
+                "confidence": data.get("confidence", "EXTRACTED"),
+                "relation": data.get("relation", ""),
+                "note": f"Bridges graph structure (betweenness={score:.3f})",
+            })
+        return result
+
+    # Build node → community map
+    node_community = _node_community_map(communities)
+
+    surprises = []
+    for u, v, data in G.edges(data=True):
+        cid_u = node_community.get(u)
+        cid_v = node_community.get(v)
+        if cid_u is None or cid_v is None or cid_u == cid_v:
+            continue
+        # Skip file hub nodes and plain structural edges
+        if _is_file_node(G, u) or _is_file_node(G, v):
+            continue
+        relation = data.get("relation", "")
+        if relation in ("imports", "imports_from", "contains", "method"):
+            continue
+        # This edge crosses community boundaries - interesting
+        confidence = data.get("confidence", "EXTRACTED")
+        src_id = data.get("_src", u)
+        tgt_id = data.get("_tgt", v)
+        surprises.append({
+            "source": G.nodes[src_id].get("label", src_id),
+            "target": G.nodes[tgt_id].get("label", tgt_id),
+            "source_files": [
+                G.nodes[src_id].get("source_file", ""),
+                G.nodes[tgt_id].get("source_file", ""),
+            ],
+            "confidence": confidence,
+            "relation": relation,
+            "note": f"Bridges community {cid_u} → community {cid_v}",
+            "_pair": tuple(sorted([cid_u, cid_v])),
+        })
+
+    # Sort: AMBIGUOUS first, then INFERRED, then EXTRACTED
+    order = {"AMBIGUOUS": 0, "INFERRED": 1, "EXTRACTED": 2}
+    surprises.sort(key=lambda x: order.get(x["confidence"], 3))
+
+    # Deduplicate by community pair - one representative edge per (A→B) boundary.
+    # Without this, a single high-betweenness god node dominates all results.
+    seen_pairs: set[tuple] = set()
+    deduped = []
+    for s in surprises:
+        pair = s.pop("_pair")
+        if pair not in seen_pairs:
+            seen_pairs.add(pair)
+            deduped.append(s)
+    return deduped[:top_n]
+
+
+def suggest_questions(
+    G: nx.Graph,
+    communities: dict[int, list[str]],
+    community_labels: dict[int, str],
+    top_n: int = 7,
+) -> list[dict]:
+    """
+    Generate questions the graph is uniquely positioned to answer.
+    Based on: AMBIGUOUS edges, bridge nodes, underexplored god nodes, isolated nodes.
+    Each question has a 'type', 'question', and 'why' field.
+    """
+    questions = []
+    node_community = _node_community_map(communities)
+
+    # 1. AMBIGUOUS edges → unresolved relationship questions
+    for u, v, data in G.edges(data=True):
+        if data.get("confidence") == "AMBIGUOUS":
+            ul = G.nodes[u].get("label", u)
+            vl = G.nodes[v].get("label", v)
+            relation = data.get("relation", "related to")
+            questions.append({
+                "type": "ambiguous_edge",
+                "question": f"What is the exact relationship between `{ul}` and `{vl}`?",
+                "why": f"Edge tagged AMBIGUOUS (relation: {relation}) - confidence is low.",
+            })
+
+    # 2. Bridge nodes (high betweenness) → cross-cutting concern questions
+    if G.number_of_edges() > 0:
+        betweenness = nx.betweenness_centrality(G)
+        # Top bridge nodes that are NOT file-level hubs
+        bridges = sorted(
+            [(n, s) for n, s in betweenness.items()
+             if not _is_file_node(G, n) and not _is_concept_node(G, n) and s > 0],
+            key=lambda x: x[1],
+            reverse=True,
+        )[:3]
+        for node_id, score in bridges:
+            label = G.nodes[node_id].get("label", node_id)
+            cid = node_community.get(node_id)
+            comm_label = community_labels.get(cid, f"Community {cid}") if cid is not None else "unknown"
+            neighbors = list(G.neighbors(node_id))
+            neighbor_comms = {node_community.get(n) for n in neighbors if node_community.get(n) != cid}
+            if neighbor_comms:
+                other_labels = [community_labels.get(c, f"Community {c}") for c in neighbor_comms]
+                questions.append({
+                    "type": "bridge_node",
+                    "question": f"Why does `{label}` connect `{comm_label}` to {', '.join(f'`{l}`' for l in other_labels)}?",
+                    "why": f"High betweenness centrality ({score:.3f}) - this node is a cross-community bridge.",
+                })
+
+    # 3. God nodes with many INFERRED edges → verification questions
+    degree = dict(G.degree())
+    top_nodes = sorted(
+        [(n, d) for n, d in degree.items() if not _is_file_node(G, n)],
+        key=lambda x: x[1],
+        reverse=True,
+    )[:5]
+    for node_id, _ in top_nodes:
+        inferred = [
+            (u, v, d) for u, v, d in G.edges(node_id, data=True)
+            if d.get("confidence") == "INFERRED"
+        ]
+        if len(inferred) >= 2:
+            label = G.nodes[node_id].get("label", node_id)
+            # Use _src/_tgt to get the correct direction; fall back to v (the other node)
+            others = []
+            for u, v, d in inferred[:2]:
+                src_id = d.get("_src", u)
+                tgt_id = d.get("_tgt", v)
+                other_id = tgt_id if src_id == node_id else src_id
+                others.append(G.nodes[other_id].get("label", other_id))
+            questions.append({
+                "type": "verify_inferred",
+                "question": f"Are the {len(inferred)} inferred relationships involving `{label}` (e.g. with `{others[0]}` and `{others[1]}`) actually correct?",
+                "why": f"`{label}` has {len(inferred)} INFERRED edges - model-reasoned connections that need verification.",
+            })
+
+    # 4. Isolated or weakly-connected nodes → exploration questions
+    isolated = [
+        n for n in G.nodes()
+        if G.degree(n) <= 1 and not _is_file_node(G, n) and not _is_concept_node(G, n)
+    ]
+    if isolated:
+        labels = [G.nodes[n].get("label", n) for n in isolated[:3]]
+        questions.append({
+            "type": "isolated_nodes",
+            "question": f"What connects {', '.join(f'`{l}`' for l in labels)} to the rest of the system?",
+            "why": f"{len(isolated)} weakly-connected nodes found - possible documentation gaps or missing edges.",
+        })
+
+    # 5. Low-cohesion communities → structural questions
+    from .cluster import cohesion_score
+    for cid, nodes in communities.items():
+        score = cohesion_score(G, nodes)
+        if score < 0.15 and len(nodes) >= 5:
+            label = community_labels.get(cid, f"Community {cid}")
+            questions.append({
+                "type": "low_cohesion",
+                "question": f"Should `{label}` be split into smaller, more focused modules?",
+                "why": f"Cohesion score {score} - nodes in this community are weakly interconnected.",
+            })
+
+    if not questions:
+        return [{
+            "type": "no_signal",
+            "question": None,
+            "why": (
+                "Not enough signal to generate questions. "
+                "This usually means the corpus has no AMBIGUOUS edges, no bridge nodes, "
+                "no INFERRED relationships, and all communities are tightly cohesive. "
+                "Add more files or run with --mode deep to extract richer edges."
+            ),
+        }]
+
+    return questions[:top_n]
+
+
+def graph_diff(G_old: nx.Graph, G_new: nx.Graph) -> dict:
+    """Compare two graph snapshots and return what changed.
+
+    Returns:
+        {
+          "new_nodes": [{"id": ..., "label": ...}],
+          "removed_nodes": [{"id": ..., "label": ...}],
+          "new_edges": [{"source": ..., "target": ..., "relation": ..., "confidence": ...}],
+          "removed_edges": [...],
+          "summary": "3 new nodes, 5 new edges, 1 node removed"
+        }
+    """
+    old_nodes = set(G_old.nodes())
+    new_nodes = set(G_new.nodes())
+
+    added_node_ids = new_nodes - old_nodes
+    removed_node_ids = old_nodes - new_nodes
+
+    new_nodes_list = [
+        {"id": n, "label": G_new.nodes[n].get("label", n)}
+        for n in added_node_ids
+    ]
+    removed_nodes_list = [
+        {"id": n, "label": G_old.nodes[n].get("label", n)}
+        for n in removed_node_ids
+    ]
+
+    def edge_key(G: nx.Graph, u: str, v: str, data: dict) -> tuple:
+        return (u, v, data.get("relation", ""))
+
+    old_edge_keys = {
+        edge_key(G_old, u, v, d)
+        for u, v, d in G_old.edges(data=True)
+    }
+    new_edge_keys = {
+        edge_key(G_new, u, v, d)
+        for u, v, d in G_new.edges(data=True)
+    }
+
+    added_edge_keys = new_edge_keys - old_edge_keys
+    removed_edge_keys = old_edge_keys - new_edge_keys
+
+    new_edges_list = []
+    for u, v, d in G_new.edges(data=True):
+        if edge_key(G_new, u, v, d) in added_edge_keys:
+            new_edges_list.append({
+                "source": u,
+                "target": v,
+                "relation": d.get("relation", ""),
+                "confidence": d.get("confidence", ""),
+            })
+
+    removed_edges_list = []
+    for u, v, d in G_old.edges(data=True):
+        if edge_key(G_old, u, v, d) in removed_edge_keys:
+            removed_edges_list.append({
+                "source": u,
+                "target": v,
+                "relation": d.get("relation", ""),
+                "confidence": d.get("confidence", ""),
+            })
+
+    parts = []
+    if new_nodes_list:
+        parts.append(f"{len(new_nodes_list)} new node{'s' if len(new_nodes_list) != 1 else ''}")
+    if new_edges_list:
+        parts.append(f"{len(new_edges_list)} new edge{'s' if len(new_edges_list) != 1 else ''}")
+    if removed_nodes_list:
+        parts.append(f"{len(removed_nodes_list)} node{'s' if len(removed_nodes_list) != 1 else ''} removed")
+    if removed_edges_list:
+        parts.append(f"{len(removed_edges_list)} edge{'s' if len(removed_edges_list) != 1 else ''} removed")
+    summary = ", ".join(parts) if parts else "no changes"
+
+    return {
+        "new_nodes": new_nodes_list,
+        "removed_nodes": removed_nodes_list,
+        "new_edges": new_edges_list,
+        "removed_edges": removed_edges_list,
+        "summary": summary,
+    }

worked/mixed-corpus/raw/attention_notes.md

@@ -0,0 +1,76 @@
+# Attention Mechanism Notes
+
+Notes on the Transformer architecture from Vaswani et al., 2017.
+arXiv: 1706.03762
+
+## Abstract
+
+The dominant sequence transduction models are based on complex recurrent or convolutional neural networks that include an encoder and a decoder. The best performing models also connect the encoder and decoder through an attention mechanism. The Transformer is a model architecture eschewing recurrence and instead relying entirely on an attention mechanism to draw global dependencies between input and output.
+
+## Multi-Head Attention
+
+The model uses h=8 parallel attention heads. For each head, d_k = d_v = d_model/h = 64.
+
+Scaled dot-product attention:
+
+    Attention(Q, K, V) = softmax(QK^T / sqrt(d_k)) V
+
+Multi-head attention runs h attention functions in parallel, then concatenates and projects:
+
+    MultiHead(Q, K, V) = Concat(head_1, ..., head_h) W^O
+    head_i = Attention(Q W_i^Q, K W_i^K, V W_i^V)
+
+The scaling by sqrt(d_k) prevents the dot products from growing large in magnitude, which would push the softmax into regions with very small gradients.
+
+## Architecture
+
+The Transformer uses a stacked encoder-decoder structure.
+
+Encoder: 6 identical layers, each with two sublayers:
+1. Multi-head self-attention
+2. Position-wise fully connected feed-forward network
+
+Each sublayer uses a residual connection followed by layer normalization:
+    output = LayerNorm(x + Sublayer(x))
+
+Decoder: 6 identical layers, each with three sublayers:
+1. Masked multi-head self-attention (prevents positions from attending to subsequent positions)
+2. Multi-head attention over encoder output
+3. Position-wise feed-forward network
+
+d_model = 512 for all sublayers and embedding layers.
+Feed-forward inner dimension = 2048.
+
+## Positional Encoding
+
+Since the model contains no recurrence and no convolution, positional encodings are added to the input embeddings to give the model information about the relative position of tokens:
+
+    PE(pos, 2i) = sin(pos / 10000^(2i/d_model))
+    PE(pos, 2i+1) = cos(pos / 10000^(2i/d_model))
+
+This allows the model to easily learn to attend by relative positions.
+
+## Why attention over recurrence
+
+Three main advantages:
+1. Total computational complexity per layer is lower for self-attention when sequence length is smaller than representation dimensionality
+2. Computations that can be parallelized — recurrent layers require O(n) sequential operations
+3. Path length between long-range dependencies is O(1) for self-attention vs O(n) for recurrence
+
+## Results
+
+WMT 2014 English-to-German: 28.4 BLEU, outperforming all previously published results by over 2 BLEU.
+WMT 2014 English-to-French: 41.0 BLEU, new state of the art.
+Training cost: 3.5 days on 8 P100 GPUs.
+
+## Open questions
+
+[1] Does the choice of h=8 heads generalize, or is it architecture-specific?
+[2] The scaling factor sqrt(d_k) is justified empirically — is there a theoretical justification?
+[3] How does learned positional encoding compare to sinusoidal at longer sequence lengths?
+
+## References
+
+[1] Vaswani, A., Shazeer, N., Parmar, N., et al. (2017). Attention Is All You Need. arXiv:1706.03762
+[2] Ba, J., Kiros, J., Hinton, G. (2016). Layer Normalization. arXiv:1607.06450
+[3] He, K., et al. (2016). Deep Residual Learning for Image Recognition. CVPR 2016.

worked/mixed-corpus/raw/build.py

@@ -0,0 +1,39 @@
+# assemble node+edge dicts into a NetworkX graph, preserving edge direction
+from __future__ import annotations
+import sys
+import networkx as nx
+from .validate import validate_extraction
+
+
+def build_from_json(extraction: dict) -> nx.Graph:
+    errors = validate_extraction(extraction)
+    # Dangling edges (stdlib/external imports) are expected - only warn about real schema errors.
+    real_errors = [e for e in errors if "does not match any node id" not in e]
+    if real_errors:
+        print(f"[graphify] Extraction warning ({len(real_errors)} issues): {real_errors[0]}", file=sys.stderr)
+    G = nx.Graph()
+    for node in extraction.get("nodes", []):
+        G.add_node(node["id"], **{k: v for k, v in node.items() if k != "id"})
+    node_set = set(G.nodes())
+    for edge in extraction.get("edges", []):
+        src, tgt = edge["source"], edge["target"]
+        if src not in node_set or tgt not in node_set:
+            continue  # skip edges to external/stdlib nodes - expected, not an error
+        attrs = {k: v for k, v in edge.items() if k not in ("source", "target")}
+        # Preserve original edge direction - undirected graphs lose it otherwise,
+        # causing display functions to show edges backwards.
+        attrs["_src"] = src
+        attrs["_tgt"] = tgt
+        G.add_edge(src, tgt, **attrs)
+    return G
+
+
+def build(extractions: list[dict]) -> nx.Graph:
+    """Merge multiple extraction results into one graph."""
+    combined: dict = {"nodes": [], "edges": [], "input_tokens": 0, "output_tokens": 0}
+    for ext in extractions:
+        combined["nodes"].extend(ext.get("nodes", []))
+        combined["edges"].extend(ext.get("edges", []))
+        combined["input_tokens"] += ext.get("input_tokens", 0)
+        combined["output_tokens"] += ext.get("output_tokens", 0)
+    return build_from_json(combined)

worked/mixed-corpus/raw/cluster.py

@@ -0,0 +1,104 @@
+"""Leiden community detection on NetworkX graphs. Splits oversized communities. Returns cohesion scores."""
+from __future__ import annotations
+import networkx as nx
+
+
+def build_graph(nodes: list[dict], edges: list[dict]) -> nx.Graph:
+    """Build a NetworkX graph from graphify node/edge dicts.
+
+    Preserves original edge direction as _src/_tgt attributes so that
+    display functions can show relationships in the correct direction,
+    even though the graph is undirected for structural analysis.
+    """
+    G = nx.Graph()
+    for n in nodes:
+        G.add_node(n["id"], **{k: v for k, v in n.items() if k != "id"})
+    for e in edges:
+        attrs = {k: v for k, v in e.items() if k not in ("source", "target")}
+        attrs["_src"] = e["source"]
+        attrs["_tgt"] = e["target"]
+        G.add_edge(e["source"], e["target"], **attrs)
+    return G
+
+_MAX_COMMUNITY_FRACTION = 0.25   # communities larger than 25% of graph get split
+_MIN_SPLIT_SIZE = 10             # only split if community has at least this many nodes
+
+
+def cluster(G: nx.Graph) -> dict[int, list[str]]:
+    """Run Leiden community detection. Returns {community_id: [node_ids]}.
+
+    Community IDs are stable across runs: 0 = largest community after splitting.
+    Oversized communities (> 25% of graph nodes, min 10) are split by running
+    a second Leiden pass on the subgraph.
+    """
+    if G.number_of_nodes() == 0:
+        return {}
+    if G.number_of_edges() == 0:
+        return {i: [n] for i, n in enumerate(sorted(G.nodes))}
+
+    from graspologic.partition import leiden  # lazy - avoids 15s numba JIT on import
+
+    # Leiden warns and drops isolates - handle them separately
+    isolates = [n for n in G.nodes() if G.degree(n) == 0]
+    connected_nodes = [n for n in G.nodes() if G.degree(n) > 0]
+    connected = G.subgraph(connected_nodes)
+
+    raw: dict[int, list[str]] = {}
+    if connected.number_of_nodes() > 0:
+        partition: dict[str, int] = leiden(connected)
+        for node, cid in partition.items():
+            raw.setdefault(cid, []).append(node)
+
+    # Each isolate becomes its own single-node community
+    next_cid = max(raw.keys(), default=-1) + 1
+    for node in isolates:
+        raw[next_cid] = [node]
+        next_cid += 1
+
+    # Split oversized communities
+    max_size = max(_MIN_SPLIT_SIZE, int(G.number_of_nodes() * _MAX_COMMUNITY_FRACTION))
+    final_communities: list[list[str]] = []
+    for nodes in raw.values():
+        if len(nodes) > max_size:
+            final_communities.extend(_split_community(G, nodes))
+        else:
+            final_communities.append(nodes)
+
+    # Re-index by size descending for deterministic ordering
+    final_communities.sort(key=len, reverse=True)
+    return {i: sorted(nodes) for i, nodes in enumerate(final_communities)}
+
+
+def _split_community(G: nx.Graph, nodes: list[str]) -> list[list[str]]:
+    """Run a second Leiden pass on a community subgraph to split it further."""
+    subgraph = G.subgraph(nodes)
+    if subgraph.number_of_edges() == 0:
+        # No edges - split into individual nodes
+        return [[n] for n in sorted(nodes)]
+    try:
+        from graspologic.partition import leiden
+        sub_partition: dict[str, int] = leiden(subgraph)
+        sub_communities: dict[int, list[str]] = {}
+        for node, cid in sub_partition.items():
+            sub_communities.setdefault(cid, []).append(node)
+        if len(sub_communities) <= 1:
+            # Leiden couldn't split it - return as-is
+            return [sorted(nodes)]
+        return [sorted(v) for v in sub_communities.values()]
+    except Exception:
+        return [sorted(nodes)]
+
+
+def cohesion_score(G: nx.Graph, community_nodes: list[str]) -> float:
+    """Ratio of actual intra-community edges to maximum possible."""
+    n = len(community_nodes)
+    if n <= 1:
+        return 1.0
+    subgraph = G.subgraph(community_nodes)
+    actual = subgraph.number_of_edges()
+    possible = n * (n - 1) / 2
+    return round(actual / possible, 2) if possible > 0 else 0.0
+
+
+def score_all(G: nx.Graph, communities: dict[int, list[str]]) -> dict[int, float]:
+    return {cid: cohesion_score(G, nodes) for cid, nodes in communities.items()}