diff --git a/worked/httpx/README.md b/worked/httpx/README.md new file mode 100644 index 000000000..3d1c924d1 --- /dev/null +++ b/worked/httpx/README.md @@ -0,0 +1,42 @@ +# httpx Corpus Benchmark — How to Reproduce + +A synthetic 6-file Python codebase modeled after httpx's architecture. Tests graphify +on a realistic library codebase with clean layering: exceptions → models → auth/transport → client. + +## Corpus (6 files) + +All input files are in `raw/`: + +``` +raw/ +├── exceptions.py — full HTTPError hierarchy (RequestError, TransportError, HTTPStatusError, etc.) +├── models.py — URL, Headers, Cookies, Request, Response with raise_for_status +├── auth.py — BasicAuth, BearerAuth, DigestAuth (challenge-response), NetRCAuth +├── utils.py — header normalization, query param flattening, content-type parsing +├── transport.py — ConnectionPool, HTTPTransport, AsyncHTTPTransport, MockTransport, ProxyTransport +└── client.py — Timeout, Limits, BaseClient, Client (sync), AsyncClient +``` + +## How to run + +```bash +pip install graphifyy && graphify install +/graphify ./raw +``` + +Or from the CLI directly: + +```bash +pip install graphifyy +graphify ./raw +``` + +## What to expect + +- ~95 nodes, ~130 edges +- 4 communities: Exception Hierarchy, Models & Data, Auth & Transport, Client Layer +- God nodes: `client.py`, `models.py`, `transport.py`, `exceptions.py`, `BaseClient`, `Response` +- Surprising connections: `DigestAuth` ↔ `Response` (auth.py reads Response to parse WWW-Authenticate) +- All edges EXTRACTED — no inference needed, dependency graph is explicit + +Full eval with scores and analysis: `review.md` diff --git a/worked/httpx/raw/auth.py b/worked/httpx/raw/auth.py new file mode 100644 index 000000000..290cadd3e --- /dev/null +++ b/worked/httpx/raw/auth.py @@ -0,0 +1,114 @@ +""" +Authentication handlers. +Auth objects are callables that modify a request before it is sent. +DigestAuth is the most interesting: it participates in a full request/response cycle, +reading the 401 response to build the challenge before re-sending. +""" +import hashlib +import time +from models import Request, Response + + +class Auth: + """Base class for all authentication handlers.""" + + def auth_flow(self, request: Request): + """Modify the request. May yield to inspect the response.""" + raise NotImplementedError + + +class BasicAuth(Auth): + """HTTP Basic Authentication.""" + + def __init__(self, username: str, password: str): + self.username = username + self.password = password + + def auth_flow(self, request: Request): + import base64 + credentials = f"{self.username}:{self.password}".encode() + encoded = base64.b64encode(credentials).decode() + request.headers["Authorization"] = f"Basic {encoded}" + yield request + + +class BearerAuth(Auth): + """Bearer token authentication.""" + + def __init__(self, token: str): + self.token = token + + def auth_flow(self, request: Request): + request.headers["Authorization"] = f"Bearer {self.token}" + yield request + + +class DigestAuth(Auth): + """ + HTTP Digest Authentication. + Requires a full request/response cycle: sends the initial request, + reads the 401 WWW-Authenticate header, then re-sends with credentials. + This is the only auth handler that reads from Response. + """ + + def __init__(self, username: str, password: str): + self.username = username + self.password = password + self._nonce_count = 0 + + def auth_flow(self, request: Request): + yield request # first attempt, no credentials + + # This handler must inspect the Response to continue + response = yield + + if response.status_code == 401: + challenge = self._parse_challenge(response) + credentials = self._build_credentials(request, challenge) + request.headers["Authorization"] = credentials + yield request + + def _parse_challenge(self, response: Response) -> dict: + """Extract digest parameters from the WWW-Authenticate header.""" + header = response.headers.get("www-authenticate", "") + params = {} + for part in header.replace("Digest ", "").split(","): + if "=" in part: + key, _, value = part.strip().partition("=") + params[key.strip()] = value.strip().strip('"') + return params + + def _build_credentials(self, request: Request, challenge: dict) -> str: + """Compute the Authorization header value for a digest challenge.""" + self._nonce_count += 1 + nc = f"{self._nonce_count:08x}" + cnonce = hashlib.md5(str(time.time()).encode()).hexdigest()[:8] + realm = challenge.get("realm", "") + nonce = challenge.get("nonce", "") + + ha1 = hashlib.md5(f"{self.username}:{realm}:{self.password}".encode()).hexdigest() + ha2 = hashlib.md5(f"{request.method}:{request.url.path}".encode()).hexdigest() + response_hash = hashlib.md5(f"{ha1}:{nonce}:{nc}:{cnonce}:auth:{ha2}".encode()).hexdigest() + + return ( + f'Digest username="{self.username}", realm="{realm}", ' + f'nonce="{nonce}", uri="{request.url.path}", ' + f'nc={nc}, cnonce="{cnonce}", response="{response_hash}"' + ) + + +class NetRCAuth(Auth): + """Load credentials from ~/.netrc based on the request host.""" + + def auth_flow(self, request: Request): + import netrc + try: + credentials = netrc.netrc().authenticators(request.url.host) + if credentials: + username, _, password = credentials + basic = BasicAuth(username, password) + yield from basic.auth_flow(request) + return + except Exception: + pass + yield request diff --git a/worked/httpx/raw/client.py b/worked/httpx/raw/client.py new file mode 100644 index 000000000..d506dd613 --- /dev/null +++ b/worked/httpx/raw/client.py @@ -0,0 +1,161 @@ +""" +The main Client and AsyncClient classes. +BaseClient holds all shared logic. Client and AsyncClient extend it for sync/async. +This is the integration hub of the library - it imports from every other module. +""" +from models import Request, Response, URL, Headers, Cookies +from auth import Auth, BasicAuth +from transport import BaseTransport, HTTPTransport, AsyncHTTPTransport +from exceptions import TooManyRedirects, InvalidURL +from utils import build_url_with_params, obfuscate_sensitive_headers + + +DEFAULT_MAX_REDIRECTS = 20 + + +class Timeout: + def __init__(self, timeout=5.0, *, connect=None, read=None, write=None, pool=None): + self.connect = connect or timeout + self.read = read or timeout + self.write = write or timeout + self.pool = pool or timeout + + +class Limits: + def __init__(self, max_connections=100, max_keepalive_connections=20, keepalive_expiry=5.0): + self.max_connections = max_connections + self.max_keepalive_connections = max_keepalive_connections + self.keepalive_expiry = keepalive_expiry + + +class BaseClient: + """ + Shared implementation for Client and AsyncClient. + Handles auth, redirects, cookies, and header defaults. + """ + + def __init__( + self, + *, + auth=None, + headers=None, + cookies=None, + timeout=Timeout(), + max_redirects=DEFAULT_MAX_REDIRECTS, + base_url="", + ): + self._auth = auth + self._headers = Headers(headers or {}) + self._cookies = Cookies(cookies or {}) + self._timeout = timeout + self._max_redirects = max_redirects + self._base_url = URL(base_url) if base_url else None + + def _build_request(self, method: str, url: str, **kwargs) -> Request: + if self._base_url: + url = self._base_url.raw.rstrip("/") + "/" + url.lstrip("/") + if kwargs.get("params"): + url = build_url_with_params(url, kwargs.pop("params")) + headers = Headers(kwargs.get("headers", {})) + for k, v in self._headers.items(): + if k not in headers: + headers[k] = v + return Request(method, url, headers=headers, content=kwargs.get("content"), cookies=self._cookies) + + def _merge_cookies(self, response: Response) -> None: + for name, value in response.cookies.items(): + self._cookies.set(name, value) + + +class Client(BaseClient): + """Synchronous HTTP client.""" + + def __init__(self, *, transport: BaseTransport = None, **kwargs): + super().__init__(**kwargs) + self._transport = transport or HTTPTransport() + + def request(self, method: str, url: str, **kwargs) -> Response: + request = self._build_request(method, url, **kwargs) + auth = kwargs.get("auth") or self._auth + if auth: + flow = auth.auth_flow(request) + request = next(flow) + response = self._transport.handle_request(request) + self._merge_cookies(response) + if auth: + try: + flow.send(response) + except StopIteration: + pass + return response + + def get(self, url: str, **kwargs) -> Response: + return self.request("GET", url, **kwargs) + + def post(self, url: str, **kwargs) -> Response: + return self.request("POST", url, **kwargs) + + def put(self, url: str, **kwargs) -> Response: + return self.request("PUT", url, **kwargs) + + def patch(self, url: str, **kwargs) -> Response: + return self.request("PATCH", url, **kwargs) + + def delete(self, url: str, **kwargs) -> Response: + return self.request("DELETE", url, **kwargs) + + def head(self, url: str, **kwargs) -> Response: + return self.request("HEAD", url, **kwargs) + + def send(self, request: Request) -> Response: + return self._transport.handle_request(request) + + def close(self) -> None: + self._transport.close() + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + +class AsyncClient(BaseClient): + """Asynchronous HTTP client.""" + + def __init__(self, *, transport=None, **kwargs): + super().__init__(**kwargs) + self._transport = transport or AsyncHTTPTransport() + + async def request(self, method: str, url: str, **kwargs) -> Response: + request = self._build_request(method, url, **kwargs) + response = await self._transport.handle_async_request(request) + self._merge_cookies(response) + return response + + async def get(self, url: str, **kwargs) -> Response: + return await self.request("GET", url, **kwargs) + + async def post(self, url: str, **kwargs) -> Response: + return await self.request("POST", url, **kwargs) + + async def put(self, url: str, **kwargs) -> Response: + return await self.request("PUT", url, **kwargs) + + async def patch(self, url: str, **kwargs) -> Response: + return await self.request("PATCH", url, **kwargs) + + async def delete(self, url: str, **kwargs) -> Response: + return await self.request("DELETE", url, **kwargs) + + async def send(self, request: Request) -> Response: + return await self._transport.handle_async_request(request) + + async def aclose(self) -> None: + await self._transport.aclose() + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + await self.aclose() diff --git a/worked/httpx/raw/exceptions.py b/worked/httpx/raw/exceptions.py new file mode 100644 index 000000000..ff5392fee --- /dev/null +++ b/worked/httpx/raw/exceptions.py @@ -0,0 +1,90 @@ +""" +httpx-like exception hierarchy. +All exceptions inherit from HTTPError at the top. +""" + + +class HTTPError(Exception): + """Base class for all httpx exceptions.""" + def __init__(self, message, *, request=None): + self.request = request + super().__init__(message) + + +class RequestError(HTTPError): + """An error occurred while issuing a request.""" + + +class TransportError(RequestError): + """An error occurred at the transport layer.""" + + +class TimeoutException(TransportError): + """A timeout occurred.""" + + +class ConnectTimeout(TimeoutException): + """Timed out while connecting to the host.""" + + +class ReadTimeout(TimeoutException): + """Timed out while receiving data from the host.""" + + +class WriteTimeout(TimeoutException): + """Timed out while sending data to the host.""" + + +class PoolTimeout(TimeoutException): + """Timed out waiting to acquire a connection from the pool.""" + + +class NetworkError(TransportError): + """A network error occurred.""" + + +class ConnectError(NetworkError): + """Failed to establish a connection.""" + + +class ReadError(NetworkError): + """Failed to receive data from the network.""" + + +class WriteError(NetworkError): + """Failed to send data through the network.""" + + +class CloseError(NetworkError): + """Failed to close a connection.""" + + +class ProxyError(TransportError): + """An error occurred while establishing a proxy connection.""" + + +class ProtocolError(TransportError): + """A protocol was violated.""" + + +class DecodingError(RequestError): + """Decoding of the response failed.""" + + +class TooManyRedirects(RequestError): + """Too many redirects.""" + + +class HTTPStatusError(HTTPError): + """A 4xx or 5xx response was received.""" + def __init__(self, message, *, request, response): + self.response = response + super().__init__(message, request=request) + + +class InvalidURL(Exception): + """URL is improperly formed or cannot be parsed.""" + + +class CookieConflict(Exception): + """Attempted to look up a cookie by name but multiple cookies exist.""" diff --git a/worked/httpx/raw/models.py b/worked/httpx/raw/models.py new file mode 100644 index 000000000..80582b6fa --- /dev/null +++ b/worked/httpx/raw/models.py @@ -0,0 +1,120 @@ +""" +Core data models: URL, Headers, Cookies, Request, Response. +These are the central data types that everything else in the library references. +""" +import json as _json +from exceptions import HTTPStatusError + + +class URL: + def __init__(self, url: str): + self.raw = url + self.scheme, _, rest = url.partition("://") + self.host, _, self.path = rest.partition("/") + self.path = "/" + self.path + + def copy_with(self, **kwargs) -> "URL": + return URL(kwargs.get("url", self.raw)) + + def __str__(self): + return self.raw + + def __repr__(self): + return f"URL({self.raw!r})" + + +class Headers: + def __init__(self, headers=None): + self._store = {} + for k, v in (headers or {}).items(): + self._store[k.lower()] = v + + def get(self, key: str, default=None): + return self._store.get(key.lower(), default) + + def items(self): + return self._store.items() + + def __setitem__(self, key, value): + self._store[key.lower()] = value + + def __getitem__(self, key): + return self._store[key.lower()] + + def __contains__(self, key): + return key.lower() in self._store + + +class Cookies: + def __init__(self, cookies=None): + self._jar = dict(cookies or {}) + + def set(self, name: str, value: str, domain: str = "") -> None: + self._jar[name] = value + + def get(self, name: str, default=None): + return self._jar.get(name, default) + + def delete(self, name: str) -> None: + self._jar.pop(name, None) + + def clear(self) -> None: + self._jar.clear() + + def items(self): + return self._jar.items() + + +class Request: + def __init__(self, method: str, url, *, headers=None, content=None, cookies=None): + self.method = method.upper() + self.url = URL(url) if isinstance(url, str) else url + self.headers = Headers(headers) + self.content = content or b"" + self.cookies = Cookies(cookies) + + def __repr__(self): + return f"" + + +class Response: + def __init__(self, status_code: int, *, headers=None, content=None, request=None): + self.status_code = status_code + self.headers = Headers(headers) + self.content = content or b"" + self.request = request + + @property + def text(self) -> str: + return self.content.decode("utf-8", errors="replace") + + def json(self): + return _json.loads(self.content) + + def read(self) -> bytes: + return self.content + + @property + def is_success(self) -> bool: + return 200 <= self.status_code < 300 + + @property + def is_error(self) -> bool: + return self.status_code >= 400 + + def raise_for_status(self) -> None: + if self.is_error: + message = f"{self.status_code} Error" + raise HTTPStatusError(message, request=self.request, response=self) + + @property + def cookies(self) -> Cookies: + jar = Cookies() + for header in self.headers.get("set-cookie", "").split(","): + if "=" in header: + name, _, value = header.strip().partition("=") + jar.set(name.strip(), value.split(";")[0].strip()) + return jar + + def __repr__(self): + return f"" diff --git a/worked/httpx/raw/transport.py b/worked/httpx/raw/transport.py new file mode 100644 index 000000000..5bd9b9166 --- /dev/null +++ b/worked/httpx/raw/transport.py @@ -0,0 +1,135 @@ +""" +Transport layer: connection management and low-level HTTP sending. +HTTPTransport wraps a connection pool. ProxyTransport sits in front of it. +MockTransport is used in tests. +""" +from models import Request, Response +from exceptions import TransportError, ConnectError, TimeoutException + + +class BaseTransport: + """Sync transport interface.""" + + def handle_request(self, request: Request) -> Response: + raise NotImplementedError + + def close(self) -> None: + pass + + +class AsyncBaseTransport: + """Async transport interface.""" + + async def handle_async_request(self, request: Request) -> Response: + raise NotImplementedError + + async def aclose(self) -> None: + pass + + +class ConnectionPool: + """ + Manages a pool of persistent HTTP connections. + Keys connections by (scheme, host, port). + """ + + def __init__(self, max_connections=100, max_keepalive_connections=20): + self.max_connections = max_connections + self.max_keepalive_connections = max_keepalive_connections + self._pool = {} + + def _get_connection_key(self, request: Request) -> tuple: + url = request.url + port = 443 if url.scheme == "https" else 80 + return (url.scheme, url.host, port) + + def get_connection(self, request: Request): + key = self._get_connection_key(request) + return self._pool.get(key) + + def return_connection(self, request: Request, conn) -> None: + key = self._get_connection_key(request) + if len(self._pool) < self.max_keepalive_connections: + self._pool[key] = conn + + def close(self) -> None: + self._pool.clear() + + +class HTTPTransport(BaseTransport): + """ + The main sync HTTP transport. + Uses a ConnectionPool for connection reuse. + """ + + def __init__(self, verify=True, cert=None, limits=None): + self.verify = verify + self.cert = cert + self._pool = ConnectionPool() + + def handle_request(self, request: Request) -> Response: + conn = self._pool.get_connection(request) + try: + response = self._send(request, conn) + self._pool.return_connection(request, conn) + return response + except TimeoutException: + raise + except Exception as exc: + raise ConnectError(str(exc)) from exc + + def _send(self, request: Request, conn) -> Response: + # Simplified: in real httpx this does the actual socket I/O + return Response(200, headers={}, content=b"", request=request) + + def close(self) -> None: + self._pool.close() + + +class AsyncHTTPTransport(AsyncBaseTransport): + """The async variant of HTTPTransport.""" + + def __init__(self, verify=True, cert=None): + self.verify = verify + self.cert = cert + + async def handle_async_request(self, request: Request) -> Response: + return Response(200, headers={}, content=b"", request=request) + + async def aclose(self) -> None: + pass + + +class MockTransport(BaseTransport): + """ + A transport for testing that returns predefined responses. + Pass a handler function that receives a Request and returns a Response. + """ + + def __init__(self, handler): + self.handler = handler + + def handle_request(self, request: Request) -> Response: + return self.handler(request) + + +class ProxyTransport(BaseTransport): + """ + Routes requests through an HTTP/HTTPS proxy. + Wraps an inner transport and prepends proxy connection handling. + """ + + def __init__(self, proxy_url: str, *, inner: BaseTransport = None): + self.proxy_url = proxy_url + self._inner = inner or HTTPTransport() + + def handle_request(self, request: Request) -> Response: + try: + return self._inner.handle_request(request) + except TransportError: + raise + except Exception as exc: + raise TransportError(f"Proxy error: {exc}") from exc + + def close(self) -> None: + self._inner.close() diff --git a/worked/httpx/raw/utils.py b/worked/httpx/raw/utils.py new file mode 100644 index 000000000..84ca4a3b8 --- /dev/null +++ b/worked/httpx/raw/utils.py @@ -0,0 +1,85 @@ +""" +Utility functions shared across the library. +Small helpers that don't belong in any one module. +""" +import re +from models import Cookies + + +SENSITIVE_HEADERS = {"authorization", "cookie", "set-cookie", "proxy-authorization"} + + +def primitive_value_to_str(value) -> str: + """Convert a primitive value to its string representation.""" + if isinstance(value, bool): + return "true" if value else "false" + return str(value) + + +def normalize_header_key(key: str) -> str: + """Convert a header key to its canonical Title-Case form.""" + return "-".join(word.capitalize() for word in key.split("-")) + + +def flatten_queryparams(params: dict) -> list: + """ + Expand a params dict into a flat list of (key, value) pairs. + List values become multiple pairs with the same key. + """ + result = [] + for key, value in params.items(): + if isinstance(value, list): + for item in value: + result.append((key, primitive_value_to_str(item))) + else: + result.append((key, primitive_value_to_str(value))) + return result + + +def parse_content_type(content_type: str) -> tuple: + """ + Parse a Content-Type header value. + Returns (media_type, params_dict). + Example: 'application/json; charset=utf-8' -> ('application/json', {'charset': 'utf-8'}) + """ + parts = [p.strip() for p in content_type.split(";")] + media_type = parts[0] + params = {} + for part in parts[1:]: + if "=" in part: + key, _, value = part.partition("=") + params[key.strip()] = value.strip() + return media_type, params + + +def obfuscate_sensitive_headers(headers: dict) -> dict: + """Return a copy of headers with sensitive values replaced by [obfuscated].""" + return { + k: "[obfuscated]" if k.lower() in SENSITIVE_HEADERS else v + for k, v in headers.items() + } + + +def unset_all_cookies(cookies: Cookies) -> None: + """Clear all cookies from a cookie jar in place.""" + cookies.clear() + + +def is_known_encoding(encoding: str) -> bool: + """Check if a character encoding label is recognized by Python's codec system.""" + import codecs + try: + codecs.lookup(encoding) + return True + except LookupError: + return False + + +def build_url_with_params(base_url: str, params: dict) -> str: + """Append query parameters to a URL string.""" + if not params: + return base_url + pairs = flatten_queryparams(params) + query = "&".join(f"{k}={v}" for k, v in pairs) + separator = "&" if "?" in base_url else "?" + return f"{base_url}{separator}{query}" diff --git a/worked/karpathy-repos/README.md b/worked/karpathy-repos/README.md new file mode 100644 index 000000000..849897284 --- /dev/null +++ b/worked/karpathy-repos/README.md @@ -0,0 +1,63 @@ +# Karpathy Repos Benchmark — How to Reproduce + +This is the corpus that produced the 71.5x token reduction benchmark. + +## Corpus (52 files) + +### Code — clone these 3 repos + +```bash +git clone https://github.com/karpathy/nanoGPT +git clone https://github.com/karpathy/minGPT +git clone https://github.com/karpathy/micrograd +``` + +### Papers — download these 5 PDFs + +- Attention Is All You Need — https://arxiv.org/abs/1706.03762 +- FlashAttention: Fast and Memory-Efficient Exact Attention — https://arxiv.org/abs/2205.14135 +- FlashAttention-2 — https://arxiv.org/abs/2307.08691 +- Neural Attention Residuals — https://arxiv.org/abs/2505.03840 +- NeuralWalker: Graph Neural Networks with Walk-Based Attention — https://arxiv.org/abs/2502.02593 + +### Images — save these 4 + +- `gpt2_124M_loss.png` — nanoGPT training loss curve (in the nanoGPT repo) +- `gout.svg` — micrograd computation graph (in the micrograd repo) +- `moon_mlp.png` — MLP decision boundary (in the micrograd repo) +- Any screenshot or diagram from the Attention Is All You Need paper + +## How to run + +Put all files into a single folder called `raw/`: + +``` +raw/ +├── nanoGPT/ (cloned repo) +├── minGPT/ (cloned repo) +├── micrograd/ (cloned repo) +├── attention.pdf +├── flashattention.pdf +├── flashattention2.pdf +├── attn_residuals.pdf +├── neuralwalker.pdf +├── gpt2_124M_loss.png +├── gout.svg +└── moon_mlp.png +``` + +Then in Claude Code: + +``` +pip install graphifyy && graphify install +/graphify ./raw +``` + +## What to expect + +- ~285 nodes, ~340 edges, ~17 meaningful communities +- God nodes: `Value` (micrograd), `GPT` (nanoGPT), `Training Script`, `Layer` +- Surprising connections: nanoGPT Block and minGPT Block linked across repos, FlashAttention paper bridging into CausalSelfAttention in both repos +- Token reduction: 71.5x vs reading all 52 files cold + +Full eval with scores and analysis: `review.md` diff --git a/worked/mixed-corpus/README.md b/worked/mixed-corpus/README.md new file mode 100644 index 000000000..e62c32fef --- /dev/null +++ b/worked/mixed-corpus/README.md @@ -0,0 +1,45 @@ +# Mixed Corpus Benchmark — How to Reproduce + +A small but realistic mixed-input corpus: Python source files, a markdown paper with +arXiv citations, and one image. Tests graphify's ability to handle different file types +in a single run. + +## Corpus (5 files) + +All input files are in `raw/`: + +``` +raw/ +├── analyze.py — graphify's graph analysis module (god_nodes, surprising_connections, etc.) +├── build.py — graphify's graph builder (build_from_json, networkx wrapper) +├── cluster.py — graphify's Leiden community detection (cluster, score_all) +├── attention_notes.md — Transformer paper notes (Vaswani et al., 2017), with arXiv citation +``` + +Note: the original benchmark included `attention_arabic.png` (an Arabic-language figure from the +Attention paper). PNG files are not stored in this repo. To reproduce with the image, save any +diagram or figure from the Attention Is All You Need paper as `raw/attention_arabic.png`. + +## How to run + +```bash +pip install graphifyy && graphify install +/graphify ./raw +``` + +Or from the CLI directly: + +```bash +pip install graphifyy +graphify ./raw +``` + +## What to expect + +- ~20 nodes, ~19 edges from AST alone (3 Python modules) +- 3 communities: Graph Analysis, Clustering & Scoring, Graph Building +- God nodes: `analyze.py`, `cluster.py`, `build.py` +- `attention_notes.md` classified as `paper` (arXiv heuristic fires on `1706.03762`) +- If you include the image: 1 extra node describing the figure content via vision + +Full eval with scores and analysis: `review.md` diff --git a/worked/mixed-corpus/raw/analyze.py b/worked/mixed-corpus/raw/analyze.py new file mode 100644 index 000000000..cf5344960 --- /dev/null +++ b/worked/mixed-corpus/raw/analyze.py @@ -0,0 +1,517 @@ +"""Graph analysis: god nodes (most connected), surprising connections (cross-community), suggested questions.""" +from __future__ import annotations +import networkx as nx + + +def _node_community_map(communities: dict[int, list[str]]) -> dict[str, int]: + """Invert communities dict: node_id -> community_id.""" + return {n: cid for cid, nodes in communities.items() for n in nodes} + + +def _is_file_node(G: nx.Graph, node_id: str) -> bool: + """ + Return True if this node is a file-level hub node (e.g. 'client', 'models') + or an AST method stub (e.g. '.auth_flow()', '.__init__()'). + + These are synthetic nodes created by the AST extractor and should be excluded + from god nodes, surprising connections, and knowledge gap reporting. + """ + label = G.nodes[node_id].get("label", "") + if not label: + return False + # File-level hub: label is a filename with a code extension + if label.split(".")[-1] in ("py", "ts", "js", "go", "rs", "java", "rb", "cpp", "c", "h"): + return True + # Method stub: AST extractor labels methods as '.method_name()' + if label.startswith(".") and label.endswith("()"): + return True + # Module-level function stub: labeled 'function_name()' - only has a contains edge + # These are real functions but structurally isolated by definition; not a gap worth flagging + if label.endswith("()") and G.degree(node_id) <= 1: + return True + return False + + +def god_nodes(G: nx.Graph, top_n: int = 10) -> list[dict]: + """Return the top_n most-connected real entities - the core abstractions. + + File-level hub nodes are excluded: they accumulate import/contains edges + mechanically and don't represent meaningful architectural abstractions. + """ + degree = dict(G.degree()) + sorted_nodes = sorted(degree.items(), key=lambda x: x[1], reverse=True) + result = [] + for node_id, deg in sorted_nodes: + if _is_file_node(G, node_id) or _is_concept_node(G, node_id): + continue + result.append({ + "id": node_id, + "label": G.nodes[node_id].get("label", node_id), + "edges": deg, + }) + if len(result) >= top_n: + break + return result + + +def surprising_connections( + G: nx.Graph, + communities: dict[int, list[str]] | None = None, + top_n: int = 5, +) -> list[dict]: + """ + Find connections that are genuinely surprising - not obvious from file structure. + + Strategy: + - Multi-file corpora: cross-file edges between real entities (not concept nodes). + Sorted AMBIGUOUS → INFERRED → EXTRACTED. + - Single-file / single-source corpora: cross-community edges that bridge + distant parts of the graph (betweenness centrality on edges). + These reveal non-obvious structural couplings. + + Concept nodes (empty source_file, or injected semantic annotations) are excluded + from surprising connections because they are intentional, not discovered. + """ + # Identify unique source files (ignore empty/null source_file) + source_files = { + data.get("source_file", "") + for _, data in G.nodes(data=True) + if data.get("source_file", "") + } + is_multi_source = len(source_files) > 1 + + if is_multi_source: + return _cross_file_surprises(G, communities or {}, top_n) + else: + return _cross_community_surprises(G, communities or {}, top_n) + + +def _is_concept_node(G: nx.Graph, node_id: str) -> bool: + """ + Return True if this node is a manually-injected semantic concept node + rather than a real entity found in source code. + + Signals: + - Empty source_file + - source_file doesn't look like a real file path (no extension) + """ + data = G.nodes[node_id] + source = data.get("source_file", "") + if not source: + return True + # Has no file extension → probably a concept label, not a real file + if "." not in source.split("/")[-1]: + return True + return False + + +_CODE_EXTENSIONS = {"py", "ts", "tsx", "js", "go", "rs", "java", "rb", "cpp", "c", "h", "cs", "kt", "scala", "php"} +_DOC_EXTENSIONS = {"md", "txt", "rst"} +_PAPER_EXTENSIONS = {"pdf"} +_IMAGE_EXTENSIONS = {"png", "jpg", "jpeg", "webp", "gif", "svg"} + + +def _file_category(path: str) -> str: + ext = path.rsplit(".", 1)[-1].lower() if "." in path else "" + if ext in _CODE_EXTENSIONS: + return "code" + if ext in _PAPER_EXTENSIONS: + return "paper" + if ext in _IMAGE_EXTENSIONS: + return "image" + return "doc" + + +def _top_level_dir(path: str) -> str: + """Return the first path component - used to detect cross-repo edges.""" + return path.split("/")[0] if "/" in path else path + + +def _surprise_score( + G: nx.Graph, + u: str, + v: str, + data: dict, + node_community: dict[str, int], + u_source: str, + v_source: str, +) -> tuple[int, list[str]]: + """Score how surprising a cross-file edge is. Returns (score, reasons).""" + score = 0 + reasons: list[str] = [] + + # 1. Confidence weight - uncertain connections are more noteworthy + conf = data.get("confidence", "EXTRACTED") + conf_bonus = {"AMBIGUOUS": 3, "INFERRED": 2, "EXTRACTED": 1}.get(conf, 1) + score += conf_bonus + if conf in ("AMBIGUOUS", "INFERRED"): + reasons.append(f"{conf.lower()} connection - not explicitly stated in source") + + # 2. Cross file-type bonus - code↔paper or code↔image is non-obvious + cat_u = _file_category(u_source) + cat_v = _file_category(v_source) + if cat_u != cat_v: + score += 2 + reasons.append(f"crosses file types ({cat_u} ↔ {cat_v})") + + # 3. Cross-repo bonus - different top-level directory + if _top_level_dir(u_source) != _top_level_dir(v_source): + score += 2 + reasons.append("connects across different repos/directories") + + # 4. Cross-community bonus - Leiden says these are structurally distant + cid_u = node_community.get(u) + cid_v = node_community.get(v) + if cid_u is not None and cid_v is not None and cid_u != cid_v: + score += 1 + reasons.append("bridges separate communities") + + # 5. Peripheral→hub: a low-degree node connecting to a high-degree one + deg_u = G.degree(u) + deg_v = G.degree(v) + if min(deg_u, deg_v) <= 2 and max(deg_u, deg_v) >= 5: + score += 1 + peripheral = G.nodes[u].get("label", u) if deg_u <= 2 else G.nodes[v].get("label", v) + hub = G.nodes[v].get("label", v) if deg_u <= 2 else G.nodes[u].get("label", u) + reasons.append(f"peripheral node `{peripheral}` unexpectedly reaches hub `{hub}`") + + return score, reasons + + +def _cross_file_surprises(G: nx.Graph, communities: dict[int, list[str]], top_n: int) -> list[dict]: + """ + Cross-file edges between real code/doc entities, ranked by a composite + surprise score rather than confidence alone. + + Surprise score accounts for: + - Confidence (AMBIGUOUS > INFERRED > EXTRACTED) + - Cross file-type (code↔paper is more surprising than code↔code) + - Cross-repo (different top-level directory) + - Cross-community (Leiden says structurally distant) + - Peripheral→hub (low-degree node reaching a god node) + + Each result includes a 'why' field explaining what makes it non-obvious. + """ + node_community = _node_community_map(communities) + candidates = [] + + for u, v, data in G.edges(data=True): + relation = data.get("relation", "") + if relation in ("imports", "imports_from", "contains", "method"): + continue + if _is_concept_node(G, u) or _is_concept_node(G, v): + continue + if _is_file_node(G, u) or _is_file_node(G, v): + continue + + u_source = G.nodes[u].get("source_file", "") + v_source = G.nodes[v].get("source_file", "") + + if not u_source or not v_source or u_source == v_source: + continue + + score, reasons = _surprise_score(G, u, v, data, node_community, u_source, v_source) + src_id = data.get("_src", u) + tgt_id = data.get("_tgt", v) + candidates.append({ + "_score": score, + "source": G.nodes[src_id].get("label", src_id), + "target": G.nodes[tgt_id].get("label", tgt_id), + "source_files": [ + G.nodes[src_id].get("source_file", ""), + G.nodes[tgt_id].get("source_file", ""), + ], + "confidence": data.get("confidence", "EXTRACTED"), + "relation": relation, + "why": "; ".join(reasons) if reasons else "cross-file semantic connection", + }) + + candidates.sort(key=lambda x: x["_score"], reverse=True) + for c in candidates: + c.pop("_score") + + if candidates: + return candidates[:top_n] + + return _cross_community_surprises(G, communities, top_n) + + +def _cross_community_surprises( + G: nx.Graph, + communities: dict[int, list[str]], + top_n: int, +) -> list[dict]: + """ + For single-source corpora: find edges that bridge different communities. + These are surprising because Leiden grouped everything else tightly - + these edges cut across the natural structure. + + Falls back to high-betweenness edges if no community info is provided. + """ + if not communities: + # No community info - use edge betweenness centrality + if G.number_of_edges() == 0: + return [] + betweenness = nx.edge_betweenness_centrality(G) + top_edges = sorted(betweenness.items(), key=lambda x: x[1], reverse=True)[:top_n] + result = [] + for (u, v), score in top_edges: + data = G.edges[u, v] + result.append({ + "source": G.nodes[u].get("label", u), + "target": G.nodes[v].get("label", v), + "source_files": [ + G.nodes[u].get("source_file", ""), + G.nodes[v].get("source_file", ""), + ], + "confidence": data.get("confidence", "EXTRACTED"), + "relation": data.get("relation", ""), + "note": f"Bridges graph structure (betweenness={score:.3f})", + }) + return result + + # Build node → community map + node_community = _node_community_map(communities) + + surprises = [] + for u, v, data in G.edges(data=True): + cid_u = node_community.get(u) + cid_v = node_community.get(v) + if cid_u is None or cid_v is None or cid_u == cid_v: + continue + # Skip file hub nodes and plain structural edges + if _is_file_node(G, u) or _is_file_node(G, v): + continue + relation = data.get("relation", "") + if relation in ("imports", "imports_from", "contains", "method"): + continue + # This edge crosses community boundaries - interesting + confidence = data.get("confidence", "EXTRACTED") + src_id = data.get("_src", u) + tgt_id = data.get("_tgt", v) + surprises.append({ + "source": G.nodes[src_id].get("label", src_id), + "target": G.nodes[tgt_id].get("label", tgt_id), + "source_files": [ + G.nodes[src_id].get("source_file", ""), + G.nodes[tgt_id].get("source_file", ""), + ], + "confidence": confidence, + "relation": relation, + "note": f"Bridges community {cid_u} → community {cid_v}", + "_pair": tuple(sorted([cid_u, cid_v])), + }) + + # Sort: AMBIGUOUS first, then INFERRED, then EXTRACTED + order = {"AMBIGUOUS": 0, "INFERRED": 1, "EXTRACTED": 2} + surprises.sort(key=lambda x: order.get(x["confidence"], 3)) + + # Deduplicate by community pair - one representative edge per (A→B) boundary. + # Without this, a single high-betweenness god node dominates all results. + seen_pairs: set[tuple] = set() + deduped = [] + for s in surprises: + pair = s.pop("_pair") + if pair not in seen_pairs: + seen_pairs.add(pair) + deduped.append(s) + return deduped[:top_n] + + +def suggest_questions( + G: nx.Graph, + communities: dict[int, list[str]], + community_labels: dict[int, str], + top_n: int = 7, +) -> list[dict]: + """ + Generate questions the graph is uniquely positioned to answer. + Based on: AMBIGUOUS edges, bridge nodes, underexplored god nodes, isolated nodes. + Each question has a 'type', 'question', and 'why' field. + """ + questions = [] + node_community = _node_community_map(communities) + + # 1. AMBIGUOUS edges → unresolved relationship questions + for u, v, data in G.edges(data=True): + if data.get("confidence") == "AMBIGUOUS": + ul = G.nodes[u].get("label", u) + vl = G.nodes[v].get("label", v) + relation = data.get("relation", "related to") + questions.append({ + "type": "ambiguous_edge", + "question": f"What is the exact relationship between `{ul}` and `{vl}`?", + "why": f"Edge tagged AMBIGUOUS (relation: {relation}) - confidence is low.", + }) + + # 2. Bridge nodes (high betweenness) → cross-cutting concern questions + if G.number_of_edges() > 0: + betweenness = nx.betweenness_centrality(G) + # Top bridge nodes that are NOT file-level hubs + bridges = sorted( + [(n, s) for n, s in betweenness.items() + if not _is_file_node(G, n) and not _is_concept_node(G, n) and s > 0], + key=lambda x: x[1], + reverse=True, + )[:3] + for node_id, score in bridges: + label = G.nodes[node_id].get("label", node_id) + cid = node_community.get(node_id) + comm_label = community_labels.get(cid, f"Community {cid}") if cid is not None else "unknown" + neighbors = list(G.neighbors(node_id)) + neighbor_comms = {node_community.get(n) for n in neighbors if node_community.get(n) != cid} + if neighbor_comms: + other_labels = [community_labels.get(c, f"Community {c}") for c in neighbor_comms] + questions.append({ + "type": "bridge_node", + "question": f"Why does `{label}` connect `{comm_label}` to {', '.join(f'`{l}`' for l in other_labels)}?", + "why": f"High betweenness centrality ({score:.3f}) - this node is a cross-community bridge.", + }) + + # 3. God nodes with many INFERRED edges → verification questions + degree = dict(G.degree()) + top_nodes = sorted( + [(n, d) for n, d in degree.items() if not _is_file_node(G, n)], + key=lambda x: x[1], + reverse=True, + )[:5] + for node_id, _ in top_nodes: + inferred = [ + (u, v, d) for u, v, d in G.edges(node_id, data=True) + if d.get("confidence") == "INFERRED" + ] + if len(inferred) >= 2: + label = G.nodes[node_id].get("label", node_id) + # Use _src/_tgt to get the correct direction; fall back to v (the other node) + others = [] + for u, v, d in inferred[:2]: + src_id = d.get("_src", u) + tgt_id = d.get("_tgt", v) + other_id = tgt_id if src_id == node_id else src_id + others.append(G.nodes[other_id].get("label", other_id)) + questions.append({ + "type": "verify_inferred", + "question": f"Are the {len(inferred)} inferred relationships involving `{label}` (e.g. with `{others[0]}` and `{others[1]}`) actually correct?", + "why": f"`{label}` has {len(inferred)} INFERRED edges - model-reasoned connections that need verification.", + }) + + # 4. Isolated or weakly-connected nodes → exploration questions + isolated = [ + n for n in G.nodes() + if G.degree(n) <= 1 and not _is_file_node(G, n) and not _is_concept_node(G, n) + ] + if isolated: + labels = [G.nodes[n].get("label", n) for n in isolated[:3]] + questions.append({ + "type": "isolated_nodes", + "question": f"What connects {', '.join(f'`{l}`' for l in labels)} to the rest of the system?", + "why": f"{len(isolated)} weakly-connected nodes found - possible documentation gaps or missing edges.", + }) + + # 5. Low-cohesion communities → structural questions + from .cluster import cohesion_score + for cid, nodes in communities.items(): + score = cohesion_score(G, nodes) + if score < 0.15 and len(nodes) >= 5: + label = community_labels.get(cid, f"Community {cid}") + questions.append({ + "type": "low_cohesion", + "question": f"Should `{label}` be split into smaller, more focused modules?", + "why": f"Cohesion score {score} - nodes in this community are weakly interconnected.", + }) + + if not questions: + return [{ + "type": "no_signal", + "question": None, + "why": ( + "Not enough signal to generate questions. " + "This usually means the corpus has no AMBIGUOUS edges, no bridge nodes, " + "no INFERRED relationships, and all communities are tightly cohesive. " + "Add more files or run with --mode deep to extract richer edges." + ), + }] + + return questions[:top_n] + + +def graph_diff(G_old: nx.Graph, G_new: nx.Graph) -> dict: + """Compare two graph snapshots and return what changed. + + Returns: + { + "new_nodes": [{"id": ..., "label": ...}], + "removed_nodes": [{"id": ..., "label": ...}], + "new_edges": [{"source": ..., "target": ..., "relation": ..., "confidence": ...}], + "removed_edges": [...], + "summary": "3 new nodes, 5 new edges, 1 node removed" + } + """ + old_nodes = set(G_old.nodes()) + new_nodes = set(G_new.nodes()) + + added_node_ids = new_nodes - old_nodes + removed_node_ids = old_nodes - new_nodes + + new_nodes_list = [ + {"id": n, "label": G_new.nodes[n].get("label", n)} + for n in added_node_ids + ] + removed_nodes_list = [ + {"id": n, "label": G_old.nodes[n].get("label", n)} + for n in removed_node_ids + ] + + def edge_key(G: nx.Graph, u: str, v: str, data: dict) -> tuple: + return (u, v, data.get("relation", "")) + + old_edge_keys = { + edge_key(G_old, u, v, d) + for u, v, d in G_old.edges(data=True) + } + new_edge_keys = { + edge_key(G_new, u, v, d) + for u, v, d in G_new.edges(data=True) + } + + added_edge_keys = new_edge_keys - old_edge_keys + removed_edge_keys = old_edge_keys - new_edge_keys + + new_edges_list = [] + for u, v, d in G_new.edges(data=True): + if edge_key(G_new, u, v, d) in added_edge_keys: + new_edges_list.append({ + "source": u, + "target": v, + "relation": d.get("relation", ""), + "confidence": d.get("confidence", ""), + }) + + removed_edges_list = [] + for u, v, d in G_old.edges(data=True): + if edge_key(G_old, u, v, d) in removed_edge_keys: + removed_edges_list.append({ + "source": u, + "target": v, + "relation": d.get("relation", ""), + "confidence": d.get("confidence", ""), + }) + + parts = [] + if new_nodes_list: + parts.append(f"{len(new_nodes_list)} new node{'s' if len(new_nodes_list) != 1 else ''}") + if new_edges_list: + parts.append(f"{len(new_edges_list)} new edge{'s' if len(new_edges_list) != 1 else ''}") + if removed_nodes_list: + parts.append(f"{len(removed_nodes_list)} node{'s' if len(removed_nodes_list) != 1 else ''} removed") + if removed_edges_list: + parts.append(f"{len(removed_edges_list)} edge{'s' if len(removed_edges_list) != 1 else ''} removed") + summary = ", ".join(parts) if parts else "no changes" + + return { + "new_nodes": new_nodes_list, + "removed_nodes": removed_nodes_list, + "new_edges": new_edges_list, + "removed_edges": removed_edges_list, + "summary": summary, + } diff --git a/worked/mixed-corpus/raw/attention_notes.md b/worked/mixed-corpus/raw/attention_notes.md new file mode 100644 index 000000000..6a60166f8 --- /dev/null +++ b/worked/mixed-corpus/raw/attention_notes.md @@ -0,0 +1,76 @@ +# Attention Mechanism Notes + +Notes on the Transformer architecture from Vaswani et al., 2017. +arXiv: 1706.03762 + +## Abstract + +The dominant sequence transduction models are based on complex recurrent or convolutional neural networks that include an encoder and a decoder. The best performing models also connect the encoder and decoder through an attention mechanism. The Transformer is a model architecture eschewing recurrence and instead relying entirely on an attention mechanism to draw global dependencies between input and output. + +## Multi-Head Attention + +The model uses h=8 parallel attention heads. For each head, d_k = d_v = d_model/h = 64. + +Scaled dot-product attention: + + Attention(Q, K, V) = softmax(QK^T / sqrt(d_k)) V + +Multi-head attention runs h attention functions in parallel, then concatenates and projects: + + MultiHead(Q, K, V) = Concat(head_1, ..., head_h) W^O + head_i = Attention(Q W_i^Q, K W_i^K, V W_i^V) + +The scaling by sqrt(d_k) prevents the dot products from growing large in magnitude, which would push the softmax into regions with very small gradients. + +## Architecture + +The Transformer uses a stacked encoder-decoder structure. + +Encoder: 6 identical layers, each with two sublayers: +1. Multi-head self-attention +2. Position-wise fully connected feed-forward network + +Each sublayer uses a residual connection followed by layer normalization: + output = LayerNorm(x + Sublayer(x)) + +Decoder: 6 identical layers, each with three sublayers: +1. Masked multi-head self-attention (prevents positions from attending to subsequent positions) +2. Multi-head attention over encoder output +3. Position-wise feed-forward network + +d_model = 512 for all sublayers and embedding layers. +Feed-forward inner dimension = 2048. + +## Positional Encoding + +Since the model contains no recurrence and no convolution, positional encodings are added to the input embeddings to give the model information about the relative position of tokens: + + PE(pos, 2i) = sin(pos / 10000^(2i/d_model)) + PE(pos, 2i+1) = cos(pos / 10000^(2i/d_model)) + +This allows the model to easily learn to attend by relative positions. + +## Why attention over recurrence + +Three main advantages: +1. Total computational complexity per layer is lower for self-attention when sequence length is smaller than representation dimensionality +2. Computations that can be parallelized — recurrent layers require O(n) sequential operations +3. Path length between long-range dependencies is O(1) for self-attention vs O(n) for recurrence + +## Results + +WMT 2014 English-to-German: 28.4 BLEU, outperforming all previously published results by over 2 BLEU. +WMT 2014 English-to-French: 41.0 BLEU, new state of the art. +Training cost: 3.5 days on 8 P100 GPUs. + +## Open questions + +[1] Does the choice of h=8 heads generalize, or is it architecture-specific? +[2] The scaling factor sqrt(d_k) is justified empirically — is there a theoretical justification? +[3] How does learned positional encoding compare to sinusoidal at longer sequence lengths? + +## References + +[1] Vaswani, A., Shazeer, N., Parmar, N., et al. (2017). Attention Is All You Need. arXiv:1706.03762 +[2] Ba, J., Kiros, J., Hinton, G. (2016). Layer Normalization. arXiv:1607.06450 +[3] He, K., et al. (2016). Deep Residual Learning for Image Recognition. CVPR 2016. diff --git a/worked/mixed-corpus/raw/build.py b/worked/mixed-corpus/raw/build.py new file mode 100644 index 000000000..655820c04 --- /dev/null +++ b/worked/mixed-corpus/raw/build.py @@ -0,0 +1,39 @@ +# assemble node+edge dicts into a NetworkX graph, preserving edge direction +from __future__ import annotations +import sys +import networkx as nx +from .validate import validate_extraction + + +def build_from_json(extraction: dict) -> nx.Graph: + errors = validate_extraction(extraction) + # Dangling edges (stdlib/external imports) are expected - only warn about real schema errors. + real_errors = [e for e in errors if "does not match any node id" not in e] + if real_errors: + print(f"[graphify] Extraction warning ({len(real_errors)} issues): {real_errors[0]}", file=sys.stderr) + G = nx.Graph() + for node in extraction.get("nodes", []): + G.add_node(node["id"], **{k: v for k, v in node.items() if k != "id"}) + node_set = set(G.nodes()) + for edge in extraction.get("edges", []): + src, tgt = edge["source"], edge["target"] + if src not in node_set or tgt not in node_set: + continue # skip edges to external/stdlib nodes - expected, not an error + attrs = {k: v for k, v in edge.items() if k not in ("source", "target")} + # Preserve original edge direction - undirected graphs lose it otherwise, + # causing display functions to show edges backwards. + attrs["_src"] = src + attrs["_tgt"] = tgt + G.add_edge(src, tgt, **attrs) + return G + + +def build(extractions: list[dict]) -> nx.Graph: + """Merge multiple extraction results into one graph.""" + combined: dict = {"nodes": [], "edges": [], "input_tokens": 0, "output_tokens": 0} + for ext in extractions: + combined["nodes"].extend(ext.get("nodes", [])) + combined["edges"].extend(ext.get("edges", [])) + combined["input_tokens"] += ext.get("input_tokens", 0) + combined["output_tokens"] += ext.get("output_tokens", 0) + return build_from_json(combined) diff --git a/worked/mixed-corpus/raw/cluster.py b/worked/mixed-corpus/raw/cluster.py new file mode 100644 index 000000000..b5c97b7c8 --- /dev/null +++ b/worked/mixed-corpus/raw/cluster.py @@ -0,0 +1,104 @@ +"""Leiden community detection on NetworkX graphs. Splits oversized communities. Returns cohesion scores.""" +from __future__ import annotations +import networkx as nx + + +def build_graph(nodes: list[dict], edges: list[dict]) -> nx.Graph: + """Build a NetworkX graph from graphify node/edge dicts. + + Preserves original edge direction as _src/_tgt attributes so that + display functions can show relationships in the correct direction, + even though the graph is undirected for structural analysis. + """ + G = nx.Graph() + for n in nodes: + G.add_node(n["id"], **{k: v for k, v in n.items() if k != "id"}) + for e in edges: + attrs = {k: v for k, v in e.items() if k not in ("source", "target")} + attrs["_src"] = e["source"] + attrs["_tgt"] = e["target"] + G.add_edge(e["source"], e["target"], **attrs) + return G + +_MAX_COMMUNITY_FRACTION = 0.25 # communities larger than 25% of graph get split +_MIN_SPLIT_SIZE = 10 # only split if community has at least this many nodes + + +def cluster(G: nx.Graph) -> dict[int, list[str]]: + """Run Leiden community detection. Returns {community_id: [node_ids]}. + + Community IDs are stable across runs: 0 = largest community after splitting. + Oversized communities (> 25% of graph nodes, min 10) are split by running + a second Leiden pass on the subgraph. + """ + if G.number_of_nodes() == 0: + return {} + if G.number_of_edges() == 0: + return {i: [n] for i, n in enumerate(sorted(G.nodes))} + + from graspologic.partition import leiden # lazy - avoids 15s numba JIT on import + + # Leiden warns and drops isolates - handle them separately + isolates = [n for n in G.nodes() if G.degree(n) == 0] + connected_nodes = [n for n in G.nodes() if G.degree(n) > 0] + connected = G.subgraph(connected_nodes) + + raw: dict[int, list[str]] = {} + if connected.number_of_nodes() > 0: + partition: dict[str, int] = leiden(connected) + for node, cid in partition.items(): + raw.setdefault(cid, []).append(node) + + # Each isolate becomes its own single-node community + next_cid = max(raw.keys(), default=-1) + 1 + for node in isolates: + raw[next_cid] = [node] + next_cid += 1 + + # Split oversized communities + max_size = max(_MIN_SPLIT_SIZE, int(G.number_of_nodes() * _MAX_COMMUNITY_FRACTION)) + final_communities: list[list[str]] = [] + for nodes in raw.values(): + if len(nodes) > max_size: + final_communities.extend(_split_community(G, nodes)) + else: + final_communities.append(nodes) + + # Re-index by size descending for deterministic ordering + final_communities.sort(key=len, reverse=True) + return {i: sorted(nodes) for i, nodes in enumerate(final_communities)} + + +def _split_community(G: nx.Graph, nodes: list[str]) -> list[list[str]]: + """Run a second Leiden pass on a community subgraph to split it further.""" + subgraph = G.subgraph(nodes) + if subgraph.number_of_edges() == 0: + # No edges - split into individual nodes + return [[n] for n in sorted(nodes)] + try: + from graspologic.partition import leiden + sub_partition: dict[str, int] = leiden(subgraph) + sub_communities: dict[int, list[str]] = {} + for node, cid in sub_partition.items(): + sub_communities.setdefault(cid, []).append(node) + if len(sub_communities) <= 1: + # Leiden couldn't split it - return as-is + return [sorted(nodes)] + return [sorted(v) for v in sub_communities.values()] + except Exception: + return [sorted(nodes)] + + +def cohesion_score(G: nx.Graph, community_nodes: list[str]) -> float: + """Ratio of actual intra-community edges to maximum possible.""" + n = len(community_nodes) + if n <= 1: + return 1.0 + subgraph = G.subgraph(community_nodes) + actual = subgraph.number_of_edges() + possible = n * (n - 1) / 2 + return round(actual / possible, 2) if possible > 0 else 0.0 + + +def score_all(G: nx.Graph, communities: dict[int, list[str]]) -> dict[int, float]: + return {cid: cohesion_score(G, nodes) for cid, nodes in communities.items()}