From f704972b3ebceffe2efa65eaeb19e82df9c72765 Mon Sep 17 00:00:00 2001 From: Chris Stephens Date: Wed, 6 May 2026 11:37:11 -0400 Subject: [PATCH] Add optional Google Workspace shortcut export --- CHANGELOG.md | 1 + README.md | 16 +++ docs/how-it-works.md | 6 + graphify/__main__.py | 14 ++- graphify/detect.py | 33 ++++- graphify/google_workspace.py | 214 +++++++++++++++++++++++++++++++++ pyproject.toml | 1 + tests/test_detect.py | 34 ++++++ tests/test_google_workspace.py | 81 +++++++++++++ 9 files changed, 395 insertions(+), 5 deletions(-) create mode 100644 graphify/google_workspace.py create mode 100644 tests/test_google_workspace.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 51196b11..371c8071 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ Full release notes with details on each version: [GitHub Releases](https://githu ## 0.7.8 (2026-05-06) +- Feat: optional Google Workspace shortcut export for headless extraction -- `graphify extract ./docs --google-workspace` converts `.gdoc`, `.gsheet`, and `.gslides` files into Markdown sidecars with the `gws` CLI before semantic extraction; `[google]` extra adds Sheets table rendering support - Feat: Gemini and OpenAI backends -- `graphify extract ./docs --backend gemini` (GEMINI_API_KEY / GOOGLE_API_KEY) or `--backend openai` (OPENAI_API_KEY); `[gemini]` and `[openai]` extras added (#735) - Feat: Groovy and Spock support -- `.groovy` and `.gradle` extracted via tree-sitter-groovy; Spock spec files (`def "feature"()` syntax) handled via regex fallback (#732) - Feat: Luau support -- `.luau` (Roblox Luau) added to code extraction using the Lua tree-sitter parser (#745) diff --git a/README.md b/README.md index 4ca52acc..1c7dd0d8 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,7 @@ Uninstall with the matching command (e.g. `graphify claude uninstall`). | Code (28 languages) | `.py .ts .js .jsx .tsx .go .rs .java .c .cpp .rb .cs .kt .scala .php .swift .lua .luau .zig .ps1 .ex .exs .m .jl .vue .svelte .groovy .gradle .sql .f .F .f90 .F90 .f95 .F95 .f03 .F03 .f08 .F08` | | Docs | `.md .mdx .html .txt .rst .yaml .yml` | | Office | `.docx .xlsx` (requires `pip install graphifyy[office]`) | +| Google Workspace | `.gdoc .gsheet .gslides` (opt-in; requires `gws` auth and `--google-workspace`; Sheets need `pip install graphifyy[google]`) | | PDFs | `.pdf` | | Images | `.png .jpg .webp .gif` | | Video / Audio | `.mp4 .mov .mp3 .wav` and more (requires `pip install graphifyy[video]`) | @@ -134,6 +135,20 @@ Uninstall with the matching command (e.g. `graphify claude uninstall`). Code is extracted locally with no API calls (AST via tree-sitter). Everything else goes through your AI assistant's model API. +Google Drive for desktop `.gdoc`, `.gsheet`, and `.gslides` files are shortcut +pointers, not document content. To include native Google Docs, Sheets, and Slides +in a headless extraction, install and authenticate the +[`gws` CLI](https://github.com/googleworkspace/cli), then run: + +```bash +pip install "graphifyy[google]" # needed for Google Sheets table rendering +gws auth login -s drive +graphify extract ./docs --google-workspace +``` + +You can also set `GRAPHIFY_GOOGLE_WORKSPACE=1`. Graphify exports shortcuts into +`graphify-out/converted/` as Markdown sidecars, then extracts those files. + --- ## Common commands @@ -277,6 +292,7 @@ graphify extract ./docs # headless LLM extraction for CI graphify extract ./docs --backend gemini # explicit backend: gemini, kimi, claude, openai, or ollama graphify extract ./docs --backend gemini --model gemini-3.1-pro-preview graphify extract ./docs --backend ollama # local Ollama (set OLLAMA_BASE_URL / OLLAMA_MODEL) +graphify extract ./docs --google-workspace # export .gdoc/.gsheet/.gslides via gws before extraction graphify extract ./docs --no-cluster # raw extraction only, skip clustering graphify extract ./docs --dedup-llm # LLM tiebreaker for ambiguous entity pairs (uses same API key) graphify extract ./docs --global --as myrepo # extract and register into the cross-project global graph diff --git a/docs/how-it-works.md b/docs/how-it-works.md index c79ab5cf..83fbedbb 100644 --- a/docs/how-it-works.md +++ b/docs/how-it-works.md @@ -13,6 +13,12 @@ Video and audio files are transcribed with faster-whisper. To focus the transcri **Pass 3 — Docs, papers, images (Claude subagents, costs tokens)** Claude runs in parallel over markdown, PDFs, images, and transcripts. Each subagent reads a batch of files and outputs a JSON fragment: nodes, edges, and any group relationships. The fragments are merged into a single graph. +Before Pass 3, optional converters turn supported pointer/binary formats into +Markdown sidecars under `graphify-out/converted/`. Office files (`.docx`, +`.xlsx`) use the `[office]` extra. Google Workspace shortcuts (`.gdoc`, +`.gsheet`, `.gslides`) are opt-in with `--google-workspace` or +`GRAPHIFY_GOOGLE_WORKSPACE=1` and require an authenticated `gws` CLI. + --- ## How community detection works diff --git a/graphify/__main__.py b/graphify/__main__.py index 4eae7315..7910eae4 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -1099,6 +1099,7 @@ def main() -> None: print(" --backend B gemini|kimi|claude|openai|ollama (default: whichever API key is set)") print(" --model M override backend default model") print(" --out DIR output dir (default: ); writes /graphify-out/") + print(" --google-workspace export .gdoc/.gsheet/.gslides shortcuts via gws before extraction") print(" --no-cluster skip clustering, write raw extraction only") print(" --global also merge the resulting graph into the global graph") print(" --as repo tag for --global (default: target directory name)") @@ -2006,7 +2007,7 @@ def main() -> None: if len(sys.argv) < 3: print( "Usage: graphify extract [--backend gemini|kimi|claude|openai] " - "[--out DIR] [--no-cluster]", + "[--out DIR] [--google-workspace] [--no-cluster]", file=sys.stderr, ) sys.exit(1) @@ -2021,6 +2022,7 @@ def main() -> None: out_dir: Path | None = None no_cluster = False dedup_llm = False + google_workspace = False global_merge = False global_repo_tag: str | None = None args = sys.argv[3:] @@ -2043,6 +2045,8 @@ def main() -> None: no_cluster = True; i += 1 elif a == "--dedup-llm": dedup_llm = True; i += 1 + elif a == "--google-workspace": + google_workspace = True; i += 1 elif a == "--global": global_merge = True; i += 1 elif a == "--as" and i + 1 < len(args): @@ -2104,10 +2108,14 @@ def main() -> None: if incremental_mode: print(f"[graphify extract] incremental scan of {target}") - detection = _detect_incremental(target, manifest_path=str(manifest_path)) + detection = _detect_incremental( + target, + manifest_path=str(manifest_path), + google_workspace=google_workspace or None, + ) else: print(f"[graphify extract] scanning {target}") - detection = _detect(target) + detection = _detect(target, google_workspace=google_workspace or None) files_by_type = detection.get("files", {}) if incremental_mode: diff --git a/graphify/detect.py b/graphify/detect.py index 519a838a..5d2dc9c6 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -7,6 +7,12 @@ import re from enum import Enum from pathlib import Path +from graphify.google_workspace import ( + GOOGLE_WORKSPACE_EXTENSIONS, + convert_google_workspace_file, + google_workspace_enabled, +) + class FileType(str, Enum): CODE = "code" @@ -130,6 +136,8 @@ def classify_file(path: Path) -> FileType | None: return FileType.DOCUMENT if ext in OFFICE_EXTENSIONS: return FileType.DOCUMENT + if ext in GOOGLE_WORKSPACE_EXTENSIONS: + return FileType.DOCUMENT if ext in VIDEO_EXTENSIONS: return FileType.VIDEO return None @@ -618,8 +626,9 @@ def _could_contain_included_path(path: Path, root: Path, patterns: list[tuple[Pa return False -def detect(root: Path, *, follow_symlinks: bool = False) -> dict: +def detect(root: Path, *, follow_symlinks: bool = False, google_workspace: bool | None = None) -> dict: root = root.resolve() + google_workspace = google_workspace_enabled() if google_workspace is None else google_workspace files: dict[FileType, list[str]] = { FileType.CODE: [], FileType.DOCUMENT: [], @@ -694,6 +703,25 @@ def detect(root: Path, *, follow_symlinks: bool = False) -> dict: continue ftype = classify_file(p) if ftype: + if p.suffix.lower() in GOOGLE_WORKSPACE_EXTENSIONS: + if not google_workspace: + skipped_sensitive.append( + str(p) + + " [Google Workspace shortcut skipped - pass --google-workspace " + "or set GRAPHIFY_GOOGLE_WORKSPACE=1]" + ) + continue + try: + md_path = convert_google_workspace_file(p, converted_dir, xlsx_to_markdown=xlsx_to_markdown) + except Exception as exc: + skipped_sensitive.append(str(p) + f" [Google Workspace export failed: {exc}]") + continue + if md_path: + files[ftype].append(str(md_path)) + total_words += count_words(md_path) + else: + skipped_sensitive.append(str(p) + " [Google Workspace export produced no readable text]") + continue # Office files: convert to markdown sidecar so subagents can read them if p.suffix.lower() in OFFICE_EXTENSIONS: md_path = convert_office_file(p, converted_dir) @@ -776,6 +804,7 @@ def detect_incremental( manifest_path: str = _MANIFEST_PATH, *, follow_symlinks: bool = False, + google_workspace: bool | None = None, ) -> dict: """Like detect(), but returns only new or modified files since the last run. @@ -790,7 +819,7 @@ def detect_incremental( directory outside the scan root) are scanned consistently between full and incremental runs. """ - full = detect(root, follow_symlinks=follow_symlinks) + full = detect(root, follow_symlinks=follow_symlinks, google_workspace=google_workspace) manifest = load_manifest(manifest_path) if not manifest: diff --git a/graphify/google_workspace.py b/graphify/google_workspace.py new file mode 100644 index 00000000..86dd0f9d --- /dev/null +++ b/graphify/google_workspace.py @@ -0,0 +1,214 @@ +"""Optional Google Workspace shortcut export support. + +Google Drive for desktop stores native Docs, Sheets, and Slides as small JSON +shortcut files (.gdoc, .gsheet, .gslides). Those files are pointers, not the +document content. This module exports them to Markdown sidecars via the +googleworkspace CLI (`gws`) so Graphify can extract their actual contents. +""" +from __future__ import annotations + +import hashlib +import json +import os +import re +import shutil +import subprocess +import tempfile +import urllib.parse +from pathlib import Path +from typing import Callable, Any + + +GOOGLE_WORKSPACE_EXTENSIONS = {".gdoc", ".gsheet", ".gslides"} + + +def google_workspace_enabled(value: str | None = None) -> bool: + """Return True when Google Workspace shortcut export is enabled.""" + raw = value if value is not None else os.environ.get("GRAPHIFY_GOOGLE_WORKSPACE", "") + return raw.strip().lower() in {"1", "true", "yes", "on"} + + +def _safe_yaml_str(value: str) -> str: + return value.replace("\\", "\\\\").replace('"', '\\"').replace("\n", " ").replace("\r", " ") + + +def _extract_file_id_from_url(url: str) -> str | None: + """Extract a Drive file ID from common Google Docs/Drive URL shapes.""" + if not url: + return None + parsed = urllib.parse.urlparse(url) + query = urllib.parse.parse_qs(parsed.query) + if query.get("id"): + return query["id"][0] + match = re.search(r"/(?:document|spreadsheets|presentation|file)/d/([^/?#]+)", parsed.path) + if match: + return match.group(1) + return None + + +def _extract_resource_key(url: str, data: dict[str, Any]) -> str | None: + for key in ("resource_key", "resourceKey"): + value = data.get(key) + if value: + return str(value) + if not url: + return None + parsed = urllib.parse.urlparse(url) + query = urllib.parse.parse_qs(parsed.query) + if query.get("resourcekey"): + return query["resourcekey"][0] + return None + + +def read_google_shortcut(path: Path) -> dict[str, str | None]: + """Read a .gdoc/.gsheet/.gslides shortcut and return export metadata.""" + try: + data = json.loads(path.read_text(encoding="utf-8")) + except Exception as exc: + raise RuntimeError(f"could not read Google Workspace shortcut {path}: {exc}") from exc + + url = str(data.get("url") or "") + file_id = ( + data.get("doc_id") + or data.get("file_id") + or data.get("fileId") + or data.get("id") + or _extract_file_id_from_url(url) + ) + if not file_id: + resource_id = str(data.get("resource_id") or "") + if ":" in resource_id: + file_id = resource_id.split(":", 1)[1] + + if not file_id: + raise RuntimeError(f"Google Workspace shortcut {path} does not include a Drive file ID") + + return { + "file_id": str(file_id), + "url": url or None, + "resource_key": _extract_resource_key(url, data), + "account": str(data.get("email")) if data.get("email") else None, + } + + +def _run_gws_export(file_id: str, mime_type: str, output: Path, resource_key: str | None = None) -> None: + exe = shutil.which("gws") + if not exe: + raise RuntimeError( + "gws is required for Google Workspace export. Install it from " + "https://github.com/googleworkspace/cli and run `gws auth login -s drive`." + ) + + params: dict[str, str] = {"fileId": file_id, "mimeType": mime_type} + if resource_key: + params["resourceKey"] = resource_key + timeout = int(os.environ.get("GRAPHIFY_GOOGLE_WORKSPACE_TIMEOUT", "120")) + result = subprocess.run( + [exe, "drive", "files", "export", "--params", json.dumps(params), "-o", str(output)], + capture_output=True, + text=True, + timeout=timeout, + ) + if result.returncode != 0: + stderr = (result.stderr or result.stdout or "").strip() + if len(stderr) > 1200: + stderr = stderr[:1200] + "..." + raise RuntimeError(f"gws export failed for {file_id}: {stderr}") + + +def _sidecar_path(path: Path, out_dir: Path) -> Path: + name_hash = hashlib.sha256(str(path.resolve()).encode()).hexdigest()[:8] + return out_dir / f"{path.stem}_{name_hash}.md" + + +def _with_frontmatter(path: Path, shortcut: dict[str, str | None], body: str, exported_mime_type: str) -> str: + source_url = shortcut.get("url") or "" + account = shortcut.get("account") or "" + return ( + "---\n" + f'source_file: "{_safe_yaml_str(str(path))}"\n' + 'source_type: "google_workspace"\n' + f'google_file_id: "{_safe_yaml_str(shortcut["file_id"] or "")}"\n' + f'google_export_mime_type: "{_safe_yaml_str(exported_mime_type)}"\n' + f'source_url: "{_safe_yaml_str(source_url)}"\n' + f'google_account: "{_safe_yaml_str(account)}"\n' + "---\n\n" + f"\n\n" + f"{body.strip()}\n" + ) + + +def convert_google_workspace_file( + path: Path, + out_dir: Path, + *, + xlsx_to_markdown: Callable[[Path], str] | None = None, +) -> Path | None: + """Export a Google Workspace shortcut to a Markdown sidecar. + + Returns the converted Markdown path, or None when conversion is unsupported + or produced no readable content. + """ + ext = path.suffix.lower() + if ext not in GOOGLE_WORKSPACE_EXTENSIONS: + return None + + shortcut = read_google_shortcut(path) + out_dir.mkdir(parents=True, exist_ok=True) + out_path = _sidecar_path(path, out_dir) + + if ext == ".gdoc": + with tempfile.NamedTemporaryFile("w+b", suffix=".md", delete=False, dir=out_dir) as tmp: + tmp_path = Path(tmp.name) + try: + _run_gws_export(shortcut["file_id"] or "", "text/markdown", tmp_path, shortcut.get("resource_key")) + body = tmp_path.read_text(encoding="utf-8", errors="replace") + finally: + tmp_path.unlink(missing_ok=True) + if not body.strip(): + return None + out_path.write_text(_with_frontmatter(path, shortcut, body, "text/markdown"), encoding="utf-8") + return out_path + + if ext == ".gslides": + with tempfile.NamedTemporaryFile("w+b", suffix=".txt", delete=False, dir=out_dir) as tmp: + tmp_path = Path(tmp.name) + try: + _run_gws_export(shortcut["file_id"] or "", "text/plain", tmp_path, shortcut.get("resource_key")) + body = tmp_path.read_text(encoding="utf-8", errors="replace") + finally: + tmp_path.unlink(missing_ok=True) + if not body.strip(): + return None + out_path.write_text(_with_frontmatter(path, shortcut, body, "text/plain"), encoding="utf-8") + return out_path + + if ext == ".gsheet": + if xlsx_to_markdown is None: + raise RuntimeError("Google Sheets export requires the office extra: pip install graphifyy[office,google]") + with tempfile.NamedTemporaryFile("w+b", suffix=".xlsx", delete=False, dir=out_dir) as tmp: + tmp_path = Path(tmp.name) + try: + _run_gws_export( + shortcut["file_id"] or "", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + tmp_path, + shortcut.get("resource_key"), + ) + body = xlsx_to_markdown(tmp_path) + finally: + tmp_path.unlink(missing_ok=True) + if not body.strip(): + return None + out_path.write_text( + _with_frontmatter( + path, + shortcut, + body, + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ), + encoding="utf-8", + ) + return out_path + + return None diff --git a/pyproject.toml b/pyproject.toml index 464737ae..1a940252 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,7 @@ watch = ["watchdog"] svg = ["matplotlib"] leiden = ["graspologic; python_version < '3.13'"] office = ["python-docx", "openpyxl"] +google = ["openpyxl"] video = ["faster-whisper", "yt-dlp"] kimi = ["openai", "tiktoken"] ollama = ["openai"] diff --git a/tests/test_detect.py b/tests/test_detect.py index 000de351..c1754a72 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -258,6 +258,40 @@ def test_classify_video_extensions(): assert classify_file(Path("audio.m4a")) == FileType.VIDEO +def test_classify_google_workspace_shortcuts(): + assert classify_file(Path("notes.gdoc")) == FileType.DOCUMENT + assert classify_file(Path("budget.gsheet")) == FileType.DOCUMENT + assert classify_file(Path("deck.gslides")) == FileType.DOCUMENT + + +def test_detect_skips_google_workspace_shortcuts_by_default(tmp_path): + (tmp_path / "notes.gdoc").write_text('{"doc_id":"doc-1"}', encoding="utf-8") + + result = detect(tmp_path) + + assert not result["files"]["document"] + assert any("Google Workspace shortcut skipped" in item for item in result["skipped_sensitive"]) + + +def test_detect_converts_google_workspace_shortcuts_when_enabled(tmp_path, monkeypatch): + shortcut = tmp_path / "notes.gdoc" + shortcut.write_text('{"doc_id":"doc-1"}', encoding="utf-8") + + def fake_convert(path, out_dir, *, xlsx_to_markdown=None): + out_dir.mkdir(parents=True, exist_ok=True) + out = out_dir / "notes_converted.md" + out.write_text("# Notes\n\nA converted Google Doc.", encoding="utf-8") + return out + + monkeypatch.setattr("graphify.detect.convert_google_workspace_file", fake_convert) + + result = detect(tmp_path, google_workspace=True) + + assert len(result["files"]["document"]) == 1 + assert result["files"]["document"][0].endswith("notes_converted.md") + assert result["total_words"] > 0 + + def test_detect_includes_video_key(tmp_path): """detect() result always includes a 'video' key even with no video files.""" (tmp_path / "main.py").write_text("x = 1") diff --git a/tests/test_google_workspace.py b/tests/test_google_workspace.py new file mode 100644 index 00000000..0f0c1b42 --- /dev/null +++ b/tests/test_google_workspace.py @@ -0,0 +1,81 @@ +from pathlib import Path + +import graphify.google_workspace as gw + + +def test_read_google_shortcut_doc_id(tmp_path): + shortcut = tmp_path / "Planning.gdoc" + shortcut.write_text( + '{"url":"https://docs.google.com/document/d/doc-123/edit","doc_id":"doc-123","email":"me@example.com"}', + encoding="utf-8", + ) + + metadata = gw.read_google_shortcut(shortcut) + + assert metadata["file_id"] == "doc-123" + assert metadata["account"] == "me@example.com" + + +def test_read_google_shortcut_extracts_id_from_url(tmp_path): + shortcut = tmp_path / "Budget.gsheet" + shortcut.write_text( + '{"url":"https://docs.google.com/spreadsheets/d/sheet-456/edit?resourcekey=key-1"}', + encoding="utf-8", + ) + + metadata = gw.read_google_shortcut(shortcut) + + assert metadata["file_id"] == "sheet-456" + assert metadata["resource_key"] == "key-1" + + +def test_convert_gdoc_to_markdown_sidecar(tmp_path, monkeypatch): + shortcut = tmp_path / "Planning.gdoc" + shortcut.write_text( + '{"url":"https://docs.google.com/document/d/doc-123/edit","doc_id":"doc-123"}', + encoding="utf-8", + ) + + def fake_export(file_id, mime_type, output, resource_key=None): + assert file_id == "doc-123" + assert mime_type == "text/markdown" + output.write_text("# Planning\n\nExported doc text.", encoding="utf-8") + + monkeypatch.setattr(gw, "_run_gws_export", fake_export) + + out = gw.convert_google_workspace_file(shortcut, tmp_path / "converted") + + assert out is not None + assert out.suffix == ".md" + content = out.read_text(encoding="utf-8") + assert 'source_type: "google_workspace"' in content + assert "# Planning" in content + + +def test_convert_gsheet_uses_xlsx_markdown_callback(tmp_path, monkeypatch): + shortcut = tmp_path / "Budget.gsheet" + shortcut.write_text('{"doc_id":"sheet-456"}', encoding="utf-8") + + def fake_export(file_id, mime_type, output, resource_key=None): + assert file_id == "sheet-456" + assert mime_type == "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + output.write_bytes(b"xlsx") + + monkeypatch.setattr(gw, "_run_gws_export", fake_export) + + out = gw.convert_google_workspace_file( + shortcut, + tmp_path / "converted", + xlsx_to_markdown=lambda path: "## Sheet: Main\n\n| A |\n| --- |\n| 1 |", + ) + + assert out is not None + assert "## Sheet: Main" in out.read_text(encoding="utf-8") + + +def test_google_workspace_enabled_env(monkeypatch): + monkeypatch.setenv("GRAPHIFY_GOOGLE_WORKSPACE", "yes") + assert gw.google_workspace_enabled() + + monkeypatch.setenv("GRAPHIFY_GOOGLE_WORKSPACE", "0") + assert not gw.google_workspace_enabled()