feature: support table reader functionality

Signed-off-by: Timothée Mazzucotelli <dev@pawamoy.fr>

python/tests/unit/extensions/test_macros.py

@@ -25,13 +25,18 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
+import pandas
 import pytest
 from jinja2.exceptions import TemplateSyntaxError, UndefinedError
 
 from zensical.extensions.context import ContextPreprocessor
 from zensical.extensions.macros import (
     MacroEnv,
+    _add_indentation,
+    _convert_to_md_table,
     _fix_url,
+    _get_fake_table_readers,
+    _get_table_readers,
     _load_module,
     _load_one_yaml,
     _merge_include_yaml,
@@ -42,6 +47,7 @@ if TYPE_CHECKING:
     from pathlib import Path
 
     from markdown import Markdown
+    from pandas import DataFrame
 
 
 # ---------------------------------------------------------------------------
@@ -443,3 +449,271 @@ class TestPreprocessor:
         result = md.convert("{{ code_snippet('python', 'x = 1 + 1') }}")
         assert "<code>python" not in result
         assert "x = 1 + 1" not in result  # we expect spans
+
+
+# ---------------------------------------------------------------------------
+# Table helpers
+# ---------------------------------------------------------------------------
+
+
+# Unit tests for the table conversion helper functions.
+class TestTableHelpers:
+    def test_add_indentation_spaces(self) -> None:
+        result = _add_indentation("line1\nline2", spaces=4)
+        assert result == "    line1\n    line2"
+
+    def test_add_indentation_tabs(self) -> None:
+        result = _add_indentation("line1\nline2", tabs=2)
+        assert result == "\t\tline1\n\t\tline2"
+
+    def test_add_indentation_none_returns_unchanged(self) -> None:
+        assert _add_indentation("hello") == "hello"
+
+    def test_add_indentation_raises_when_both_specified(self) -> None:
+        with pytest.raises(ValueError, match="spaces or tabs"):
+            _add_indentation("x", spaces=2, tabs=1)
+
+    def test_convert_to_md_table_basic(self) -> None:
+        df: DataFrame = pandas.DataFrame(
+            {"Name": ["Alice", "Bob"], "Age": [30, 25]}
+        )
+        result = _convert_to_md_table(df)
+        assert "|" in result
+        assert "Name" in result
+        assert "Age" in result
+        assert "Alice" in result
+        assert "Bob" in result
+
+    def test_convert_to_md_table_escapes_pipes_in_cells(self) -> None:
+        df: DataFrame = pandas.DataFrame({"Col": ["a|b", "c"]})
+        result = _convert_to_md_table(df)
+        assert r"a\|b" in result
+
+    def test_convert_to_md_table_escapes_pipes_in_column_names(self) -> None:
+        df: DataFrame = pandas.DataFrame({"Na|me": ["Alice"]})
+        result = _convert_to_md_table(df)
+        assert r"Na\|me" in result
+
+    def test_convert_to_md_table_omits_index_by_default(self) -> None:
+        # Custom index values must not leak into the output.
+        # Verifies that the `index=False` default is applied.
+        df: DataFrame = pandas.DataFrame({"X": [1, 2]}, index=[100, 200])
+        result = _convert_to_md_table(df)
+        assert "100" not in result
+        assert "200" not in result
+
+
+# ---------------------------------------------------------------------------
+# Table readers
+# ---------------------------------------------------------------------------
+
+
+class TestTableReaders:
+    def test_fake_readers_raise_runtime_error_when_pandas_missing(
+        self,
+    ) -> None:
+        readers = _get_fake_table_readers()
+        for reader in readers.values():
+            with pytest.raises(RuntimeError, match="table reading requires"):
+                reader("irrelevant.csv")
+
+    # CSV
+    def test_read_csv(self, tmp_path: Path) -> None:
+        (tmp_path / "data.csv").write_text(
+            "Name,Age\nAlice,30\nBob,25\n", encoding="utf-8"
+        )
+        readers = _get_table_readers(tmp_path)
+        result = readers["read_csv"]("data.csv")
+        assert "Name" in result
+        assert "Alice" in result
+        assert "Bob" in result
+
+    def test_pd_read_csv_returns_dataframe(self, tmp_path: Path) -> None:
+        (tmp_path / "data.csv").write_text("X,Y\n1,2\n3,4\n", encoding="utf-8")
+        readers = _get_table_readers(tmp_path)
+        df: DataFrame = readers["pd_read_csv"]("data.csv")
+        assert list(df.columns) == ["X", "Y"]
+        assert len(df) == 2
+
+    # JSON
+    def test_read_json(self, tmp_path: Path) -> None:
+        (tmp_path / "data.json").write_text(
+            '[{"Name": "Alice", "Age": 30}, {"Name": "Bob", "Age": 25}]',
+            encoding="utf-8",
+        )
+        readers = _get_table_readers(tmp_path)
+        result = readers["read_json"]("data.json")
+        assert "Name" in result
+        assert "Alice" in result
+        assert "Bob" in result
+
+    def test_pd_read_json_returns_dataframe(self, tmp_path: Path) -> None:
+        (tmp_path / "data.json").write_text(
+            '[{"X": 1, "Y": 2}, {"X": 3, "Y": 4}]', encoding="utf-8"
+        )
+        readers = _get_table_readers(tmp_path)
+        df: DataFrame = readers["pd_read_json"]("data.json")
+        assert list(df.columns) == ["X", "Y"]
+        assert len(df) == 2
+
+    # YAML
+    def test_read_yaml(self, tmp_path: Path) -> None:
+        (tmp_path / "data.yaml").write_text(
+            "- Name: Alice\n  Age: 30\n- Name: Bob\n  Age: 25\n",
+            encoding="utf-8",
+        )
+        readers = _get_table_readers(tmp_path)
+        result = readers["read_yaml"]("data.yaml")
+        assert "Name" in result
+        assert "Alice" in result
+        assert "Bob" in result
+
+    def test_pd_read_yaml_returns_dataframe(self, tmp_path: Path) -> None:
+        (tmp_path / "data.yaml").write_text(
+            "- X: 1\n  Y: 2\n- X: 3\n  Y: 4\n", encoding="utf-8"
+        )
+        readers = _get_table_readers(tmp_path)
+        df: DataFrame = readers["pd_read_yaml"]("data.yaml")
+        assert list(df.columns) == ["X", "Y"]
+        assert len(df) == 2
+
+    # Table (tab-separated)
+    def test_read_table(self, tmp_path: Path) -> None:
+        (tmp_path / "data.tsv").write_text(
+            "Name\tAge\nAlice\t30\nBob\t25\n", encoding="utf-8"
+        )
+        readers = _get_table_readers(tmp_path)
+        result = readers["read_table"]("data.tsv")
+        assert "Name" in result
+        assert "Alice" in result
+        assert "Bob" in result
+
+    def test_pd_read_table_returns_dataframe(self, tmp_path: Path) -> None:
+        (tmp_path / "data.tsv").write_text(
+            "X\tY\n1\t2\n3\t4\n", encoding="utf-8"
+        )
+        readers = _get_table_readers(tmp_path)
+        df: DataFrame = readers["pd_read_table"]("data.tsv")
+        assert list(df.columns) == ["X", "Y"]
+        assert len(df) == 2
+
+    # FWF (fixed-width format)
+    def test_read_fwf(self, tmp_path: Path) -> None:
+        content = "Name     Age\nAlice    30\nBob      25\n"
+        (tmp_path / "data.fwf").write_text(content, encoding="utf-8")
+        readers = _get_table_readers(tmp_path)
+        result = readers["read_fwf"]("data.fwf")
+        assert "Name" in result
+        assert "Alice" in result
+        assert "Bob" in result
+
+    def test_pd_read_fwf_returns_dataframe(self, tmp_path: Path) -> None:
+        (tmp_path / "data.fwf").write_text(
+            "X    Y\n1    2\n3    4\n", encoding="utf-8"
+        )
+        readers = _get_table_readers(tmp_path)
+        df: DataFrame = readers["pd_read_fwf"]("data.fwf")
+        assert list(df.columns) == ["X", "Y"]
+        assert len(df) == 2
+
+    # Excel (.xlsx)
+    def test_read_excel(self, tmp_path: Path) -> None:
+        pytest.importorskip("openpyxl")
+        df: DataFrame = pandas.DataFrame(
+            {"Name": ["Alice", "Bob"], "Age": [30, 25]}
+        )
+        df.to_excel(tmp_path / "data.xlsx", index=False)
+        readers = _get_table_readers(tmp_path)
+        result = readers["read_excel"]("data.xlsx")
+        assert "Name" in result
+        assert "Alice" in result
+        assert "Bob" in result
+
+    def test_pd_read_excel_returns_dataframe(self, tmp_path: Path) -> None:
+        pytest.importorskip("openpyxl")
+        df_in: DataFrame = pandas.DataFrame({"X": [1, 3], "Y": [2, 4]})
+        df_in.to_excel(tmp_path / "data.xlsx", index=False)
+        readers = _get_table_readers(tmp_path)
+        df: DataFrame = readers["pd_read_excel"]("data.xlsx")
+        assert list(df.columns) == ["X", "Y"]
+        assert len(df) == 2
+
+    # Feather
+    def test_read_feather(self, tmp_path: Path) -> None:
+        pytest.importorskip("pyarrow")
+        df: DataFrame = pandas.DataFrame(
+            {"Name": ["Alice", "Bob"], "Age": [30, 25]}
+        )
+        df.to_feather(tmp_path / "data.feather")
+        readers = _get_table_readers(tmp_path)
+        result = readers["read_feather"]("data.feather")
+        assert "Name" in result
+        assert "Alice" in result
+        assert "Bob" in result
+
+    def test_pd_read_feather_returns_dataframe(self, tmp_path: Path) -> None:
+        pytest.importorskip("pyarrow")
+        df_in: DataFrame = pandas.DataFrame({"X": [1, 3], "Y": [2, 4]})
+        df_in.to_feather(tmp_path / "data.feather")
+        readers = _get_table_readers(tmp_path)
+        df: DataFrame = readers["pd_read_feather"]("data.feather")
+        assert list(df.columns) == ["X", "Y"]
+        assert len(df) == 2
+
+    # End-to-end: CSV rendered through the Jinja2 / Markdown pipeline
+    @pytest.mark.parametrize(
+        "md",
+        [
+            pytest.param(
+                {
+                    "config": {
+                        "markdown_extensions": {
+                            "zensical.extensions.macros": {
+                                "render_by_default": True,
+                            },
+                        },
+                    }
+                },
+                id="end_to_end_csv",
+            ),
+        ],
+        indirect=["md"],
+    )
+    def test_end_to_end_csv_via_template(
+        self, md: Markdown, tmp_path: Path
+    ) -> None:
+        (tmp_path / "scores.csv").write_text(
+            "Player,Score\nAlice,100\nBob,80\n", encoding="utf-8"
+        )
+        result = md.convert("{{ read_csv('scores.csv') }}")
+        assert "Player" in result
+        assert "Alice" in result
+        assert "Score" in result
+
+    @pytest.mark.parametrize(
+        "md",
+        [
+            pytest.param(
+                {
+                    "config": {
+                        "markdown_extensions": {
+                            "zensical.extensions.macros": {
+                                "render_by_default": True,
+                            },
+                        },
+                    }
+                },
+                id="end_to_end_pd_filter",
+            ),
+        ],
+        indirect=["md"],
+    )
+    def test_end_to_end_pd_read_csv_with_convert_filter(
+        self, md: Markdown, tmp_path: Path
+    ) -> None:
+        (tmp_path / "nums.csv").write_text("X,Y\n1,2\n3,4\n", encoding="utf-8")
+        result = md.convert(
+            "{{ pd_read_csv('nums.csv') | convert_to_md_table }}"
+        )
+        assert "X" in result
+        assert "Y" in result

python/zensical/extensions/macros.py

@@ -24,16 +24,18 @@
 from __future__ import annotations
 
 import importlib.util
-import inspect
 import platform
+import re
 import subprocess
 import traceback
 from collections.abc import Callable, Iterable
 from dataclasses import asdict, dataclass, field
 from datetime import datetime
-from functools import cache
+from functools import cache, wraps
+from inspect import signature
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Literal, TypeAlias
+from textwrap import indent
+from typing import TYPE_CHECKING, Any, Literal, NoReturn, TypeAlias
 from urllib.parse import urlparse
 
 import jinja2
@@ -47,6 +49,7 @@ from zensical.extensions.context import ContextPreprocessor
 if TYPE_CHECKING:
     from jinja2 import Environment
     from markdown import Markdown
+    from pandas import DataFrame
 
 
 # -----------------------------------------------------------------------------
@@ -230,6 +233,8 @@ class MacrosPreprocessor(Preprocessor):
         filters: dict[str, Callable] = {
             "pretty": _pretty,
             "fix_url": _fix_url,
+            "add_indentation": _add_indentation,
+            "convert_to_md_table": _convert_to_md_table,
         }
 
         # Merge extra into variables
@@ -324,7 +329,17 @@ class MacrosPreprocessor(Preprocessor):
         }
         if page:
             env_globals["page"] = page
+
+        try:
+            import pandas  # noqa: F401,PLC0415
+            import tabulate  # noqa: F401,PLC0415
+        except ImportError:
+            env_globals.update(_get_fake_table_readers())
+        else:
+            env_globals.update(_get_table_readers(project_root))
+
         env.globals.update(env_globals)
+
         # This copies the environment filters and globals
         # into a new environment so this call must be last
         env.globals["macros_info"] = _macros_info_closure(env)  # ty:ignore[invalid-assignment]
@@ -386,11 +401,17 @@ def makeExtension(**kwargs: Any) -> MacrosExtension:
     return MacrosExtension(**kwargs)
 
 
+# -----------------------------------------------------------------------------
+
+
 def _now() -> datetime:
     """Return current datetime (`datetime.now()`)."""
     return datetime.now()  # noqa: DTZ005
 
 
+# -----------------------------------------------------------------------------
+
+
 def _macros_info_closure(env: Environment) -> Callable[[], str]:
     new_env = jinja2.Environment()  # noqa: S701
     new_env.filters.update(env.filters)
@@ -403,6 +424,9 @@ def _macros_info_closure(env: Environment) -> Callable[[], str]:
     return macros_info
 
 
+# -----------------------------------------------------------------------------
+
+
 def _fix_url(url: str) -> str:
     parsed = urlparse(url)
     if (not parsed.scheme) and parsed.path:
@@ -410,6 +434,9 @@ def _fix_url(url: str) -> str:
     return url
 
 
+# -----------------------------------------------------------------------------
+
+
 def _list_items(obj: Any) -> Iterable[tuple[str | int, Any]]:
     try:
         return sorted(obj.items())
@@ -426,7 +453,7 @@ def _format_value(value: Any) -> str:
         else:
             return ""
         try:
-            param_names = ", ".join(inspect.signature(value).parameters)
+            param_names = ", ".join(signature(value).parameters)
         except ValueError:
             return doc
         else:
@@ -464,6 +491,9 @@ def _context_closure(
     return context
 
 
+# -----------------------------------------------------------------------------
+
+
 def _make_table(
     rows: list[tuple[str, str, str]],
     header: tuple[str, str, str],
@@ -496,6 +526,147 @@ def _pretty(var_list: list[Any]) -> str:
         return f"#{type(error).__name__}: {error}\n{traceback.format_exc()}"
 
 
+# -----------------------------------------------------------------------------
+
+
+def _pandas_read_yaml(
+    func: Callable[..., DataFrame],
+) -> Callable[..., DataFrame]:
+    @wraps(func)
+    def inner(filepath: str | Path, **kwargs: Any) -> DataFrame:
+        with open(filepath, encoding="utf8") as file:
+            return func(yaml.safe_load(file), **kwargs)
+
+    return inner
+
+
+def _relative_pandas_reader(
+    root: Path, func: Callable[..., DataFrame]
+) -> Callable[..., DataFrame]:
+    @wraps(func)
+    def inner(filepath: str | Path, **kwargs: Any) -> DataFrame:
+        if not isinstance(filepath, (str, Path)):
+            raise TypeError(
+                f"Only str and Path are supported in pd_{func.__name__}"  # ty:ignore[unresolved-attribute]
+            )
+        filepath = Path(filepath)
+        if not filepath.is_absolute():
+            filepath = root.joinpath(filepath)
+        filepath = filepath.resolve()
+        return func(filepath, **kwargs)
+
+    return inner
+
+
+def _add_indentation(text: str, *, spaces: int = 0, tabs: int = 0) -> str:
+    """Indent text using spaces or tabs."""
+    if spaces and tabs:
+        raise ValueError(
+            "You can only specify either spaces or tabs, not both."
+        )
+    if spaces:
+        prefix = " " * spaces
+    elif tabs:
+        prefix = "\t" * tabs
+    else:
+        return text
+
+    return "\n".join(indent(line, prefix) for line in text.split("\n"))
+
+
+def _convert_to_md_table(df: DataFrame, **kwargs: Any) -> str:
+    """Convert a pandas dataframe to a Markdown table."""
+
+    def escape_pipes(text: str) -> str:
+        return re.sub(r"(?<!\\)\|", "\\|", text)
+
+    df.columns = [
+        escape_pipes(c) if isinstance(c, str) else c for c in df.columns
+    ]
+    df = df.map(lambda s: escape_pipes(s) if isinstance(s, str) else s)
+    kwargs.setdefault("index", False)
+    kwargs.setdefault("tablefmt", "pipe")
+    return df.to_markdown(**kwargs)
+
+
+def _param_names(func: Callable) -> list[str]:
+    return [
+        param.name
+        for param in signature(func).parameters.values()
+        if param.kind not in (param.VAR_POSITIONAL, param.VAR_KEYWORD)
+    ]
+
+
+def _filter_kwargs(
+    kwargs: dict[str, Any], param_names: Iterable[str]
+) -> tuple[dict[str, Any], dict[str, Any]]:
+    into, not_into = {}, {}
+    for k, v in kwargs.items():
+        if k in param_names:
+            into[k] = v
+        else:
+            not_into[k] = v
+    return into, not_into
+
+
+def _relative_reader(rpr: Callable[..., DataFrame]) -> Callable[..., str]:
+    reader_params = _param_names(rpr)
+
+    def inner(filepath: str | Path, **kwargs: Any) -> str:
+        read_args, write_args = _filter_kwargs(kwargs, reader_params)
+        df = rpr(filepath, **read_args)
+        return _convert_to_md_table(df, **write_args)
+
+    inner.__doc__ = (
+        "Read data using pandas and convert it to a Markdown table. "
+        "Keyword arguments are split and passed to the relevant pandas reader "
+        "as well as dataframes' `to_markdown()` method."
+    )
+    return inner
+
+
+@cache
+def _get_table_readers(project_root: Path) -> dict[str, Callable]:
+    import pandas  # noqa: PLC0415
+
+    pandas_readers = {
+        "csv": pandas.read_csv,
+        "fwf": pandas.read_fwf,
+        "yaml": _pandas_read_yaml(pandas.json_normalize),
+        "json": pandas.read_json,
+        "table": pandas.read_table,
+        "excel": pandas.read_excel,
+        "feather": pandas.read_feather,
+    }
+
+    readers = {}
+    for fmt, reader in pandas_readers.items():
+        rpr = _relative_pandas_reader(project_root, reader)
+        readers[f"pd_read_{fmt}"] = rpr
+        readers[f"read_{fmt}"] = _relative_reader(rpr)
+    return readers
+
+
+@cache
+def _get_fake_table_readers() -> dict[str, Callable]:
+    message = "table reading requires pandas and tabulate packages"
+
+    def raiser() -> Callable[..., NoReturn]:
+        def inner(*args: Any, **kwargs: Any) -> NoReturn:  # noqa: ARG001
+            raise RuntimeError(message)
+
+        return inner
+
+    readers = {}
+    for fmt in ("csv", "fwf", "yaml", "json", "table", "excel", "feather"):
+        readers[f"pd_read_{fmt}"] = raiser()
+        readers[f"read_{fmt}"] = raiser()
+    return readers
+
+
+# -----------------------------------------------------------------------------
+
+
 def _load_module(
     module_name: str, project_root: Path | None = None
 ) -> VariablesMacrosFiltersType: