fix(extract): keep AST progress denominator consistent to the end (#1693)

Intermediate progress lines count against len(uncached_work) ("X/Y uncached
files"), but the final line switched to total_files ("Y/Y files"), which includes
cached hits and files with no extractor that never entered uncached_work. On a
large corpus with unsupported-language files, the total jumped upward right after
99% with no explanation. Both the parallel and sequential final lines now report
the same uncached_work denominator, so the count no longer appears to change
mid-run.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
safishamsi
2026-07-07 00:12:36 +01:00
co-authored by Claude Opus 4.8
parent 377dc7f384
commit f5d50adbd0
2 changed files with 27 additions and 2 deletions
+9 -2
View File
@@ -16254,8 +16254,13 @@ def _extract_parallel(
)
return False
if total_files >= _PROGRESS_INTERVAL:
# Report the same denominator the intermediate lines used (uncached files
# actually processed this run), not total_files — switching to the full
# corpus made the count jump upward at the end (cached hits + files with no
# extractor never entered uncached_work), which read as inconsistent (#1693).
_done = len(uncached_work)
print(
f" AST extraction: {total_files}/{total_files} files (100%) [{max_workers} workers]",
f" AST extraction: {_done}/{_done} uncached files (100%) [{max_workers} workers]",
flush=True,
)
return True
@@ -16290,7 +16295,9 @@ def _extract_sequential(
save_cached(path, result, effective_root)
per_file[idx] = result
if total_files >= _PROGRESS_INTERVAL:
print(f" AST extraction: {total_files}/{total_files} files (100%)", flush=True)
# Consistent denominator with the intermediate lines (#1693).
_done = len(uncached_work)
print(f" AST extraction: {_done}/{_done} uncached files (100%)", flush=True)
_PARALLEL_THRESHOLD = 20
+18
View File
@@ -1795,3 +1795,21 @@ def test_extract_no_warning_when_all_code_has_extractors(tmp_path, capsys):
extract([py], cache_root=tmp_path)
err = capsys.readouterr().err
assert "no AST extractor" not in err
def test_extract_progress_final_line_uses_consistent_denominator(tmp_path, capsys):
# #1693: intermediate progress lines count against uncached_work; the final
# "100%" line must NOT switch to total_files (which includes cached hits and
# files with no extractor), or the count appears to jump upward at the end.
for i in range(100):
(tmp_path / f"m{i}.py").write_text(f"def f{i}():\n return {i}\n")
for i in range(5):
(tmp_path / f"s{i}.r").write_text(f"g{i} <- function(x) x\n") # no extractor
paths = sorted(tmp_path.glob("*.py")) + sorted(tmp_path.glob("*.r")) # total 105
extract(paths, cache_root=tmp_path, parallel=False)
out = capsys.readouterr().out
# final progress line reports the uncached count (100), not the total (105)
assert "100/100 uncached files (100%)" in out
assert "105/105 files" not in out, "final line must not switch to total_files (#1693)"