lee/changedetection.io
1
0
Fork 0
mirror of https://github.com/dgtlmoon/changedetection.io.git synced 2025-12-11 18:45:34 +00:00
Code Issues Actions 7 Packages Projects Releases Wiki Activity

CSS Filter - strip text of whitespacing, preserve new lines where applicable, remove extra newlines

Browse Source
This commit is contained in:
dgtlmoon
2021-06-23 12:29:14 +10:00
parent c6a4709987
commit 47f7698b32
1 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
backend/fetch_site_status.py
View File

@@ -89,7 +89,10 @@ class perform_site_check():
soup = BeautifulSoup(r.content, "html.parser")
stripped_text_from_html = ""
for item in soup.select(css_filter):
text = str(item.get_text()).strip() + '\n'
# By default, bs4's get_text will lump the text together
text = str(item.get_text(separator="\n", strip=True)).strip()
# Try to cut back on excessive linefeeds if there are any
text = text.replace("\n\n","\n")
stripped_text_from_html += text
else: