memory GC workaround

.github/ISSUE_TEMPLATE/bug_report.md

@@ -7,6 +7,20 @@ assignees: 'dgtlmoon'
 
 ---
 
+**DO NOT USE THIS FORM TO REPORT THAT A PARTICULAR WEBSITE IS NOT SCRAPING/WATCHING AS EXPECTED**
+
+This form is only for direct bugs and feature requests todo directly with the software.
+
+Please report watched websites (full URL and _any_ settings) that do not work with changedetection.io as expected [**IN THE DISCUSSION FORUMS**](https://github.com/dgtlmoon/changedetection.io/discussions) or your report will be deleted
+
+CONSIDER TAKING OUT A SUBSCRIPTION FOR A SMALL PRICE PER MONTH, YOU GET THE BENEFIT OF USING OUR PAID PROXIES AND FURTHERING THE DEVELOPMENT OF CHANGEDETECTION.IO
+
+THANK YOU
+
+
+
+
+
 **Describe the bug**
 A clear and concise description of what the bug is.
 

README.md

@@ -1,7 +1,6 @@
 ## Web Site Change Detection, Monitoring and Notification.
 
-[**Try our $6.99/month subscription - Unlimited checks and watches!**](https://lemonade.changedetection.io/start)
- 
+Live your data-life pro-actively, track website content changes and receive notifications via Discord, Email, Slack, Telegram and 70+ more
 
 [<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://lemonade.changedetection.io/start)
 
@@ -11,7 +10,7 @@
 
 Know when important content changes, we support notifications via Discord, Telegram, Home-Assistant, Slack, Email and 70+ more
 
-[**Try our $6.99/month subscription - unlimited checks and watches!**](https://lemonade.changedetection.io/start) , _half the price of other website change monitoring services and comes with unlimited watches & checks!_
+[**Don't have time? Let us host it for you! try our $6.99/month subscription - use our proxies and support!**](https://lemonade.changedetection.io/start) , _half the price of other website change monitoring services and comes with unlimited watches & checks!_
 
 
 
@@ -40,7 +39,18 @@ Know when important content changes, we support notifications via Discord, Teleg
 - Monitor HTML source code for unexpected changes, strengthen your PCI compliance
 - You have a very sensitive list of URLs to watch and you do _not_ want to use the paid alternatives. (Remember, _you_ are the product)
 
-_Need an actual Chrome runner with Javascript support? We support fetching via WebDriver!</a>_
+_Need an actual Chrome runner with Javascript support? We support fetching via WebDriver and Playwright!</a>_
+
+#### Key Features
+
+- Lots of trigger filters, such as "Trigger on text", "Remove text by selector", "Ignore text", "Extract text", also using regular-expressions!
+- Target elements with xPath and CSS Selectors, Easily monitor complex JSON with JsonPath rules
+- Switch between fast non-JS and Chrome JS based "fetchers"
+- Easily specify how often a site should be checked
+- Execute JS before extracting text (Good for logging in, see examples in the UI!)
+- Override Request Headers, Specify `POST` or `GET` and other methods
+- Use the "Visual Selector" to help target specific elements
+
 
 ## Screenshots
 

changedetectionio/__init__.py

@@ -503,7 +503,7 @@ def changedetection_app(config=None, datastore_o=None):
         from changedetectionio import fetch_site_status
 
         # Get the most recent one
-        newest_history_key = datastore.data['watching'][uuid].get('newest_history_key')
+        newest_history_key = datastore.get_val(uuid, 'newest_history_key')
 
         # 0 means that theres only one, so that there should be no 'unviewed' history available
         if newest_history_key == 0:

changedetectionio/content_fetcher.py

@@ -407,7 +407,8 @@ class base_html_playwright(Fetcher):
             else:
                 page.evaluate("var css_filter=''")
 
-            self.xpath_data = page.evaluate("async () => {" + self.xpath_element_js + "}")
+            # str() here must create a dereferenced copy, which allows the GC to release correctly
+            self.xpath_data = str(page.evaluate("async () => {" + self.xpath_element_js + "}"))
 
             # Bug 3 in Playwright screenshot handling
             # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it

changedetectionio/fetch_site_status.py

@@ -63,11 +63,13 @@ class perform_site_check():
 
 
     def run(self, uuid):
+        timestamp = int(time.time())  # used for storage etc too
+
         changed_detected = False
         screenshot = False  # as bytes
         stripped_text_from_html = ""
 
-        watch = self.datastore.data['watching'].get(uuid)
+        watch = self.datastore.data['watching'][uuid]
 
         # Protect against file:// access
         if re.search(r'^file', watch['url'], re.IGNORECASE) and not os.getenv('ALLOW_FILE_URI', False):
@@ -78,7 +80,7 @@ class perform_site_check():
         # Unset any existing notification error
         update_obj = {'last_notification_error': False, 'last_error': False}
 
-        extra_headers =self.datastore.data['watching'][uuid].get('headers')
+        extra_headers = self.datastore.get_val(uuid, 'headers')
 
         # Tweak the base config with the per-watch ones
         request_headers = self.datastore.data['settings']['headers'].copy()
@@ -91,9 +93,9 @@ class perform_site_check():
             request_headers['Accept-Encoding'] = request_headers['Accept-Encoding'].replace(', br', '')
 
         timeout = self.datastore.data['settings']['requests']['timeout']
-        url = watch.get('url')
-        request_body = self.datastore.data['watching'][uuid].get('body')
-        request_method = self.datastore.data['watching'][uuid].get('method')
+        url = self.datastore.get_val(uuid, 'url')
+        request_body = self.datastore.get_val(uuid, 'body')
+        request_method = self.datastore.get_val(uuid, 'method')
         ignore_status_codes = self.datastore.data['watching'][uuid].get('ignore_status_codes', False)
 
         # source: support

changedetectionio/store.py

@@ -244,6 +244,10 @@ class ChangeDetectionStore:
 
         return False
 
+    def get_val(self, uuid, val):
+        # Probably their should be dict...
+        return self.data['watching'][uuid].get(val)
+
     # Remove a watchs data but keep the entry (URL etc)
     def clear_watch_history(self, uuid):
         import pathlib

changedetectionio/templates/preview.html

@@ -57,6 +57,7 @@
          </br>
          {% if is_html_webdriver %}
            {% if screenshot %}
+             <div class="snapshot-age">{{watch.snapshot_screenshot_ctime|format_timestamp_timeago}}</div>
              <img style="max-width: 80%" id="screenshot-img" alt="Current screenshot from most recent request"/>
            {% else %}
               No screenshot available just yet! Try rechecking the page.