0.50.43

This reverts commit e2b407c6f3.

changedetectionio/__init__.py

@@ -2,7 +2,7 @@
 
 # Read more https://github.com/dgtlmoon/changedetection.io/wiki
 
-__version__ = '0.50.40'
+__version__ = '0.50.43'
 
 from changedetectionio.strtobool import strtobool
 from json.decoder import JSONDecodeError
@@ -74,6 +74,12 @@ def main():
 
     datastore_path = None
     do_cleanup = False
+    # Optional URL to watch since start
+    default_url = None
+    # Set a default logger level
+    logger_level = 'DEBUG'
+    include_default_watches = True
+
     host = os.environ.get("LISTEN_HOST", "0.0.0.0").strip()
     port = int(os.environ.get('PORT', 5000))
     ssl_mode = False
@@ -87,15 +93,13 @@ def main():
         datastore_path = os.path.join(os.getcwd(), "../datastore")
 
     try:
-        opts, args = getopt.getopt(sys.argv[1:], "6Ccsd:h:p:l:", "port")
+        opts, args = getopt.getopt(sys.argv[1:], "6Ccsd:h:p:l:u:", "port")
     except getopt.GetoptError:
-        print('backend.py -s SSL enable -h [host] -p [port] -d [datastore path] -l [debug level - TRACE, DEBUG(default), INFO, SUCCESS, WARNING, ERROR, CRITICAL]')
+        print('backend.py -s SSL enable -h [host] -p [port] -d [datastore path] -u [default URL to watch] -l [debug level - TRACE, DEBUG(default), INFO, SUCCESS, WARNING, ERROR, CRITICAL]')
         sys.exit(2)
 
     create_datastore_dir = False
 
-    # Set a default logger level
-    logger_level = 'DEBUG'
     # Set a logger level via shell env variable
     # Used: Dockerfile for CICD
     # To set logger level for pytest, see the app function in tests/conftest.py
@@ -116,6 +120,10 @@ def main():
         if opt == '-d':
             datastore_path = arg
 
+        if opt == '-u':
+            default_url = arg
+            include_default_watches = False
+
         # Cleanup (remove text files that arent in the index)
         if opt == '-c':
             do_cleanup = True
@@ -172,13 +180,16 @@ def main():
             sys.exit(2)
 
     try:
-        datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=__version__)
+        datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=__version__, include_default_watches=include_default_watches)
     except JSONDecodeError as e:
         # Dont' start if the JSON DB looks corrupt
         logger.critical(f"ERROR: JSON DB or Proxy List JSON at '{app_config['datastore_path']}' appears to be corrupt, aborting.")
         logger.critical(str(e))
         return
 
+    if default_url:
+        datastore.add_watch(url = default_url)
+
     app = changedetection_app(app_config, datastore)
 
     # Get the SocketIO instance from the Flask app (created in flask_app.py)

changedetectionio/api/__init__.py

@@ -41,7 +41,7 @@ def get_openapi_spec():
         # Possibly for pip3 packages
         spec_path = os.path.join(os.path.dirname(__file__), '../docs/api-spec.yaml')
 
-    with open(spec_path, 'r') as f:
+    with open(spec_path, 'r', encoding='utf-8') as f:
         spec_dict = yaml.safe_load(f)
     _openapi_spec = OpenAPI.from_dict(spec_dict)
     return _openapi_spec

changedetectionio/content_fetchers/playwright.py

@@ -139,7 +139,7 @@ class fetcher(Fetcher):
         content = await self.page.content()
         destination = os.path.join(self.browser_steps_screenshot_path, 'step_{}.html'.format(step_n))
         logger.debug(f"Saving step HTML to {destination}")
-        with open(destination, 'w') as f:
+        with open(destination, 'w', encoding='utf-8') as f:
             f.write(content)
 
     async def run(self,

changedetectionio/flask_app.py

@@ -101,12 +101,12 @@ def init_app_secret(datastore_path):
     path = os.path.join(datastore_path, "secret.txt")
 
     try:
-        with open(path, "r") as f:
+        with open(path, "r", encoding='utf-8') as f:
             secret = f.read()
 
     except FileNotFoundError:
         import secrets
-        with open(path, "w") as f:
+        with open(path, "w", encoding='utf-8') as f:
             secret = secrets.token_hex(32)
             f.write(secret)
 

changedetectionio/model/App.py

@@ -81,7 +81,7 @@ class model(dict):
 
 def parse_headers_from_text_file(filepath):
     headers = {}
-    with open(filepath, 'r') as f:
+    with open(filepath, 'r', encoding='utf-8') as f:
         for l in f.readlines():
             l = l.strip()
             if not l.startswith('#') and ':' in l:

changedetectionio/model/Watch.py

@@ -188,7 +188,7 @@ class model(watch_base):
         fname = os.path.join(self.watch_data_dir, "history.txt")
         if os.path.isfile(fname):
             logger.debug(f"Reading watch history index for {self.get('uuid')}")
-            with open(fname, "r") as f:
+            with open(fname, "r", encoding='utf-8') as f:
                 for i in f.readlines():
                     if ',' in i:
                         k, v = i.strip().split(',', 2)
@@ -594,7 +594,7 @@ class model(watch_base):
         """Return the text saved from a previous request that resulted in a non-200 error"""
         fname = os.path.join(self.watch_data_dir, "last-error.txt")
         if os.path.isfile(fname):
-            with open(fname, 'r') as f:
+            with open(fname, 'r', encoding='utf-8') as f:
                 return f.read()
         return False
 

changedetectionio/processors/text_json_diff/processor.py

@@ -280,7 +280,7 @@ class ContentProcessor:
 
         # Sort JSON to avoid false alerts from reordering
         try:
-            content = json.dumps(json.loads(content), sort_keys=True, indent=4)
+            content = json.dumps(json.loads(content), sort_keys=True, indent=2, ensure_ascii=False)
         except Exception:
             # Might be malformed JSON, continue anyway
             pass

changedetectionio/run_basic_tests.sh

@@ -11,6 +11,56 @@ set -e
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
+# Since theres no curl installed lets roll with python3
+check_sanity() {
+  local port="$1"
+  if [ -z "$port" ]; then
+    echo "Usage: check_sanity <port>" >&2
+    return 1
+  fi
+
+  python3 - "$port" <<'PYCODE'
+import sys, time, urllib.request, socket
+
+port = sys.argv[1]
+url = f'http://localhost:{port}'
+ok = False
+
+for _ in range(6):  # --retry 6
+    try:
+        r = urllib.request.urlopen(url, timeout=3).read().decode()
+        if 'est-url-is-sanity' in r:
+            ok = True
+            break
+    except (urllib.error.URLError, ConnectionRefusedError, socket.error):
+        time.sleep(1)
+sys.exit(0 if ok else 1)
+PYCODE
+}
+
+data_sanity_test () {
+  # Restart data sanity test
+  cd ..
+  TMPDIR=$(mktemp -d)
+  PORT_N=$((5000 + RANDOM % (6501 - 5000)))
+  ./changedetection.py -p $PORT_N -d $TMPDIR -u "https://localhost?test-url-is-sanity=1" &
+  PID=$!
+  sleep 5
+  kill $PID
+  sleep 2
+  ./changedetection.py -p $PORT_N -d $TMPDIR &
+  PID=$!
+  sleep 5
+  # On a restart the URL should still be there
+  check_sanity $PORT_N || exit 1
+  kill $PID
+  cd $OLDPWD
+
+  # datastore looks alright, continue
+}
+
+data_sanity_test
+
 # REMOVE_REQUESTS_OLD_SCREENSHOTS disabled so that we can write a screenshot and send it in test_notifications.py without a real browser
 REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest -n 30 --dist load  tests/test_*.py
 
@@ -41,3 +91,6 @@ FETCH_WORKERS=130 pytest  tests/test_history_consistency.py -v -l
 # Check file:// will pickup a file when enabled
 echo "Hello world" > /tmp/test-file.txt
 ALLOW_FILE_URI=yes pytest tests/test_security.py
+
+
+

changedetectionio/store.py

@@ -78,37 +78,46 @@ class ChangeDetectionStore:
                 self.__data['build_sha'] = f.read()
 
         try:
-            # @todo retest with ", encoding='utf-8'"
-            with open(self.json_store_path) as json_file:
-                from_disk = json.load(json_file)
+            if HAS_ORJSON:
+                # orjson.loads() expects UTF-8 encoded bytes #3611
+                with open(self.json_store_path, 'rb') as json_file:
+                    from_disk = orjson.loads(json_file.read())
+            else:
+                with open(self.json_store_path, encoding='utf-8') as json_file:
+                    from_disk = json.load(json_file)
 
-                # @todo isnt there a way todo this dict.update recursively?
-                # Problem here is if the one on the disk is missing a sub-struct, it wont be present anymore.
-                if 'watching' in from_disk:
-                    self.__data['watching'].update(from_disk['watching'])
+            if not from_disk:
+                # No FileNotFound exception was thrown but somehow the JSON was empty - abort for safety.
+                logger.critical(f"JSON DB existed but was empty on load - empty JSON file? '{self.json_store_path}' Aborting")
+                raise Exception('JSON DB existed but was empty on load - Aborting')
 
-                if 'app_guid' in from_disk:
-                    self.__data['app_guid'] = from_disk['app_guid']
+            # @todo isnt there a way todo this dict.update recursively?
+            # Problem here is if the one on the disk is missing a sub-struct, it wont be present anymore.
+            if 'watching' in from_disk:
+                self.__data['watching'].update(from_disk['watching'])
 
-                if 'settings' in from_disk:
-                    if 'headers' in from_disk['settings']:
-                        self.__data['settings']['headers'].update(from_disk['settings']['headers'])
+            if 'app_guid' in from_disk:
+                self.__data['app_guid'] = from_disk['app_guid']
 
-                    if 'requests' in from_disk['settings']:
-                        self.__data['settings']['requests'].update(from_disk['settings']['requests'])
+            if 'settings' in from_disk:
+                if 'headers' in from_disk['settings']:
+                    self.__data['settings']['headers'].update(from_disk['settings']['headers'])
 
-                    if 'application' in from_disk['settings']:
-                        self.__data['settings']['application'].update(from_disk['settings']['application'])
+                if 'requests' in from_disk['settings']:
+                    self.__data['settings']['requests'].update(from_disk['settings']['requests'])
 
-                # Convert each existing watch back to the Watch.model object
-                for uuid, watch in self.__data['watching'].items():
-                    self.__data['watching'][uuid] = self.rehydrate_entity(uuid, watch)
-                    logger.info(f"Watching: {uuid} {watch['url']}")
+                if 'application' in from_disk['settings']:
+                    self.__data['settings']['application'].update(from_disk['settings']['application'])
 
-                # And for Tags also, should be Restock type because it has extra settings
-                for uuid, tag in self.__data['settings']['application']['tags'].items():
-                    self.__data['settings']['application']['tags'][uuid] = self.rehydrate_entity(uuid, tag, processor_override='restock_diff')
-                    logger.info(f"Tag: {uuid} {tag['title']}")
+            # Convert each existing watch back to the Watch.model object
+            for uuid, watch in self.__data['watching'].items():
+                self.__data['watching'][uuid] = self.rehydrate_entity(uuid, watch)
+                logger.info(f"Watching: {uuid} {watch['url']}")
+
+            # And for Tags also, should be Restock type because it has extra settings
+            for uuid, tag in self.__data['settings']['application']['tags'].items():
+                self.__data['settings']['application']['tags'][uuid] = self.rehydrate_entity(uuid, tag, processor_override='restock_diff')
+                logger.info(f"Tag: {uuid} {tag['title']}")
 
         # First time ran, Create the datastore.
         except (FileNotFoundError):
@@ -435,12 +444,13 @@ class ChangeDetectionStore:
                 # system was out of memory, out of RAM etc
                 if HAS_ORJSON:
                     # Use orjson for faster serialization
+                    # orjson.dumps() always returns UTF-8 encoded bytes #3611
                     with open(self.json_store_path+".tmp", 'wb') as json_file:
                         json_file.write(orjson.dumps(data, option=orjson.OPT_INDENT_2))
                 else:
                     # Fallback to standard json module
-                    with open(self.json_store_path+".tmp", 'w') as json_file:
-                        json.dump(data, json_file, indent=2)
+                    with open(self.json_store_path+".tmp", 'w', encoding='utf-8') as json_file:
+                        json.dump(data, json_file, indent=2, ensure_ascii=False)
                 os.replace(self.json_store_path+".tmp", self.json_store_path)
             except Exception as e:
                 logger.error(f"Error writing JSON!! (Main JSON file save was skipped) : {str(e)}")
@@ -502,8 +512,13 @@ class ChangeDetectionStore:
 
         # Load from external config file
         if path.isfile(proxy_list_file):
-            with open(os.path.join(self.datastore_path, "proxies.json")) as f:
-                proxy_list = json.load(f)
+            if HAS_ORJSON:
+                # orjson.loads() expects UTF-8 encoded bytes #3611
+                with open(os.path.join(self.datastore_path, "proxies.json"), 'rb') as f:
+                    proxy_list = orjson.loads(f.read())
+            else:
+                with open(os.path.join(self.datastore_path, "proxies.json"), encoding='utf-8') as f:
+                    proxy_list = json.load(f)
 
         # Mapping from UI config if available
         extras = self.data['settings']['requests'].get('extra_proxies')
@@ -760,7 +775,16 @@ class ChangeDetectionStore:
                 logger.critical(f"Applying update_{update_n}")
                 # Wont exist on fresh installs
                 if os.path.exists(self.json_store_path):
-                    shutil.copyfile(self.json_store_path, os.path.join(self.datastore_path, f"url-watches-before-{update_n}.json"))
+                    i = 0
+                    while True:
+                        i+=1
+                        dest = os.path.join(self.datastore_path, f"url-watches-before-{update_n}-{i}.json")
+                        if not os.path.exists(dest):
+                            logger.debug(f"Copying url-watches.json DB to '{dest}' backup.")
+                            shutil.copyfile(self.json_store_path, dest)
+                            break
+                        else:
+                            logger.warning(f"Backup of url-watches.json '{dest}', DB already exists, trying {i+1}.. ")
 
                 try:
                     update_method = getattr(self, f"update_{update_n}")()

changedetectionio/tests/test_history_consistency.py

@@ -40,7 +40,7 @@ def test_consistent_history(client, live_server, measure_memory_usage, datastore
     json_db_file = os.path.join(live_server.app.config['DATASTORE'].datastore_path, 'url-watches.json')
 
     json_obj = None
-    with open(json_db_file, 'r') as f:
+    with open(json_db_file, 'r', encoding='utf-8') as f:
         json_obj = json.load(f)
 
     # assert the right amount of watches was found in the JSON
@@ -76,7 +76,7 @@ def test_consistent_history(client, live_server, measure_memory_usage, datastore
         assert len(files_in_watch_dir) == 3, "Should be just three files in the dir, html.br snapshot, history.txt and the extracted text snapshot"
 
     json_db_file = os.path.join(live_server.app.config['DATASTORE'].datastore_path, 'url-watches.json')
-    with open(json_db_file, 'r') as f:
+    with open(json_db_file, 'r', encoding='utf-8') as f:
         assert '"default"' not in f.read(), "'default' probably shouldnt be here, it came from when the 'default' Watch vars were accidently being saved"
 
 

changedetectionio/tests/test_jsonpath_jq_selector.py

@@ -442,13 +442,12 @@ def test_correct_header_detect(client, live_server, measure_memory_usage, datast
     snapshot_contents = watch.get_history_snapshot(timestamp=dates[0])
 
     assert b'"hello": 123,' in res.data # properly html escaped in the front end
-
+    import json
+    data = json.loads(snapshot_contents)
+    keys = list(data.keys())
     # Should be correctly formatted and sorted,  ("world" goes to end)
-    assert snapshot_contents == """{
-    "hello": 123,
-    "world": 123
-}"""
-
+    assert keys == ["hello", "world"]
+        
     delete_all_watches(client)
 
 def test_check_jsonpath_ext_filter(client, live_server, measure_memory_usage, datastore_path):

changedetectionio/tests/test_request.py

@@ -142,7 +142,7 @@ def test_body_in_request(client, live_server, measure_memory_usage, datastore_pa
     client.get(url_for("ui.form_watch_checknow"), follow_redirects=True)
     wait_for_all_checks(client)
     watches_with_body = 0
-    with open(os.path.join(datastore_path, 'url-watches.json')) as f:
+    with open(os.path.join(datastore_path, 'url-watches.json'), encoding='utf-8') as f:
         app_struct = json.load(f)
         for uuid in app_struct['watching']:
             if app_struct['watching'][uuid]['body']==body_value:
@@ -225,7 +225,7 @@ def test_method_in_request(client, live_server, measure_memory_usage, datastore_
     wait_for_all_checks(client)
 
     watches_with_method = 0
-    with open(os.path.join(datastore_path, 'url-watches.json')) as f:
+    with open(os.path.join(datastore_path, 'url-watches.json'), encoding='utf-8') as f:
         app_struct = json.load(f)
         for uuid in app_struct['watching']:
             if app_struct['watching'][uuid]['method'] == 'PATCH':

requirements.txt

@@ -34,7 +34,7 @@ wtforms~=3.2
 jsonpath-ng~=1.7.0
 
 # Fast JSON serialization for better performance
-orjson~=3.10
+orjson~=3.11
 
 # dnspython - Used by paho-mqtt for MQTT broker resolution  
 # Version pin removed since eventlet (which required the specific 2.6.1 pin) has been eliminated