Re #462 - check encoding

.github/workflows/containers.yml

@@ -2,20 +2,16 @@ name: Build and push containers
 
 on:
   # Automatically triggered by a testing workflow passing, but this is only checked when it lands in the `master`/default branch
-#  workflow_run:
-#    workflows: ["ChangeDetection.io Test"]
-#    branches: [master]
-#    tags: ['0.*']
-#    types: [completed]
+  workflow_run:
+    workflows: ["ChangeDetection.io Test"]
+    branches: [master]
+    tags: ['0.*']
+    types: [completed]
 
   # Or a new tagged release
   release:
     types: [published, edited]
 
-  push:
-    branches:
-      - master
-
 jobs:
   metadata:
     runs-on: ubuntu-latest
@@ -95,7 +91,8 @@ jobs:
           file: ./Dockerfile
           push: true
           tags: |
-            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest,ghcr.io/${{ github.repository }}:latest
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
+            ghcr.io/${{ github.repository }}:latest
           platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache
@@ -110,7 +107,8 @@ jobs:
           file: ./Dockerfile
           push: true
           tags: |
-            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:${{ github.event.release.tag_name }},ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
+            ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:${{ github.event.release.tag_name }}
+            ghcr.io/dgtlmoon/changedetection.io:${{ github.event.release.tag_name }}
           platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache

.gitignore

@@ -7,6 +7,4 @@ __pycache__
 .pytest_cache
 build
 dist
-venv
-*.egg-info*
 .vscode/settings.json

MANIFEST.in

@@ -2,5 +2,5 @@ recursive-include changedetectionio/templates *
 recursive-include changedetectionio/static *
 include changedetection.py
 global-exclude *.pyc
-global-exclude node_modules
+global-exclude *node_modules*
 global-exclude venv

README.md

@@ -163,9 +163,9 @@ See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Proxy-configura
 
 Raspberry Pi and linux/arm/v6 linux/arm/v7 arm64 devices are supported! See the wiki for [details](https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver)
 
-## Windows support?
+## Windows native support?
 
-YES! See the wiki https://github.com/dgtlmoon/changedetection.io/wiki/Microsoft-Windows
+Sorry not yet :( https://github.com/dgtlmoon/changedetection.io/labels/windows
 
 ## Support us
 

changedetection.py

@@ -1,11 +1,110 @@
 #!/usr/bin/python3
 
-# Entry-point for running from the CLI when not installed via Pip, Pip will handle the console_scripts entry_points's from setup.py
-# It's recommended to use `pip3 install changedetection.io` and start with `changedetection.py` instead, it will be linkd to your global path.
-# or Docker.
-# Read more https://github.com/dgtlmoon/changedetection.io/wiki
+# Launch as a eventlet.wsgi server instance.
+
+import getopt
+import os
+import sys
+
+import eventlet
+import eventlet.wsgi
+import changedetectionio
+
+from changedetectionio import store
+
+def main():
+    ssl_mode = False
+    host = ''
+    port = os.environ.get('PORT') or 5000
+    do_cleanup = False
+
+    # Must be absolute so that send_from_directory doesnt try to make it relative to backend/
+    datastore_path = os.path.join(os.getcwd(), "datastore")
+
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "Ccsd:h:p:", "port")
+    except getopt.GetoptError:
+        print('backend.py -s SSL enable -h [host] -p [port] -d [datastore path]')
+        sys.exit(2)
+
+    create_datastore_dir = False
+
+    for opt, arg in opts:
+        #        if opt == '--purge':
+        # Remove history, the actual files you need to delete manually.
+        #            for uuid, watch in datastore.data['watching'].items():
+        #                watch.update({'history': {}, 'last_checked': 0, 'last_changed': 0, 'previous_md5': None})
+
+        if opt == '-s':
+            ssl_mode = True
+
+        if opt == '-h':
+            host = arg
+
+        if opt == '-p':
+            port = int(arg)
+
+        if opt == '-d':
+            datastore_path = arg
+
+        # Cleanup (remove text files that arent in the index)
+        if opt == '-c':
+            do_cleanup = True
+
+        # Create the datadir if it doesnt exist
+        if opt == '-C':
+            create_datastore_dir = True
+
+    # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore
+    app_config = {'datastore_path': datastore_path}
+
+    if not os.path.isdir(app_config['datastore_path']):
+        if create_datastore_dir:
+            os.mkdir(app_config['datastore_path'])
+        else:
+            print ("ERROR: Directory path for the datastore '{}' does not exist, cannot start, please make sure the directory exists.\n"
+                   "Alternatively, use the -C parameter.".format(app_config['datastore_path']),file=sys.stderr)
+            sys.exit(2)
+
+    datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=changedetectionio.__version__)
+    app = changedetectionio.changedetection_app(app_config, datastore)
+
+    # Go into cleanup mode
+    if do_cleanup:
+        datastore.remove_unused_snapshots()
+
+    app.config['datastore_path'] = datastore_path
+
+
+    @app.context_processor
+    def inject_version():
+        return dict(right_sticky="v{}".format(datastore.data['version_tag']),
+                    new_version_available=app.config['NEW_VERSION_AVAILABLE'],
+                    has_password=datastore.data['settings']['application']['password'] != False
+                    )
+
+    # Proxy sub-directory support
+    # Set environment var USE_X_SETTINGS=1 on this script
+    # And then in your proxy_pass settings
+    #
+    #         proxy_set_header Host "localhost";
+    #         proxy_set_header X-Forwarded-Prefix /app;
+
+    if os.getenv('USE_X_SETTINGS'):
+        print ("USE_X_SETTINGS is ENABLED\n")
+        from werkzeug.middleware.proxy_fix import ProxyFix
+        app.wsgi_app = ProxyFix(app.wsgi_app, x_prefix=1, x_host=1)
+
+    if ssl_mode:
+        # @todo finalise SSL config, but this should get you in the right direction if you need it.
+        eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen((host, port)),
+                                               certfile='cert.pem',
+                                               keyfile='privkey.pem',
+                                               server_side=True), app)
+
+    else:
+        eventlet.wsgi.server(eventlet.listen((host, int(port))), app)
 
-from changedetectionio import changedetection
 
 if __name__ == '__main__':
-    changedetection.main()
+    main()

changedetectionio/.gitignore

@@ -1 +0,0 @@
-test-datastore

changedetectionio/__init__.py

@@ -35,11 +35,10 @@ from flask import (
     url_for,
 )
 from flask_login import login_required
-from flask_wtf import CSRFProtect
 
 from changedetectionio import html_tools
 
-__version__ = '0.39.11'
+__version__ = '0.39.10'
 
 datastore = None
 
@@ -53,10 +52,11 @@ update_q = queue.Queue()
 
 notification_q = queue.Queue()
 
+# Needs to be set this way because we also build and publish via pip
+base_path = os.path.dirname(os.path.realpath(__file__))
 app = Flask(__name__,
-            static_url_path="",
-            static_folder="static",
-            template_folder="templates")
+            static_url_path="{}/static".format(base_path),
+            template_folder="{}/templates".format(base_path))
 
 # Stop browser caching of assets
 app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0
@@ -72,9 +72,6 @@ app.config['LOGIN_DISABLED'] = False
 # Disables caching of the templates
 app.config['TEMPLATES_AUTO_RELOAD'] = True
 
-csrf = CSRFProtect()
-csrf.init_app(app)
-
 notification_debug_log=[]
 
 def init_app_secret(datastore_path):
@@ -507,13 +504,13 @@ def changedetection_app(config=None, datastore_o=None):
                           'headers': form.headers.data,
                           'body': form.body.data,
                           'method': form.method.data,
-                          'ignore_status_codes': form.ignore_status_codes.data,
                           'fetch_backend': form.fetch_backend.data,
                           'trigger_text': form.trigger_text.data,
                           'notification_title': form.notification_title.data,
                           'notification_body': form.notification_body.data,
                           'notification_format': form.notification_format.data,
-                          'extract_title_as_title': form.extract_title_as_title.data,
+                          'extract_title_as_title': form.extract_title_as_title.data
+
                           }
 
             # Notification URLs
@@ -613,15 +610,16 @@ def changedetection_app(config=None, datastore_o=None):
             form.notification_format.data = datastore.data['settings']['application']['notification_format']
             form.base_url.data = datastore.data['settings']['application']['base_url']
 
-        if request.method == 'POST' and form.data.get('removepassword_button') == True:
-            # Password unset is a GET, but we can lock the session to a salted env password to always need the password
-            if not os.getenv("SALTED_PASS", False):
+            # Password unset is a GET, but we can lock the session to always need the password
+            if not os.getenv("SALTED_PASS", False) and request.values.get('removepassword') == 'yes':
+                from pathlib import Path
                 datastore.data['settings']['application']['password'] = False
                 flash("Password protection removed.", 'notice')
                 flask_login.logout_user()
                 return redirect(url_for('settings_page'))
 
         if request.method == 'POST' and form.validate():
+
             datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data
             datastore.data['settings']['requests']['minutes_between_check'] = form.minutes_between_check.data
             datastore.data['settings']['application']['extract_title_as_title'] = form.extract_title_as_title.data

changedetectionio/changedetection.py

@@ -1,114 +0,0 @@
-#!/usr/bin/python3
-
-# Launch as a eventlet.wsgi server instance.
-
-import getopt
-import os
-import sys
-
-import eventlet
-import eventlet.wsgi
-from . import store, changedetection_app
-from . import __version__
-
-def main():
-    ssl_mode = False
-    host = ''
-    port = os.environ.get('PORT') or 5000
-    do_cleanup = False
-    datastore_path = None
-
-    # On Windows, create and use a default path.
-    if os.name == 'nt':
-        datastore_path = os.path.expandvars(r'%APPDATA%\changedetection.io')
-        os.makedirs(datastore_path, exist_ok=True)
-    else:
-        # Must be absolute so that send_from_directory doesnt try to make it relative to backend/
-        datastore_path = os.path.join(os.getcwd(), "../datastore")
-
-    try:
-        opts, args = getopt.getopt(sys.argv[1:], "Ccsd:h:p:", "port")
-    except getopt.GetoptError:
-        print('backend.py -s SSL enable -h [host] -p [port] -d [datastore path]')
-        sys.exit(2)
-
-    create_datastore_dir = False
-
-    for opt, arg in opts:
-        #        if opt == '--purge':
-        # Remove history, the actual files you need to delete manually.
-        #            for uuid, watch in datastore.data['watching'].items():
-        #                watch.update({'history': {}, 'last_checked': 0, 'last_changed': 0, 'previous_md5': None})
-
-        if opt == '-s':
-            ssl_mode = True
-
-        if opt == '-h':
-            host = arg
-
-        if opt == '-p':
-            port = int(arg)
-
-        if opt == '-d':
-            datastore_path = arg
-
-        # Cleanup (remove text files that arent in the index)
-        if opt == '-c':
-            do_cleanup = True
-
-        # Create the datadir if it doesnt exist
-        if opt == '-C':
-            create_datastore_dir = True
-
-    # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore
-    app_config = {'datastore_path': datastore_path}
-
-    if not os.path.isdir(app_config['datastore_path']):
-        if create_datastore_dir:
-            os.mkdir(app_config['datastore_path'])
-        else:
-            print(
-                "ERROR: Directory path for the datastore '{}' does not exist, cannot start, please make sure the directory exists or specify a directory with the -d option.\n"
-                "Or use the -C parameter to create the directory.".format(app_config['datastore_path']), file=sys.stderr)
-            sys.exit(2)
-
-    datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=__version__)
-    app = changedetection_app(app_config, datastore)
-
-    # Go into cleanup mode
-    if do_cleanup:
-        datastore.remove_unused_snapshots()
-
-    app.config['datastore_path'] = datastore_path
-
-
-    @app.context_processor
-    def inject_version():
-        return dict(right_sticky="v{}".format(datastore.data['version_tag']),
-                    new_version_available=app.config['NEW_VERSION_AVAILABLE'],
-                    has_password=datastore.data['settings']['application']['password'] != False
-                    )
-
-    # Proxy sub-directory support
-    # Set environment var USE_X_SETTINGS=1 on this script
-    # And then in your proxy_pass settings
-    #
-    #         proxy_set_header Host "localhost";
-    #         proxy_set_header X-Forwarded-Prefix /app;
-
-    if os.getenv('USE_X_SETTINGS'):
-        print ("USE_X_SETTINGS is ENABLED\n")
-        from werkzeug.middleware.proxy_fix import ProxyFix
-        app.wsgi_app = ProxyFix(app.wsgi_app, x_prefix=1, x_host=1)
-
-    if ssl_mode:
-        # @todo finalise SSL config, but this should get you in the right direction if you need it.
-        eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen((host, port)),
-                                               certfile='cert.pem',
-                                               keyfile='privkey.pem',
-                                               server_side=True), app)
-
-    else:
-        eventlet.wsgi.server(eventlet.listen((host, int(port))), app)
-
-

changedetectionio/content_fetcher.py

@@ -1,12 +1,10 @@
-from abc import ABC, abstractmethod
-import chardet
 import os
+import time
+from abc import ABC, abstractmethod
 from selenium import webdriver
 from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
 from selenium.webdriver.common.proxy import Proxy as SeleniumProxy
 from selenium.common.exceptions import WebDriverException
-import requests
-import time
 import urllib3.exceptions
 
 
@@ -22,7 +20,7 @@ class EmptyReply(Exception):
 class Fetcher():
     error = None
     status_code = None
-    content = None
+    content = None # Should always be bytes.
     headers = None
 
     fetcher_description ="No description"
@@ -32,13 +30,7 @@ class Fetcher():
         return self.error
 
     @abstractmethod
-    def run(self,
-            url,
-            timeout,
-            request_headers,
-            request_body,
-            request_method,
-            ignore_status_codes=False):
+    def run(self, url, timeout, request_headers, request_body, request_method):
         # Should set self.error, self.status_code and self.content
         pass
 
@@ -105,13 +97,7 @@ class html_webdriver(Fetcher):
         if proxy_args:
             self.proxy = SeleniumProxy(raw=proxy_args)
 
-    def run(self,
-            url,
-            timeout,
-            request_headers,
-            request_body,
-            request_method,
-            ignore_status_codes=False):
+    def run(self, url, timeout, request_headers, request_body, request_method):
 
         # request_body, request_method unused for now, until some magic in the future happens.
 
@@ -159,13 +145,8 @@ class html_webdriver(Fetcher):
 class html_requests(Fetcher):
     fetcher_description = "Basic fast Plaintext/HTTP Client"
 
-    def run(self,
-            url,
-            timeout,
-            request_headers,
-            request_body,
-            request_method,
-            ignore_status_codes=False):
+    def run(self, url, timeout, request_headers, request_body, request_method):
+        import requests
 
         r = requests.request(method=request_method,
                          data=request_body,
@@ -174,21 +155,16 @@ class html_requests(Fetcher):
                          timeout=timeout,
                          verify=False)
 
-        # If the response did not tell us what encoding format to expect, Then use chardet to override what `requests` thinks.
-        # For example - some sites don't tell us it's utf-8, but return utf-8 content
-        # This seems to not occur when using webdriver/selenium, it seems to detect the text encoding more reliably.
-        # https://github.com/psf/requests/issues/1604 good info about requests encoding detection
-        if not r.headers.get('content-type') or not 'charset=' in r.headers.get('content-type'):
-            encoding = chardet.detect(r.content)['encoding']
-            if encoding:
-                r.encoding = encoding
+        # https://stackoverflow.com/questions/44203397/python-requests-get-returns-improperly-decoded-text-instead-of-utf-8
+        # Return bytes here
+        html = r.text
 
         # @todo test this
         # @todo maybe you really want to test zero-byte return pages?
-        if (not ignore_status_codes and not r) or not r.content or not len(r.content):
+        if not r or not html or not len(html):
             raise EmptyReply(url=url, status_code=r.status_code)
 
         self.status_code = r.status_code
-        self.content = r.text
+        self.content = html
         self.headers = r.headers
 

changedetectionio/fetch_site_status.py

@@ -1,10 +1,10 @@
 import hashlib
-import os
 import re
 import time
-import urllib3
 
+import urllib3
 from inscriptis import get_text
+
 from changedetectionio import content_fetcher, html_tools
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
@@ -24,14 +24,8 @@ class perform_site_check():
         stripped_text_from_html = ""
 
         watch = self.datastore.data['watching'][uuid]
-
-        # Protect against file:// access
-        if re.search(r'^file', watch['url'], re.IGNORECASE) and not os.getenv('ALLOW_FILE_URI', False):
-            raise Exception(
-                "file:// type access is denied for security reasons."
-            )
-
         # Unset any existing notification error
+
         update_obj = {'last_notification_error': False, 'last_error': False}
 
         extra_headers = self.datastore.get_val(uuid, 'headers')
@@ -53,7 +47,6 @@ class perform_site_check():
             url = self.datastore.get_val(uuid, 'url')
             request_body = self.datastore.get_val(uuid, 'body')
             request_method = self.datastore.get_val(uuid, 'method')
-            ignore_status_code = self.datastore.get_val(uuid, 'ignore_status_codes')
 
             # Pluggable content fetcher
             prefer_backend = watch['fetch_backend']
@@ -65,7 +58,7 @@ class perform_site_check():
 
 
             fetcher = klass()
-            fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_code)
+            fetcher.run(url, timeout, request_headers, request_body, request_method)
             # Fetching complete, now filters
             # @todo move to class / maybe inside of fetcher abstract base?
 

changedetectionio/forms.py

@@ -325,7 +325,6 @@ class watchForm(commonSettingsForm):
     headers = StringDictKeyValue('Request Headers')
     body = TextAreaField('Request Body', [validators.Optional()])
     method = SelectField('Request Method', choices=valid_method, default=default_method)
-    ignore_status_codes = BooleanField('Ignore Status Codes (process non-2xx status codes as normal)', default=False)
     trigger_text = StringListField('Trigger/wait for text', [validators.Optional(), ValidateListRegex()])
 
     save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
@@ -351,8 +350,6 @@ class globalSettingsForm(commonSettingsForm):
                                                [validators.NumberRange(min=1)])
     extract_title_as_title = BooleanField('Extract <title> from document and use as watch title')
     base_url = StringField('Base URL', validators=[validators.Optional()])
-    global_subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
+    global_subtractive_selectors = StringListField('Ignore elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)])
     global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
     ignore_whitespace = BooleanField('Ignore whitespace')
-    save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"})
-    removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"})

changedetectionio/static/styles/styles.css

@@ -37,9 +37,6 @@ section.content {
   align-items: center;
   justify-content: center; }
 
-code {
-  background: #eee; }
-
 /* table related */
 .watch-table {
   width: 100%;

changedetectionio/static/styles/styles.scss

@@ -42,10 +42,6 @@ section.content {
   justify-content: center;
 }
 
-code {
-  background: #eee;
-}
-
 /* table related */
 .watch-table {
   width: 100%;

changedetectionio/store.py

@@ -400,7 +400,7 @@ class ChangeDetectionStore:
                 # system was out of memory, out of RAM etc
                 with open(self.json_store_path+".tmp", 'w') as json_file:
                     json.dump(data, json_file, indent=4)
-                os.replace(self.json_store_path+".tmp", self.json_store_path)
+                os.rename(self.json_store_path+".tmp", self.json_store_path)
             except Exception as e:
                 logging.error("Error writing JSON!! (Main JSON file save was skipped) : %s", str(e))
 

changedetectionio/templates/edit.html

@@ -19,7 +19,6 @@
     <div class="box-wrap inner">
         <form class="pure-form pure-form-stacked"
               action="{{ url_for('edit_page', uuid=uuid, next = request.args.get('next') ) }}" method="POST">
-             <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
 
             <div class="tab-pane-inner" id="general">
                 <fieldset>
@@ -81,9 +80,6 @@ User-Agent: wonderbra 1.0") }}
    \"car\":null
 }") }}
                     </div>
-                    <div>
-                        {{ render_field(form.ignore_status_codes) }}
-                    </div>
                 </fieldset>
                 <br/>
             </div>
@@ -117,9 +113,9 @@ User-Agent: wonderbra 1.0") }}
                         <span class="pure-form-message-inline">
                     <ul>
                         <li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
-                        <li>JSON - Limit text to this JSON rule, using <a href="https://pypi.org/project/jsonpath-ng/">JSONPath</a>, prefix with <code>"json:"</code>, use <code>json:$</code> to force re-formatting if required,  <a
+                        <li>JSON - Limit text to this JSON rule, using <a href="https://pypi.org/project/jsonpath-ng/">JSONPath</a>, prefix with <b>"json:"</b>, <a
                                 href="https://jsonpath.com/" target="new">test your JSONPath here</a></li>
-                        <li>XPath - Limit text to this XPath rule, simply start with a forward-slash, example  <code>//*[contains(@class, 'sametext')]</code>, <a
+                        <li>XPath - Limit text to this XPath rule, simply start with a forward-slash, example  <b>//*[contains(@class, 'sametext')]</b>, <a
                                 href="http://xpather.com/" target="new">test your XPath here</a></li>
                     </ul>
                     Please be sure that you thoroughly understand how to write CSS or JSONPath, XPath selector rules before filing an issue on GitHub! <a
@@ -146,7 +142,7 @@ nav
                     <span class="pure-form-message-inline">
                         <ul>
                             <li>Each line processed separately, any line matching will be ignored (removed before creating the checksum)</li>
-                            <li>Regular Expression support, wrap the line in forward slash <code>/regex/</code></li>
+                            <li>Regular Expression support, wrap the line in forward slash <b>/regex/</b></li>
                             <li>Changing this will affect the comparison checksum which may trigger an alert</li>
                             <li>Use the preview/show current tab to see ignores</li>
                         </ul>
@@ -163,7 +159,7 @@ nav
                         <li>Text to wait for before triggering a change/notification, all text and regex are tested <i>case-insensitive</i>.</li>
                         <li>Trigger text is processed from the result-text that comes out of any CSS/JSON Filters for this watch</li>
                         <li>Each line is process separately (think of each line as "OR")</li>
-                        <li>Note: Wrap in forward slash / to use regex  example: <code>/foo\d/</code></li>
+                        <li>Note: Wrap in forward slash / to use regex  example: <span style="font-family: monospace; background: #eee">/foo\d/</span></li>
                     </ul>
                         </span>
                     </div>

changedetectionio/templates/import.html

@@ -4,7 +4,6 @@
 <div class="edit-form">
      <div class="inner">
         <form class="pure-form pure-form-aligned" action="{{url_for('import_page')}}" method="POST">
-            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
             <fieldset class="pure-group">
               <legend>
                 Enter one URL per line, and optionally add tags for each URL after a space, delineated by comma (,):

changedetectionio/templates/login.html

@@ -4,7 +4,6 @@
 <div class="login-form">
  <div class="inner">
     <form class="pure-form pure-form-stacked" action="{{url_for('login')}}" method="POST">
-        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
         <fieldset>
             <div class="pure-control-group">
                 <label for="password">Password</label>

changedetectionio/templates/scrub.html

@@ -4,7 +4,6 @@
 <div class="edit-form">
     <div class="box-wrap inner">
     <form class="pure-form pure-form-stacked" action="{{url_for('scrub_page')}}" method="POST">
-        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
         <fieldset>
             <div class="pure-control-group">
                 This will remove all version snapshots/data, but keep your list of URLs. <br/>

changedetectionio/templates/settings.html

@@ -1,7 +1,7 @@
 {% extends 'base.html' %}
 
 {% block content %}
-{% from '_helpers.jinja' import render_field, render_button %}
+{% from '_helpers.jinja' import render_field %}
 {% from '_common_fields.jinja' import render_common_settings_form %}
 
 <script type="text/javascript" src="{{url_for('static_content', group='js', filename='settings.js')}}" defer></script>
@@ -18,7 +18,6 @@
     </div>
     <div class="box-wrap inner">
         <form class="pure-form pure-form-stacked settings" action="{{url_for('settings_page')}}" method="POST">
-            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
             <div class="tab-pane-inner" id="general">
                 <fieldset>
                     <div class="pure-control-group">
@@ -28,7 +27,8 @@
                     <div class="pure-control-group">
                         {% if not hide_remove_pass %}
                             {% if current_user.is_authenticated %}
-                                {{ render_button(form.removepassword_button) }}
+                            <a href="{{url_for('settings_page', removepassword='yes')}}"
+                               class="pure-button pure-button-primary">Remove password</a>
                             {% else %}
                             {{ render_field(form.password) }}
                             <span class="pure-form-message-inline">Password protection for your changedetection.io application.</span>
@@ -104,7 +104,7 @@ nav
                         <ul>
                             <li>Note: This is applied globally in addition to the per-watch rules.</li>
                             <li>Each line processed separately, any line matching will be ignored (removed before creating the checksum)</li>
-                            <li>Regular Expression support, wrap the line in forward slash <code>/regex/</code></li>
+                            <li>Regular Expression support, wrap the line in forward slash <b>/regex/</b></li>
                             <li>Changing this will affect the comparison checksum which may trigger an alert</li>
                             <li>Use the preview/show current tab to see ignores</li>
                         </ul>
@@ -114,9 +114,11 @@ nav
 
             <div id="actions">
                 <div class="pure-control-group">
-                    {{ render_button(form.save_button) }}
-                    <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
-                    <a href="{{url_for('scrub_page')}}" class="pure-button button-small button-cancel">Delete History Snapshot Data</a>
+                    <button type="submit" class="pure-button pure-button-primary">Save</button>
+                                           <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
+                        <a href="{{url_for('scrub_page')}}" class="pure-button button-small button-cancel">Delete
+                            History
+                            Snapshot Data</a>
                 </div>
             </div>
         </form>

changedetectionio/templates/watch-overview.html

@@ -5,7 +5,6 @@
 <div class="box">
 
     <form class="pure-form" action="{{ url_for('api_watch_add') }}" method="POST" id="new-watch-form">
-        <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
         <fieldset>
             <legend>Add a new change detection watch</legend>
                 {{ render_simple_field(form.url, placeholder="https://...", required=true) }}

changedetectionio/tests/conftest.py

@@ -42,9 +42,6 @@ def app(request):
     cleanup(app_config['datastore_path'])
     datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], include_default_watches=False)
     app = changedetection_app(app_config, datastore)
-
-    # Disable CSRF while running tests
-    app.config['WTF_CSRF_ENABLED'] = False
     app.config['STOP_THREADS'] = True
 
     def teardown():

changedetectionio/tests/test_access_control.py

@@ -4,8 +4,8 @@ from flask import url_for
 def test_check_access_control(app, client):
     # Still doesnt work, but this is closer.
 
-    with app.test_client(use_cookies=True) as c:
-        # Check we don't have any password protection enabled yet.
+    with app.test_client() as c:
+        # Check we dont have any password protection enabled yet.
         res = c.get(url_for("settings_page"))
         assert b"Remove password" not in res.data
 
@@ -46,20 +46,15 @@ def test_check_access_control(app, client):
         assert b"BACKUP" in res.data
         assert b"IMPORT" in res.data
         assert b"LOG OUT" in res.data
-        assert b"minutes_between_check" in res.data
-        assert b"fetch_backend" in res.data
 
-        res = c.post(
-            url_for("settings_page"),
-            data={
-                "minutes_between_check": 180,
-                "tag": "",
-                "headers": "",
-                "fetch_backend": "html_webdriver",
-                "removepassword_button": "Remove password"
-            },
-            follow_redirects=True,
-        )
+        # Now remove the password so other tests function, @todo this should happen before each test automatically
+        res = c.get(url_for("settings_page", removepassword="yes"),
+              follow_redirects=True)
+        assert b"Password protection removed." in res.data
+
+        res = c.get(url_for("index"))
+        assert b"LOG OUT" not in res.data
+
 
 # There was a bug where saving the settings form would submit a blank password
 def test_check_access_control_no_blank_password(app, client):
@@ -76,7 +71,8 @@ def test_check_access_control_no_blank_password(app, client):
             data={"password": "",
                   "minutes_between_check": 180,
                   'fetch_backend': "html_requests"},
-            follow_redirects=True
+
+        follow_redirects=True
         )
 
         assert b"Password protection enabled." not in res.data
@@ -95,8 +91,7 @@ def test_check_access_no_remote_access_to_remove_password(app, client):
         # Enable password check.
         res = c.post(
             url_for("settings_page"),
-            data={"password": "password",
-                  "minutes_between_check": 180,
+            data={"password": "password", "minutes_between_check": 180,
                   'fetch_backend': "html_requests"},
             follow_redirects=True
         )
@@ -104,17 +99,8 @@ def test_check_access_no_remote_access_to_remove_password(app, client):
         assert b"Password protection enabled." in res.data
         assert b"Login" in res.data
 
-        res = c.post(
-            url_for("settings_page"),
-            data={
-                "minutes_between_check": 180,
-                "tag": "",
-                "headers": "",
-                "fetch_backend": "html_webdriver",
-                "removepassword_button": "Remove password"
-            },
-            follow_redirects=True,
-        )
+        res = c.get(url_for("settings_page", removepassword="yes"),
+              follow_redirects=True)
         assert b"Password protection removed." not in res.data
 
         res = c.get(url_for("index"),

changedetectionio/tests/test_backend.py

@@ -25,7 +25,6 @@ def test_check_basic_change_detection_functionality(client, live_server):
         data={"urls": url_for('test_endpoint', _external=True)},
         follow_redirects=True
     )
-
     assert b"1 Imported" in res.data
 
     time.sleep(sleep_time_for_fetch_thread)

changedetectionio/tests/test_encoding.py

@@ -1,87 +0,0 @@
-#!/usr/bin/python3
-# coding=utf-8
-
-import time
-from flask import url_for
-from .util import live_server_setup
-import pytest
-
-
-def test_setup(live_server):
-    live_server_setup(live_server)
-
-
-def set_html_response():
-    test_return_data = """
-<html><body><span class="nav_second_img_text">
-                  &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;铸大国重器，挺制造脊梁，致力能源未来，赋能美好生活。
-                                  </span>
-</body></html>
-    """
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-    return None
-
-
-# In the case the server does not issue a charset= or doesnt have content_type header set
-def test_check_encoding_detection(client, live_server):
-    set_html_response()
-
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', content_type="text/html", _external=True)
-    client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-
-    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
-
-    # Give the thread time to pick it up
-    time.sleep(2)
-
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-
-    # Should see the proper string
-    assert "铸大国重".encode('utf-8') in res.data
-    # Should not see the failed encoding
-    assert b'\xc2\xa7' not in res.data
-
-
-# In the case the server does not issue a charset= or doesnt have content_type header set
-def test_check_encoding_detection_missing_content_type_header(client, live_server):
-    set_html_response()
-
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', _external=True)
-    client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-
-    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
-
-    # Give the thread time to pick it up
-    time.sleep(2)
-
-    res = client.get(
-        url_for("preview_page", uuid="first"),
-        follow_redirects=True
-    )
-
-    # Should see the proper string
-    assert "铸大国重".encode('utf-8') in res.data
-    # Should not see the failed encoding
-    assert b'\xc2\xa7' not in res.data

changedetectionio/tests/test_errorhandling.py

@@ -1,7 +1,6 @@
 #!/usr/bin/python3
 
 import time
-
 from flask import url_for
 from . util import live_server_setup
 
@@ -18,9 +17,7 @@ def test_error_handler(client, live_server):
     time.sleep(1)
 
     # Add our URL to the import page
-    test_url = url_for('test_endpoint',
-                       status_code=403,
-                       _external=True)
+    test_url = url_for('test_endpoint_403_error', _external=True)
     res = client.post(
         url_for("import_page"),
         data={"urls": test_url},

changedetectionio/tests/test_ignorestatuscode.py

@@ -1,190 +0,0 @@
-#!/usr/bin/python3
-
-import time
-from flask import url_for
-from . util import live_server_setup
-
-
-def test_setup(live_server):
-    live_server_setup(live_server)
-
-
-def set_original_response():
-    test_return_data = """<html>
-       <body>
-     Some initial text</br>
-     <p>Which is across multiple lines</p>
-     </br>
-     So let's see what happens.  </br>
-     </body>
-     </html>
-    """
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-
-
-def set_some_changed_response():
-    test_return_data = """<html>
-       <body>
-     Some initial text</br>
-     <p>Which is across multiple lines, and a new thing too.</p>
-     </br>
-     So let's see what happens.  </br>
-     </body>
-     </html>
-    """
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write(test_return_data)
-
-
-def test_normal_page_check_works_with_ignore_status_code(client, live_server):
-    sleep_time_for_fetch_thread = 3
-
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
-    set_original_response()
-
-    # Goto the settings page, add our ignore text
-    res = client.post(
-        url_for("settings_page"),
-        data={
-            "minutes_between_check": 180,
-            "ignore_status_codes": "y",
-            'fetch_backend': "html_requests"
-        },
-        follow_redirects=True
-    )
-    assert b"Settings updated." in res.data
-
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-
-    time.sleep(sleep_time_for_fetch_thread)
-    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
-
-    set_some_changed_response()
-    time.sleep(sleep_time_for_fetch_thread)
-    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
-
-    # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
-
-    # It should report nothing found (no new 'unviewed' class)
-    res = client.get(url_for("index"))
-    assert b'unviewed' in res.data
-    assert b'/test-endpoint' in res.data
-
-
-# Tests the whole stack works with staus codes ignored
-def test_403_page_check_works_with_ignore_status_code(client, live_server):
-    sleep_time_for_fetch_thread = 3
-
-    set_original_response()
-
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', status_code=403, _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-
-    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
-
-    # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
-
-    # Goto the edit page, check our ignore option
-    # Add our URL to the import page
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={"ignore_status_codes": "y", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-
-    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
-
-    # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
-    #  Make a change
-    set_some_changed_response()
-
-    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
-    # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
-
-    # It should have 'unviewed' still
-    # Because it should be looking at only that 'sametext' id
-    res = client.get(url_for("index"))
-    assert b'unviewed' in res.data
-
-
-# Tests the whole stack works with staus codes ignored
-def test_403_page_check_fails_without_ignore_status_code(client, live_server):
-    sleep_time_for_fetch_thread = 3
-
-    set_original_response()
-
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', status_code=403, _external=True)
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": test_url},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-
-    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
-
-    # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
-
-    # Goto the edit page, check our ignore option
-    # Add our URL to the import page
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={"url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
-        follow_redirects=True
-    )
-    assert b"Updated watch." in res.data
-
-    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
-
-    # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
-    #  Make a change
-    set_some_changed_response()
-
-    # Trigger a check
-    client.get(url_for("api_watch_checknow"), follow_redirects=True)
-    # Give the thread time to pick it up
-    time.sleep(sleep_time_for_fetch_thread)
-
-    # It should have 'unviewed' still
-    # Because it should be looking at only that 'sametext' id
-    res = client.get(url_for("index"))
-    assert b'Status Code 403' in res.data

changedetectionio/tests/test_security.py

@@ -1,36 +0,0 @@
-from flask import url_for
-from . util import set_original_response, set_modified_response, live_server_setup
-import time
-
-def test_setup(live_server):
-    live_server_setup(live_server)
-
-def test_file_access(client, live_server):
-
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": 'https://localhost'},
-        follow_redirects=True
-    )
-
-    assert b"1 Imported" in res.data
-
-    # Attempt to add a body with a GET method
-    res = client.post(
-        url_for("edit_page", uuid="first"),
-        data={
-              "url": 'file:///etc/passwd',
-              "tag": "",
-              "method": "GET",
-              "fetch_backend": "html_requests",
-              "body": ""},
-        follow_redirects=True
-    )
-    time.sleep(3)
-
-    res = client.get(
-        url_for("index", uuid="first"),
-        follow_redirects=True
-    )
-
-    assert b'denied for security reasons' in res.data

changedetectionio/tests/util.py

@@ -38,19 +38,21 @@ def set_modified_response():
 
 def live_server_setup(live_server):
 
+
     @live_server.app.route('/test-endpoint')
     def test_endpoint():
         ctype = request.args.get('content_type')
-        status_code = request.args.get('status_code')
 
-        try:
-            # Tried using a global var here but didn't seem to work, so reading from a file instead.
-            with open("test-datastore/endpoint-content.txt", "r") as f:
-                resp = make_response(f.read(), status_code)
-                resp.headers['Content-Type'] = ctype if ctype else 'text/html'
-                return resp
-        except FileNotFoundError:
-            return make_response('', status_code)
+        # Tried using a global var here but didn't seem to work, so reading from a file instead.
+        with open("test-datastore/endpoint-content.txt", "r") as f:
+            resp = make_response(f.read())
+            resp.headers['Content-Type'] = ctype if ctype else 'text/html'
+            return resp
+
+    @live_server.app.route('/test-403')
+    def test_endpoint_403_error():
+        resp = make_response('', 403)
+        return resp
 
     # Just return the headers in the request
     @live_server.app.route('/test-headers')

changedetectionio/update_worker.py

@@ -42,6 +42,7 @@ class update_worker(threading.Thread):
                     now = time.time()
 
                     try:
+
                         changed_detected, update_obj, contents = update_handler.run(uuid)
 
                         # Re #342
@@ -49,6 +50,8 @@ class update_worker(threading.Thread):
                         # We then convert/.decode('utf-8') for the notification etc
                         if not isinstance(contents, (bytes, bytearray)):
                             raise Exception("Error - returned data from the fetch handler SHOULD be bytes")
+
+
                     except PermissionError as e:
                         self.app.logger.error("File permission error updating", uuid, str(e))
                     except content_fetcher.EmptyReply as e:

docker-compose.yml

@@ -2,7 +2,7 @@ version: '2'
 services:
     changedetection:
       image: ghcr.io/dgtlmoon/changedetection.io
-      container_name: changedetection
+      container_name: changedetection.io
       hostname: changedetection
       volumes:
         - changedetection-data:/datastore

requirements.txt

@@ -1,5 +1,5 @@
 flask~= 2.0
-flask_wtf
+
 eventlet>=0.31.0
 validators
 timeago ~=1.0

setup.py

@@ -32,11 +32,11 @@ setup(
     long_description_content_type='text/markdown',
     keywords='website change monitor for changes notification change detection '
              'alerts tracking website tracker change alert website and monitoring',
-    entry_points={"console_scripts": ["changedetection.io=changedetectionio.changedetection:main"]},
-    zip_safe=True,
-    scripts=["changedetection.py"],
+    zip_safe=False,
+    entry_points={"console_scripts": ["changedetection.io=changedetection:main"]},
     author='dgtlmoon',
     url='https://changedetection.io',
+    scripts=['changedetection.py'],
     packages=['changedetectionio'],
     include_package_data=True,
     install_requires=install_requires,