Re #1486

.github/workflows/pypi.yml

@@ -0,0 +1,38 @@
+name: PyPi Test and Push tagged release
+
+# Triggers the workflow on push or pull request events
+on:
+  workflow_run:
+    workflows: ["ChangeDetection.io Test"]
+    tags: '*.*'
+    types: [completed]
+
+
+jobs:
+  test-build:
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+
+      - name: Test that pip builds without error
+        run: |
+          pip3 --version
+          python3 -m pip install wheel
+          python3 setup.py bdist_wheel
+          python3 -m pip install dist/changedetection.io-*-none-any.whl --force
+          changedetection.io -d /tmp -p 10000 &
+          sleep 3
+          curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
+          killall -9 changedetection.io
+
+      # https://github.com/docker/build-push-action/blob/master/docs/advanced/test-before-push.md ?
+      # https://github.com/docker/buildx/issues/59 ? Needs to be one platform?
+
+      # https://github.com/docker/buildx/issues/495#issuecomment-918925854
+#if: ${{ github.event_name == 'release'}}

.github/workflows/test-pip-build.yml

@@ -1,36 +0,0 @@
-name: ChangeDetection.io PIP package test
-
-# Triggers the workflow on push or pull request events
-
-# This line doesnt work, even tho it is the documented one
-on: [push, pull_request]
-
-  # Changes to requirements.txt packages and Dockerfile may or may not always be compatible with arm etc, so worth testing
-  # @todo: some kind of path filter for requirements.txt and Dockerfile
-jobs:
-  test-pip-build-basics:
-    runs-on: ubuntu-latest
-    steps:
-        - uses: actions/checkout@v2
-
-        - name: Set up Python 3.9
-          uses: actions/setup-python@v2
-          with:
-            python-version: 3.9
-
-
-        - name: Test that the basic pip built package runs without error
-          run: |
-            set -e
-            mkdir dist
-            pip3 install wheel
-            python3 setup.py bdist_wheel            
-            pip3 install -r requirements.txt
-            rm ./changedetection.py
-            rm -rf changedetectio
-            
-            pip3 install dist/changedetection.io*.whl
-            changedetection.io -d /tmp -p 10000 &
-            sleep 3
-            curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null
-            killall -9 changedetection.io

README.md

@@ -65,7 +65,6 @@ Requires Playwright to be enabled.
 - Get notified when certain keywords appear in Twitter search results
 - Proactively search for jobs, get notified when companies update their careers page, search job portals for keywords.
 - Get alerts when new job positions are open on Bamboo HR and other job platforms
-- Website defacement monitoring
 
 _Need an actual Chrome runner with Javascript support? We support fetching via WebDriver and Playwright!</a>_
 

changedetectionio/__init__.py

@@ -430,26 +430,15 @@ def changedetection_app(config=None, datastore_o=None):
                                  has_unviewed=datastore.has_unviewed,
                                  hosted_sticky=os.getenv("SALTED_PASS", False) == False,
                                  queued_uuids=[q_uuid.item['uuid'] for q_uuid in update_q.queue],
-                                 sort_attribute=request.args.get('sort') if request.args.get('sort') else request.cookies.get('sort'),
-                                 sort_order=request.args.get('order') if request.args.get('order') else request.cookies.get('order'),
                                  system_default_fetcher=datastore.data['settings']['application'].get('fetch_backend'),
                                  tags=existing_tags,
                                  watches=sorted_watches
                                  )
 
+
         if session.get('share-link'):
             del(session['share-link'])
-
-        resp = make_response(output)
-
-        # The template can run on cookie or url query info
-        if request.args.get('sort'):
-            resp.set_cookie('sort', request.args.get('sort'))
-        if request.args.get('order'):
-            resp.set_cookie('order', request.args.get('order'))
-
-        return resp
-
+        return output
 
 
     # AJAX endpoint for sending a test
@@ -474,19 +463,11 @@ def changedetection_app(config=None, datastore_o=None):
 
         try:
             n_object = {'watch_url': request.form['window_url'],
-                        'notification_urls': request.form['notification_urls'].splitlines()
+                        'notification_urls': request.form['notification_urls'].splitlines(),
+                        'notification_title': request.form['notification_title'].strip(),
+                        'notification_body': request.form['notification_body'].strip(),
+                        'notification_format': request.form['notification_format'].strip()
                         }
-
-            # Only use if present, if not set in n_object it should use the default system value
-            if 'notification_format' in request.form and request.form['notification_format'].strip():
-                n_object['notification_format'] = request.form.get('notification_format', '').strip()
-
-            if 'notification_title' in request.form and request.form['notification_title'].strip():
-                n_object['notification_title'] = request.form.get('notification_title', '').strip()
-
-            if 'notification_body' in request.form and request.form['notification_body'].strip():
-                n_object['notification_body'] = request.form.get('notification_body', '').strip()
-
             notification_q.put(n_object)
         except Exception as e:
             return make_response({'error': str(e)}, 400)

changedetectionio/apprise_asset.py

@@ -3,7 +3,8 @@ import apprise
 # Create our AppriseAsset and populate it with some of our new values:
 # https://github.com/caronc/apprise/wiki/Development_API#the-apprise-asset-object
 asset = apprise.AppriseAsset(
-   image_url_logo='https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png'
+   image_url_logo='https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png',
+   async_mode=False  # Maybe related to https://github.com/dgtlmoon/changedetection.io/issues/1486
 )
 
 asset.app_id = "changedetection.io"

changedetectionio/model/Watch.py

@@ -20,7 +20,6 @@ base_config = {
     'body': None,
     'check_unique_lines': False,  # On change-detected, compare against all history if its something new
     'check_count': 0,
-    'date_created': None,
     'consecutive_filter_failures': 0,  # Every time the CSS/xPath filter cannot be located, reset when all is fine.
     'extract_text': [],  # Extract text by regex after filters
     'extract_title_as_title': False,

changedetectionio/notification.py

@@ -89,7 +89,7 @@ def process_notification(n_object, datastore):
     n_body = jinja2_env.from_string(n_object.get('notification_body', default_notification_body)).render(**notification_parameters)
     n_title = jinja2_env.from_string(n_object.get('notification_title', default_notification_title)).render(**notification_parameters)
     n_format = valid_notification_formats.get(
-        n_object.get('notification_format', default_notification_format),
+        n_object['notification_format'],
         valid_notification_formats[default_notification_format],
     )
     

changedetectionio/res/stock-not-in-stock.js

@@ -4,29 +4,27 @@ function isItemInStock() {
     '0 in stock',
     'agotado',
     'artikel zurzeit vergriffen',
-    'as soon as stock is available',
     'available for back order',
     'backordered',
-    'brak na stanie',
     'brak w magazynie',
+    'brak na stanie',
     'coming soon',
     'currently unavailable',
     'en rupture de stock',
-    'item is no longer available',
+    'as soon as stock is available',
     'message if back in stock',
     'nachricht bei',
     'nicht auf lager',
     'nicht lieferbar',
     'nicht zur verfügung',
     'no disponible temporalmente',
-    'no longer in stock',
     'not available',
     'not in stock',
-    'notify me when available',
-    'não estamos a aceitar encomendas',
     'out of stock',
     'out-of-stock',
+    'não estamos a aceitar encomendas',
     'produkt niedostępny',
+    'no longer in stock',
     'sold out',
     'temporarily out of stock',
     'temporarily unavailable',
@@ -34,19 +32,7 @@ function isItemInStock() {
     'zur zeit nicht an lager',
   ];
 
-
-  const negateOutOfStockRegexs = [
-      '[0-9] in stock'
-  ]
-  var negateOutOfStockRegexs_r = [];
-  for (let i = 0; i < negateOutOfStockRegexs.length; i++) {
-    negateOutOfStockRegexs_r.push(new RegExp(negateOutOfStockRegexs[0], 'g'));
-  }
-
-
   const elementsWithZeroChildren = Array.from(document.getElementsByTagName('*')).filter(element => element.children.length === 0);
-
-  // REGEXS THAT REALLY MEAN IT'S IN STOCK
   for (let i = elementsWithZeroChildren.length - 1; i >= 0; i--) {
     const element = elementsWithZeroChildren[i];
     if (element.offsetWidth > 0 || element.offsetHeight > 0 || element.getClientRects().length > 0) {
@@ -57,39 +43,13 @@ function isItemInStock() {
         elementText = element.textContent.toLowerCase();
       }
 
-      if (elementText.length) {
-        // try which ones could mean its in stock
-        for (let i = 0; i < negateOutOfStockRegexs.length; i++) {
-          if (negateOutOfStockRegexs_r[i].test(elementText)) {
-            return 'Possibly in stock';
-          }
+      for (const outOfStockText of outOfStockTexts) {
+        if (elementText.includes(outOfStockText)) {
+          return elementText; // item is out of stock
         }
       }
     }
   }
-
-  // OTHER STUFF THAT COULD BE THAT IT'S OUT OF STOCK
-  for (let i = elementsWithZeroChildren.length - 1; i >= 0; i--) {
-    const element = elementsWithZeroChildren[i];
-    if (element.offsetWidth > 0 || element.offsetHeight > 0 || element.getClientRects().length > 0) {
-      var elementText="";
-      if (element.tagName.toLowerCase() === "input") {
-        elementText = element.value.toLowerCase();
-      } else {
-        elementText = element.textContent.toLowerCase();
-      }
-
-      if (elementText.length) {
-        // and these mean its out of stock
-        for (const outOfStockText of outOfStockTexts) {
-          if (elementText.includes(outOfStockText)) {
-            return elementText; // item is out of stock
-          }
-        }
-      }
-    }
-  }
-
   return 'Possibly in stock'; // possibly in stock, cant decide otherwise.
 }
 

changedetectionio/static/images/generic-icon.svg

@@ -1,3 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!-- Created with Inkscape (http://www.inkscape.org/) -->
-<svg width="61.649mm" height="61.649mm" version="1.1" viewBox="0 0 61.649 61.649" xml:space="preserve" xmlns="http://www.w3.org/2000/svg"><g transform="translate(66.269 -15.463)" fill="#3056d3"><g transform="matrix(1.423 0 0 1.423 101.16 69.23)" fill="#3056d3"><g transform="matrix(.8229 0 0 .8229 -23.378 -2.3935)" fill="#3056d3"><path d="m-88.248-43.007a26.323 26.323 0 0 0-26.323 26.323 26.323 26.323 0 0 0 26.323 26.323 26.323 26.323 0 0 0 26.323-26.323 26.323 26.323 0 0 0-26.323-26.323zm0 2.8417a23.482 23.482 0 0 1 23.482 23.482 23.482 23.482 0 0 1-23.482 23.482 23.482 23.482 0 0 1-23.482-23.482 23.482 23.482 0 0 1 23.482-23.482z"/><g transform="matrix(.26458 0 0 .26458 -115.65 -44.085)"><path d="m33.02 64.43c0.35-0.05 2.04-0.13 2.04-0.13h25.53s3.17 0.32 3.67 0.53c2.5 1.05 3.98 1.89 6.04 3.57 0.72 0.58 4.12 4.01 4.12 4.01l51.67 57.39s1.61 1.65 1.97 1.94c1.2 0.97 2.48 1.96 3.98 2.32 0.5 0.12 2.72 0.21 2.72 0.21h27.32l-8.83-9.04s-1.31-1.65-1.44-1.94c-0.45-0.93-0.59-2.59-0.13-3.51 0.35-0.69 1.46-1.87 2.23-1.98 1.03-0.14 2.12-0.39 3.02 0.14 0.33 0.2 1.64 1.32 1.64 1.32l17.49 17.49s1.35 1.09 1.6 1.6c0.17 0.34 0.29 0.82 0.15 1.18-0.17 0.42-1.42 1.63-1.42 1.63l-0.94 0.98-15.69 16.37s-1.44 1.4-1.79 1.67c-0.76 0.6-1.99 0.89-2.96 0.9-1.03 0-2.62-1.11-3.26-1.91-0.6-0.76-1.1-2.22-0.77-3.13 0.16-0.45 1.28-1.85 1.28-1.85l11.36-11.3-29.47-0.02-1.68 0.09s-4.16-0.66-5.26-1.03c-1.63-0.56-3.44-1.82-4.75-2.93-0.39-0.33-1.8-1.92-1.8-1.92l-51.7-59.28s-2-2.06-2.43-2.43c-1.37-1.17-2-1.62-3.76-2.34-0.44-0.18-3.45-0.55-3.45-0.55l-24.13-0.22s-2.23-0.15-2.61-0.22c-1.08-0.21-2.16-1.07-2.81-1.83-0.79-0.92-0.59-3.06 0.06-4.09 0.57-0.89 2.14-1.52 3.19-1.66z"/><path d="m86.1 109.7-17.13 19.65s-2 2.06-2.43 2.43c-1.37 1.17-2 1.62-3.76 2.34-0.44 0.18-3.45 0.55-3.45 0.55l-24.13 0.22s-2.23 0.15-2.61 0.22c-1.08 0.21-2.16 1.07-2.81 1.83-0.79 0.92-0.59 3.06 0.06 4.09 0.57 0.89 2.14 1.52 3.19 1.66 0.35 0.05 2.04 0.13 2.04 0.13h25.53s3.17-0.32 3.67-0.53c2.5-1.05 3.98-1.89 6.04-3.57 0.72-0.58 4.12-4.01 4.12-4.01l17.38-19.3z"/><path d="m177.81 67.6c-0.17-0.42-1.42-1.63-1.42-1.63l-0.94-0.98-15.69-16.37s-1.44-1.4-1.79-1.67c-0.76-0.6-1.99-0.89-2.96-0.9-1.03 0-2.62 1.11-3.26 1.91-0.6 0.76-1.1 2.22-0.77 3.13 0.16 0.45 1.28 1.85 1.28 1.85l11.36 11.3-29.47 0.02-1.68-0.09s-4.16 0.66-5.26 1.03c-1.63 0.56-3.44 1.82-4.75 2.93-0.39 0.33-1.8 1.92-1.8 1.92l-18.91 21.69 5.98 5.98 18.38-20.41s1.61-1.65 1.97-1.94c1.2-0.97 2.48-1.96 3.98-2.32 0.5-0.12 2.72-0.21 2.72-0.21h27.32l-8.83 9.04s-1.31 1.65-1.44 1.94c-0.45 0.93-0.59 2.59-0.13 3.51 0.35 0.69 1.46 1.87 2.23 1.98 1.03 0.14 2.12 0.39 3.02-0.14 0.33-0.2 1.64-1.32 1.64-1.32l17.49-17.49s1.35-1.09 1.6-1.6c0.17-0.34 0.29-0.82 0.15-1.18z"/></g></g></g></g></svg>

changedetectionio/static/js/notifications.js

@@ -26,6 +26,9 @@ $(document).ready(function() {
     data = {
         window_url : window.location.href,
         notification_urls : $('.notification-urls').val(),
+        notification_title : $('.notification-title').val(),
+        notification_body : $('.notification-body').val(),
+        notification_format : $('.notification-format').val(),
     }
     for (key in data) {
       if (!data[key].length) {

changedetectionio/store.py

@@ -316,8 +316,7 @@ class ChangeDetectionStore:
             # #Re 569
             new_watch = Watch.model(datastore_path=self.datastore_path, default={
                 'url': url,
-                'tag': tag,
-                'date_created': int(time.time())
+                'tag': tag
             })
 
             new_uuid = new_watch['uuid']
@@ -680,13 +679,3 @@ class ChangeDetectionStore:
             except:
                 continue
         return
-
-    # We don't know when the date_created was in the past until now, so just add an index number for now.
-    def update_11(self):
-        i = 0
-        for uuid, watch in self.data['watching'].items():
-            if not watch.get('date_created'):
-                watch['date_created'] = i
-            i+=1
-        return
-

changedetectionio/templates/watch-overview.html

@@ -49,18 +49,18 @@
         {% endfor %}
     </div>
 
-    {% set sort_order = sort_order or 'asc' %}
-    {% set sort_attribute = sort_attribute or 'last_changed'  %}
+    {% set sort_order = request.args.get('order', 'asc') == 'asc' %}
+    {% set sort_attribute = request.args.get('sort', 'last_changed')   %}
     {% set pagination_page = request.args.get('page', 0) %}
 
     <div id="watch-table-wrapper">
         <table class="pure-table pure-table-striped watch-table">
             <thead>
             <tr>
-                {% set link_order = "desc" if sort_order  == 'asc' else "asc" %}
-                {% set arrow_span = "" %}
-                <th><input style="vertical-align: middle" type="checkbox" id="check-all"/> <a class="{{ 'active '+link_order if sort_attribute == 'date_created' else 'inactive' }}"  href="{{url_for('index', sort='date_created', order=link_order, tag=active_tag)}}"># <span class='arrow {{link_order}}'></span></a></th>
+                <th><input style="vertical-align: middle" type="checkbox" id="check-all"/> #</th>
                 <th></th>
+                {% set link_order = "desc" if sort_order else "asc" %}
+                {% set arrow_span = "" %}
                 <th><a class="{{ 'active '+link_order if sort_attribute == 'label' else 'inactive' }}" href="{{url_for('index', sort='label', order=link_order, tag=active_tag)}}">Website <span class='arrow {{link_order}}'></span></a></th>
                 <th><a class="{{ 'active '+link_order if sort_attribute == 'last_checked' else 'inactive' }}" href="{{url_for('index', sort='last_checked', order=link_order, tag=active_tag)}}">Last Checked <span class='arrow {{link_order}}'></span></a></th>
                 <th><a class="{{ 'active '+link_order if sort_attribute == 'last_changed' else 'inactive' }}" href="{{url_for('index', sort='last_changed', order=link_order, tag=active_tag)}}">Last Changed <span class='arrow {{link_order}}'></span></a></th>
@@ -69,7 +69,8 @@
             </thead>
             <tbody>
 
-            {% for watch in watches|sort(attribute=sort_attribute, reverse=sort_order == 'asc') %}
+            {% set sorted_watches = watches|sort(attribute=sort_attribute, reverse=sort_order) %}
+            {% for watch in sorted_watches %}
 
             {# WIP for pagination, disabled for now
               {% if not ( loop.index >= 3 and loop.index <=4) %}{% continue %}{% endif %} -->

changedetectionio/tests/test_errorhandling.py

@@ -59,8 +59,6 @@ def test_http_error_handler(client, live_server):
     _runner_test_http_errors(client, live_server, 404, 'Page not found')
     _runner_test_http_errors(client, live_server, 500, '(Internal server Error) received')
     _runner_test_http_errors(client, live_server, 400, 'Error - Request returned a HTTP error code 400')
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
 
 # Just to be sure error text is properly handled
 def test_DNS_errors(client, live_server):
@@ -83,48 +81,4 @@ def test_DNS_errors(client, live_server):
     assert found_name_resolution_error
     # Should always record that we tried
     assert bytes("just now".encode('utf-8')) in res.data
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data
 
-# Re 1513
-def test_low_level_errors_clear_correctly(client, live_server):
-    #live_server_setup(live_server)
-    # Give the endpoint time to spin up
-    time.sleep(1)
-
-    with open("test-datastore/endpoint-content.txt", "w") as f:
-        f.write("<html><body><div id=here>Hello world</div></body></html>")
-
-    # Add our URL to the import page
-    test_url = url_for('test_endpoint', _external=True)
-
-    res = client.post(
-        url_for("import_page"),
-        data={"urls": "https://dfkjasdkfjaidjfsdajfksdajfksdjfDOESNTEXIST.com"},
-        follow_redirects=True
-    )
-    assert b"1 Imported" in res.data
-    time.sleep(2)
-
-    # We should see the DNS error
-    res = client.get(url_for("index"))
-    found_name_resolution_error = b"Temporary failure in name resolution" in res.data or b"Name or service not known" in res.data
-    assert found_name_resolution_error
-
-    # Update with what should work
-    client.post(
-        url_for("edit_page", uuid="first"),
-        data={
-            "url": test_url,
-            "fetch_backend": "html_requests"},
-        follow_redirects=True
-    )
-
-    # Now the error should be gone
-    time.sleep(2)
-    res = client.get(url_for("index"))
-    found_name_resolution_error = b"Temporary failure in name resolution" in res.data or b"Name or service not known" in res.data
-    assert not found_name_resolution_error
-
-    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
-    assert b'Deleted' in res.data