mirror of
https://github.com/dgtlmoon/changedetection.io.git
synced 2026-04-19 01:18:10 +00:00
Compare commits
2 Commits
0.28
...
diff-strea
| Author | SHA1 | Date | |
|---|---|---|---|
|
14d88c249e | ||
|
|
0fa443c3f2 |
77
.github/workflows/image.yml
vendored
77
.github/workflows/image.yml
vendored
@@ -1,77 +0,0 @@
|
||||
name: Test, build and push to Docker Hub
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set up Python 3.9
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.9
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install flake8 pytest
|
||||
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
|
||||
|
||||
- name: Lint with flake8
|
||||
run: |
|
||||
# stop the build if there are Python syntax errors or undefined names
|
||||
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
|
||||
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
|
||||
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
|
||||
- name: Test with pytest
|
||||
run: |
|
||||
cd backend; pytest
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v1
|
||||
with:
|
||||
image: tonistiigi/binfmt:latest
|
||||
platforms: all
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@v1
|
||||
with:
|
||||
install: true
|
||||
version: latest
|
||||
driver-opts: image=moby/buildkit:master
|
||||
- name: Build and push
|
||||
id: docker_build
|
||||
uses: docker/build-push-action@v2
|
||||
with:
|
||||
context: ./
|
||||
file: ./Dockerfile
|
||||
push: true
|
||||
tags: |
|
||||
${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
|
||||
# ${{ secrets.DOCKER_HUB_USERNAME }}:/changedetection.io:${{ env.RELEASE_VERSION }}
|
||||
platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
|
||||
cache-from: type=local,src=/tmp/.buildx-cache
|
||||
cache-to: type=local,dest=/tmp/.buildx-cache
|
||||
|
||||
- name: Image digest
|
||||
run: echo ${{ steps.docker_build.outputs.digest }}
|
||||
|
||||
- name: Cache Docker layers
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: /tmp/.buildx-cache
|
||||
key: ${{ runner.os }}-buildx-${{ github.sha }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-buildx-
|
||||
|
||||
|
||||
33
.github/workflows/python-app.yml
vendored
Normal file
33
.github/workflows/python-app.yml
vendored
Normal file
@@ -0,0 +1,33 @@
|
||||
# This workflow will install Python dependencies, run tests and lint with a single version of Python
|
||||
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
|
||||
|
||||
name: changedetection.io
|
||||
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
build:
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set up Python 3.9
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: 3.9
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install flake8 pytest
|
||||
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
|
||||
- name: Lint with flake8
|
||||
run: |
|
||||
# stop the build if there are Python syntax errors or undefined names
|
||||
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
|
||||
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
|
||||
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
|
||||
- name: Test with pytest
|
||||
run: |
|
||||
cd backend; pytest
|
||||
@@ -1,7 +1,5 @@
|
||||
FROM python:3.8-slim
|
||||
COPY requirements.txt /tmp/requirements.txt
|
||||
RUN apt-get update && apt-get install -y gcc libc-dev libxslt-dev zlib1g-dev g++
|
||||
|
||||
RUN pip3 install -r /tmp/requirements.txt
|
||||
|
||||
|
||||
|
||||
17
README.md
17
README.md
@@ -9,13 +9,11 @@
|
||||
|
||||
## Self-hosted change monitoring of web pages.
|
||||
|
||||
_Know when web pages change! Stay ontop of new information!_
|
||||
_Know when web pages change! Stay ontop of new information!_
|
||||
|
||||
Live your data-life *pro-actively* instead of *re-actively*, do not rely on manipulative social media for consuming important information.
|
||||
|
||||
|
||||
|
||||
<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring" title="Self-hosted web page change monitoring" />
|
||||
|
||||
#### Example use cases
|
||||
|
||||
Know when ...
|
||||
@@ -50,7 +48,16 @@ docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/dat
|
||||
|
||||
Examining differences in content.
|
||||
|
||||
|
||||
|
||||
<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/screenshot-diff.png" style="max-width:100%;" alt="Self-hosted web page change monitoring context difference " title="Self-hosted web page change monitoring context difference " />
|
||||
### Future plans
|
||||
|
||||
- Greater configuration of check interval times, page request headers.
|
||||
- ~~General options for timeout, default headers~~
|
||||
- On change detection, callout to another API (handy for notices/issue trackers)
|
||||
- ~~Explore the differences that were detected~~
|
||||
- Add more options to explore versions of differences
|
||||
- Use a graphic/rendered page difference instead of text (see the experimental `selenium-screenshot-diff` branch)
|
||||
|
||||
|
||||
Please :star: star :star: this project and help it grow! https://github.com/dgtlmoon/changedetection.io/
|
||||
|
||||
@@ -97,19 +97,10 @@ def changedetection_app(config=None, datastore_o=None):
|
||||
@app.route("/", methods=['GET'])
|
||||
def index():
|
||||
global messages
|
||||
|
||||
limit_tag = request.args.get('tag')
|
||||
|
||||
pause_uuid = request.args.get('pause')
|
||||
|
||||
if pause_uuid:
|
||||
try:
|
||||
datastore.data['watching'][pause_uuid]['paused'] ^= True
|
||||
datastore.needs_write = True
|
||||
|
||||
return redirect(url_for('index', limit_tag = limit_tag))
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
rss = request.args.get('rss')
|
||||
mode = request.args.get('mode')
|
||||
|
||||
# Sort by last_changed and add the uuid which is usually the key..
|
||||
sorted_watches = []
|
||||
@@ -130,7 +121,63 @@ def changedetection_app(config=None, datastore_o=None):
|
||||
sorted_watches.sort(key=lambda x: x['last_changed'], reverse=True)
|
||||
|
||||
existing_tags = datastore.get_all_tags()
|
||||
rss = request.args.get('rss')
|
||||
|
||||
if mode == 'stream':
|
||||
import difflib
|
||||
|
||||
import pprint
|
||||
streams = []
|
||||
|
||||
extra_stylesheets = ['/static/css/diff.css']
|
||||
for watch in sorted_watches:
|
||||
if not watch['viewed']:
|
||||
|
||||
# get last two date keys
|
||||
dates = list(watch['history'].keys())
|
||||
# Convert to int, sort and back to str again
|
||||
dates = [int(i) for i in dates]
|
||||
dates.sort(reverse=True)
|
||||
dates = [str(i) for i in dates]
|
||||
print ("OK", watch['uuid'])
|
||||
|
||||
if len(dates) < 2:
|
||||
print ("Skipping", watch['url'])
|
||||
continue
|
||||
else:
|
||||
try:
|
||||
path = datastore.data['watching'][watch['uuid']]['history'][str(dates[1])]
|
||||
with open(path,
|
||||
encoding='utf-8') as file:
|
||||
txt1=[line.rstrip() for line in file.readlines()]
|
||||
|
||||
path = datastore.data['watching'][watch['uuid']]['history'][str(dates[0])]
|
||||
with open(path,
|
||||
encoding='utf-8') as file:
|
||||
txt2 = [line.rstrip() for line in file.readlines()]
|
||||
except FileNotFoundError:
|
||||
print ("Skipping", watch['url'])
|
||||
continue
|
||||
|
||||
df = list(difflib.unified_diff(txt1, txt2,n=1))
|
||||
diff_entry=[]
|
||||
for line in df:
|
||||
if line[0] == '-' or line[0] == '+':
|
||||
diff_entry.append(line)
|
||||
|
||||
|
||||
# pprint(df)
|
||||
#s = pprint.pformat(df)
|
||||
streams.append(diff_entry)
|
||||
|
||||
|
||||
print ("###########", len(streams))
|
||||
|
||||
output = render_template("watch-diff-stream.html",
|
||||
streams=streams,
|
||||
extra_stylesheets=extra_stylesheets
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
if rss:
|
||||
fg = FeedGenerator()
|
||||
@@ -154,7 +201,8 @@ def changedetection_app(config=None, datastore_o=None):
|
||||
return response
|
||||
|
||||
else:
|
||||
output = render_template("watch-overview.html",
|
||||
#table = render_template('watch-table.html', watches=sorted_watches)
|
||||
output = render_template("watch-table.html",
|
||||
watches=sorted_watches,
|
||||
messages=messages,
|
||||
tags=existing_tags,
|
||||
@@ -174,16 +222,19 @@ def changedetection_app(config=None, datastore_o=None):
|
||||
|
||||
if request.method == 'POST':
|
||||
confirmtext = request.form.get('confirmtext')
|
||||
limit_timestamp = int(request.form.get('limit_date'))
|
||||
|
||||
if confirmtext == 'scrub':
|
||||
|
||||
for uuid, watch in datastore.data['watching'].items():
|
||||
if len(str(limit_timestamp)) == 10:
|
||||
datastore.scrub_watch(uuid, limit_timestamp = limit_timestamp)
|
||||
else:
|
||||
datastore.scrub_watch(uuid)
|
||||
for txt_file_path in Path(app.config['datastore_path']).rglob('*.txt'):
|
||||
os.unlink(txt_file_path)
|
||||
|
||||
for uuid, watch in datastore.data['watching'].items():
|
||||
watch['last_checked'] = 0
|
||||
watch['last_changed'] = 0
|
||||
watch['previous_md5'] = None
|
||||
watch['history'] = {}
|
||||
|
||||
datastore.needs_write = True
|
||||
messages.append({'class': 'ok', 'message': 'Cleaned all version history.'})
|
||||
else:
|
||||
messages.append({'class': 'error', 'message': 'Wrong confirm text.'})
|
||||
@@ -497,17 +548,15 @@ def changedetection_app(config=None, datastore_o=None):
|
||||
# Items that have this current tag
|
||||
for watch_uuid, watch in datastore.data['watching'].items():
|
||||
if (tag != None and tag in watch['tag']):
|
||||
if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
|
||||
i += 1
|
||||
if watch_uuid not in running_uuids:
|
||||
update_q.put(watch_uuid)
|
||||
i += 1
|
||||
|
||||
else:
|
||||
# No tag, no uuid, add everything.
|
||||
for watch_uuid, watch in datastore.data['watching'].items():
|
||||
|
||||
if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
|
||||
i += 1
|
||||
if watch_uuid not in running_uuids:
|
||||
update_q.put(watch_uuid)
|
||||
i += 1
|
||||
|
||||
messages.append({'class': 'ok', 'message': "{} watches are rechecking.".format(i)})
|
||||
return redirect(url_for('index', tag=tag))
|
||||
@@ -571,6 +620,7 @@ class Worker(threading.Thread):
|
||||
self.current_uuid = uuid
|
||||
|
||||
if uuid in list(datastore.data['watching'].keys()):
|
||||
|
||||
try:
|
||||
changed_detected, result, contents = update_handler.run(uuid)
|
||||
|
||||
@@ -578,6 +628,7 @@ class Worker(threading.Thread):
|
||||
app.logger.error("File permission error updating", uuid, str(s))
|
||||
else:
|
||||
if result:
|
||||
|
||||
datastore.update_watch(uuid=uuid, update_obj=result)
|
||||
if changed_detected:
|
||||
# A change was detected
|
||||
|
||||
@@ -267,11 +267,7 @@ footer {
|
||||
color: #e07171;
|
||||
}
|
||||
|
||||
.paused-state.state-False img {
|
||||
opacity: 0.2;
|
||||
#diff-stream {
|
||||
font-size: 10px;
|
||||
white-space: pre-wrap;
|
||||
}
|
||||
|
||||
|
||||
.paused-state.state-False:hover img{
|
||||
opacity: 0.8;
|
||||
}
|
||||
@@ -1,84 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<svg
|
||||
xmlns:dc="http://purl.org/dc/elements/1.1/"
|
||||
xmlns:cc="http://creativecommons.org/ns#"
|
||||
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
|
||||
xmlns:svg="http://www.w3.org/2000/svg"
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
version="1.1"
|
||||
id="Capa_1"
|
||||
x="0px"
|
||||
y="0px"
|
||||
viewBox="0 0 15 14.998326"
|
||||
xml:space="preserve"
|
||||
width="15"
|
||||
height="14.998326"><metadata
|
||||
id="metadata39"><rdf:RDF><cc:Work
|
||||
rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
|
||||
rdf:resource="http://purl.org/dc/dcmitype/StillImage" /><dc:title></dc:title></cc:Work></rdf:RDF></metadata><defs
|
||||
id="defs37" />
|
||||
<path
|
||||
id="path2"
|
||||
style="fill:#1b98f8;fill-opacity:1;stroke-width:0.0292893"
|
||||
d="M 7.4975161,6.5052867e-4 C 4.549072,-0.04028702 1.7055675,1.8548221 0.58868606,4.5801341 -0.57739762,7.2574642 0.02596981,10.583326 2.069916,12.671949 4.0364753,14.788409 7.2763651,15.56067 9.989207,14.57284 12.801145,13.617602 14.87442,10.855325 14.985833,7.8845744 15.172496,4.9966544 13.49856,2.1100704 10.911002,0.8209349 9.8598067,0.28073592 8.6791261,-0.00114855 7.4975161,6.5052867e-4 Z M 6.5602569,10.251923 c -0.00509,0.507593 -0.5693885,0.488472 -0.9352002,0.468629 -0.3399386,0.0018 -0.8402048,0.07132 -0.9297965,-0.374189 -0.015842,-1.8973128 -0.015872,-3.7979649 0,-5.6952784 0.1334405,-0.5224315 0.7416869,-0.3424086 1.1377562,-0.374189 0.3969969,-0.084515 0.8245634,0.1963256 0.7272405,0.6382917 0,1.7789118 0,3.5578239 0,5.3367357 z m 3.7490371,0 c -0.0051,0.507593 -0.5693888,0.488472 -0.9352005,0.468629 -0.3399386,0.0018 -0.8402048,0.07132 -0.9297965,-0.374189 -0.015842,-1.8973128 -0.015872,-3.7979649 0,-5.6952784 0.1334405,-0.5224315 0.7416869,-0.3424086 1.1377562,-0.374189 0.3969969,-0.084515 0.8245638,0.1963256 0.7272408,0.6382917 0,1.7789118 0,3.5578239 0,5.3367357 z" />
|
||||
<g
|
||||
id="g4"
|
||||
transform="translate(-0.01903604,0.02221043)">
|
||||
</g>
|
||||
<g
|
||||
id="g6"
|
||||
transform="translate(-0.01903604,0.02221043)">
|
||||
</g>
|
||||
<g
|
||||
id="g8"
|
||||
transform="translate(-0.01903604,0.02221043)">
|
||||
</g>
|
||||
<g
|
||||
id="g10"
|
||||
transform="translate(-0.01903604,0.02221043)">
|
||||
</g>
|
||||
<g
|
||||
id="g12"
|
||||
transform="translate(-0.01903604,0.02221043)">
|
||||
</g>
|
||||
<g
|
||||
id="g14"
|
||||
transform="translate(-0.01903604,0.02221043)">
|
||||
</g>
|
||||
<g
|
||||
id="g16"
|
||||
transform="translate(-0.01903604,0.02221043)">
|
||||
</g>
|
||||
<g
|
||||
id="g18"
|
||||
transform="translate(-0.01903604,0.02221043)">
|
||||
</g>
|
||||
<g
|
||||
id="g20"
|
||||
transform="translate(-0.01903604,0.02221043)">
|
||||
</g>
|
||||
<g
|
||||
id="g22"
|
||||
transform="translate(-0.01903604,0.02221043)">
|
||||
</g>
|
||||
<g
|
||||
id="g24"
|
||||
transform="translate(-0.01903604,0.02221043)">
|
||||
</g>
|
||||
<g
|
||||
id="g26"
|
||||
transform="translate(-0.01903604,0.02221043)">
|
||||
</g>
|
||||
<g
|
||||
id="g28"
|
||||
transform="translate(-0.01903604,0.02221043)">
|
||||
</g>
|
||||
<g
|
||||
id="g30"
|
||||
transform="translate(-0.01903604,0.02221043)">
|
||||
</g>
|
||||
<g
|
||||
id="g32"
|
||||
transform="translate(-0.01903604,0.02221043)">
|
||||
</g>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 2.9 KiB |
@@ -47,7 +47,6 @@ class ChangeDetectionStore:
|
||||
'tag': None,
|
||||
'last_checked': 0,
|
||||
'last_changed': 0,
|
||||
'paused': False,
|
||||
'last_viewed': 0, # history key value of the last viewed via the [diff] link
|
||||
'newest_history_key': "",
|
||||
'title': None,
|
||||
@@ -104,7 +103,7 @@ class ChangeDetectionStore:
|
||||
self.add_watch(url='https://changedetection.io', tag='Tech news')
|
||||
|
||||
|
||||
self.__data['version_tag'] = "0.28"
|
||||
self.__data['version_tag'] = "0.27"
|
||||
|
||||
if not 'app_guid' in self.__data:
|
||||
self.__data['app_guid'] = str(uuid_builder.uuid4())
|
||||
@@ -135,10 +134,6 @@ class ChangeDetectionStore:
|
||||
|
||||
def update_watch(self, uuid, update_obj):
|
||||
|
||||
# Skip if 'paused' state
|
||||
if self.__data['watching'][uuid]['paused']:
|
||||
return
|
||||
|
||||
with self.lock:
|
||||
|
||||
# In python 3.9 we have the |= dict operator, but that still will lose data on nested structures...
|
||||
@@ -157,6 +152,7 @@ class ChangeDetectionStore:
|
||||
def data(self):
|
||||
|
||||
has_unviewed = False
|
||||
|
||||
for uuid, v in self.__data['watching'].items():
|
||||
self.__data['watching'][uuid]['newest_history_key'] = self.get_newest_history_key(uuid)
|
||||
if int(v['newest_history_key']) <= int(v['last_viewed']):
|
||||
@@ -183,28 +179,12 @@ class ChangeDetectionStore:
|
||||
tags.sort()
|
||||
return tags
|
||||
|
||||
def unlink_history_file(self, path):
|
||||
try:
|
||||
os.unlink(path)
|
||||
except (FileNotFoundError, IOError):
|
||||
pass
|
||||
|
||||
# Delete a single watch by UUID
|
||||
def delete(self, uuid):
|
||||
with self.lock:
|
||||
if uuid == 'all':
|
||||
self.__data['watching'] = {}
|
||||
|
||||
# GitHub #30 also delete history records
|
||||
for uuid in self.data['watching']:
|
||||
for path in self.data['watching'][uuid]['history'].values():
|
||||
self.unlink_history_file(path)
|
||||
|
||||
else:
|
||||
for path in self.data['watching'][uuid]['history'].values():
|
||||
self.unlink_history_file(path)
|
||||
|
||||
del self.data['watching'][uuid]
|
||||
del (self.__data['watching'][uuid])
|
||||
|
||||
self.needs_write = True
|
||||
|
||||
@@ -221,47 +201,6 @@ class ChangeDetectionStore:
|
||||
# Probably their should be dict...
|
||||
return self.data['watching'][uuid].get(val)
|
||||
|
||||
# Remove a watchs data but keep the entry (URL etc)
|
||||
def scrub_watch(self, uuid, limit_timestamp = False):
|
||||
|
||||
import hashlib
|
||||
del_timestamps = []
|
||||
|
||||
for timestamp, path in self.data['watching'][uuid]['history'].items():
|
||||
if not limit_timestamp or (limit_timestamp is not False and int(timestamp) > limit_timestamp):
|
||||
self.unlink_history_file(path)
|
||||
del_timestamps.append(timestamp)
|
||||
|
||||
|
||||
|
||||
if not limit_timestamp:
|
||||
self.data['watching'][uuid]['last_checked'] = 0
|
||||
self.data['watching'][uuid]['last_changed'] = 0
|
||||
self.data['watching'][uuid]['previous_md5'] = 0
|
||||
|
||||
for timestamp in del_timestamps:
|
||||
del self.data['watching'][uuid]['history'][str(timestamp)]
|
||||
|
||||
# If there was a limitstamp, we need to reset some meta data about the entry
|
||||
# This has to happen after we remove the others from the list
|
||||
if limit_timestamp:
|
||||
newest_key = self.get_newest_history_key(uuid)
|
||||
if newest_key:
|
||||
self.data['watching'][uuid]['last_checked'] = int(newest_key)
|
||||
# @todo should be the original value if it was less than newest key
|
||||
self.data['watching'][uuid]['last_changed'] = int(newest_key)
|
||||
try:
|
||||
with open(self.data['watching'][uuid]['history'][str(newest_key)], "rb") as fp:
|
||||
content = fp.read()
|
||||
self.data['watching'][uuid]['previous_md5'] = hashlib.md5(content).hexdigest()
|
||||
except (FileNotFoundError, IOError):
|
||||
self.data['watching'][uuid]['previous_md5'] = False
|
||||
pass
|
||||
|
||||
|
||||
self.needs_write = True
|
||||
|
||||
|
||||
def add_watch(self, url, tag):
|
||||
with self.lock:
|
||||
# @todo use a common generic version of this
|
||||
@@ -289,6 +228,12 @@ class ChangeDetectionStore:
|
||||
# result_obj from fetch_site_status.run()
|
||||
def save_history_text(self, uuid, result_obj, contents):
|
||||
|
||||
output_path = "{}/{}".format(self.datastore_path, uuid)
|
||||
try:
|
||||
os.mkdir(output_path)
|
||||
except FileExistsError:
|
||||
pass
|
||||
|
||||
output_path = "{}/{}".format(self.datastore_path, uuid)
|
||||
fname = "{}/{}-{}.stripped.txt".format(output_path, result_obj['previous_md5'], str(time.time()))
|
||||
with open(fname, 'w') as f:
|
||||
@@ -303,21 +248,11 @@ class ChangeDetectionStore:
|
||||
|
||||
def sync_to_json(self):
|
||||
print("Saving..")
|
||||
data ={}
|
||||
with open(self.json_store_path, 'w') as json_file:
|
||||
json.dump(self.__data, json_file, indent=4)
|
||||
logging.info("Re-saved index")
|
||||
|
||||
try:
|
||||
data = deepcopy(self.__data)
|
||||
except RuntimeError:
|
||||
time.sleep(0.5)
|
||||
print ("! Data changed when writing to JSON, trying again..")
|
||||
self.sync_to_json()
|
||||
return
|
||||
else:
|
||||
with open(self.json_store_path, 'w') as json_file:
|
||||
json.dump(data, json_file, indent=4)
|
||||
logging.info("Re-saved index")
|
||||
|
||||
self.needs_write = False
|
||||
self.needs_write = False
|
||||
|
||||
# Thread runner, this helps with thread/write issues when there are many operations that want to update the JSON
|
||||
# by just running periodically in one thread, according to python, dict updates are threadsafe.
|
||||
@@ -327,8 +262,8 @@ class ChangeDetectionStore:
|
||||
if self.stop_thread:
|
||||
print("Shutting down datastore thread")
|
||||
return
|
||||
|
||||
if self.needs_write:
|
||||
self.sync_to_json()
|
||||
time.sleep(3)
|
||||
time.sleep(1)
|
||||
|
||||
# body of the constructor
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
<fieldset>
|
||||
|
||||
<label for="diffWords" class="pure-checkbox">
|
||||
<input type="radio" name="diff_type" id="diffWords" value="diffWords"/> Words</label>
|
||||
<input type="radio" name="diff_type" id="diffWords" value="diffWords" /> Words</label>
|
||||
<label for="diffLines" class="pure-checkbox">
|
||||
<input type="radio" name="diff_type" id="diffLines" value="diffLines" checked=""/> Lines</label>
|
||||
|
||||
@@ -19,9 +19,9 @@
|
||||
<label for="diff-version">Compare newest (<span id="current-v-date"></span>) with</label>
|
||||
<select id="diff-version" name="previous_version">
|
||||
{% for version in versions %}
|
||||
<option value="{{version}}" {% if version== current_previous_version %} selected="" {% endif %}>
|
||||
{{version}}
|
||||
</option>
|
||||
<option value="{{version}}" {% if version== current_previous_version %} selected="" {% endif %}>
|
||||
{{version}}
|
||||
</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
<button type="submit" class="pure-button pure-button-primary">Go</button>
|
||||
@@ -90,10 +90,6 @@ function changed() {
|
||||
|
||||
result.textContent = '';
|
||||
result.appendChild(fragment);
|
||||
|
||||
// Jump at start
|
||||
inputs.current=0;
|
||||
next_diff();
|
||||
}
|
||||
|
||||
window.onload = function() {
|
||||
@@ -116,7 +112,6 @@ window.onload = function() {
|
||||
|
||||
onDiffTypeChange(document.querySelector('#settings [name="diff_type"]:checked'));
|
||||
changed();
|
||||
|
||||
};
|
||||
|
||||
a.onpaste = a.onchange =
|
||||
@@ -145,7 +140,6 @@ for (var i = 0; i < radio.length; i++) {
|
||||
var inputs = document.getElementsByClassName('change');
|
||||
inputs.current=0;
|
||||
|
||||
|
||||
function next_diff() {
|
||||
|
||||
var element = inputs[inputs.current];
|
||||
@@ -165,7 +159,6 @@ function next_diff() {
|
||||
}
|
||||
|
||||
|
||||
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
@@ -17,19 +17,14 @@
|
||||
|
||||
<div class="pure-control-group">
|
||||
<br/>
|
||||
<label for="confirmtext">Confirm text</label><br/>
|
||||
<label for="confirmtext">Confirm</label><br/>
|
||||
<input type="text" id="confirmtext" required="" name="confirmtext" value="" size="10"/>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="pure-control-group">
|
||||
<br/>
|
||||
<label for="confirmtext">Limit delete history including and after date</label><br/>
|
||||
<input type="text" id="limit_date" required="" name="limit_date" value="" size="10"/>
|
||||
<br/>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
<div class="pure-control-group">
|
||||
<button type="submit" class="pure-button pure-button-primary">Scrub!</button>
|
||||
</div>
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
|
||||
<div class="pure-control-group">
|
||||
<a href="/" class="pure-button button-small button-cancel">Back</a>
|
||||
<a href="/scrub" class="pure-button button-small button-cancel">Delete history version data</a>
|
||||
<a href="/scrub" class="pure-button button-small button-cancel">Reset all version data</a>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
12
backend/templates/watch-diff-stream.html
Normal file
12
backend/templates/watch-diff-stream.html
Normal file
@@ -0,0 +1,12 @@
|
||||
{% extends 'watch-overview.html' %}
|
||||
{% block innercontent %}
|
||||
Entries: {{ streams|length }}
|
||||
|
||||
<div id="diff-stream" class="edit-form">
|
||||
{% for item in streams %}
|
||||
{{ loop.index }}
|
||||
{% for diff in item %}{% if diff[0] =='+' %}<ins>{{ diff }}</ins>{% endif %}{% if diff[0] =='-' %}<del>{{ diff }}</del>{% endif %}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% endblock %}
|
||||
@@ -24,70 +24,9 @@
|
||||
</div>
|
||||
|
||||
<div id="watch-table-wrapper">
|
||||
<table class="pure-table pure-table-striped watch-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>#</th>
|
||||
<th></th>
|
||||
<th></th>
|
||||
<th>Last Checked</th>
|
||||
<th>Last Changed</th>
|
||||
<th></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% block innercontent %}
|
||||
|
||||
|
||||
{% for watch in watches %}
|
||||
<tr id="{{ watch.uuid }}"
|
||||
class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}
|
||||
{% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
|
||||
{% if watch.paused is defined and watch.paused != False %}paused{% endif %}
|
||||
{% if watch.newest_history_key| int > watch.last_viewed| int %}unviewed{% endif %}">
|
||||
<td>{{ loop.index }}</td>
|
||||
<td class="paused-state state-{{watch.paused}}"><a href="/?pause={{ watch.uuid}}"><img src="/static/images/pause.svg" alt="Pause"/></a></td>
|
||||
<td class="title-col">{{watch.title if watch.title is not none else watch.url}}
|
||||
<a class="external" target=_blank href="{{ watch.url }}"></a>
|
||||
{% if watch.last_error is defined and watch.last_error != False %}
|
||||
<div class="fetch-error">{{ watch.last_error }}</div>
|
||||
{% endif %}
|
||||
{% if not active_tag %}
|
||||
<span class="watch-tag-list">{{ watch.tag}}</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>{{watch|format_last_checked_time}}</td>
|
||||
<td>{% if watch.history|length >= 2 and watch.last_changed %}
|
||||
{{watch.last_changed|format_timestamp_timeago}}
|
||||
{% else %}
|
||||
Not yet
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
<a href="/api/checknow?uuid={{ watch.uuid}}{% if request.args.get('tag') %}&tag={{request.args.get('tag')}}{% endif %}"
|
||||
class="pure-button button-small pure-button-primary">Recheck</a>
|
||||
<a href="/edit/{{ watch.uuid}}" class="pure-button button-small pure-button-primary">Edit</a>
|
||||
{% if watch.history|length >= 2 %}
|
||||
<a href="/diff/{{ watch.uuid}}" target="{{watch.uuid}}" class="pure-button button-small pure-button-primary">Diff</a>
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
<ul id="post-list-buttons">
|
||||
{% if has_unviewed %}
|
||||
<li>
|
||||
<a href="/api/mark-all-viewed" class="pure-button button-tag ">Mark all viewed</a>
|
||||
</li>
|
||||
{% endif %}
|
||||
<li>
|
||||
<a href="/api/checknow{% if active_tag%}?tag={{active_tag}}{%endif%}" class="pure-button button-tag ">Recheck
|
||||
all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
|
||||
</li>
|
||||
<li>
|
||||
<a href="{{ url_for('index', tag=active_tag , rss=true)}}"><img id="feed-icon" src="/static/images/Generic_Feed-icon.svg" height="15px"></a>
|
||||
</li>
|
||||
</ul>
|
||||
{% endblock %}
|
||||
</div>
|
||||
</div>
|
||||
{% endblock %}
|
||||
64
backend/templates/watch-table.html
Normal file
64
backend/templates/watch-table.html
Normal file
@@ -0,0 +1,64 @@
|
||||
{% extends 'watch-overview.html' %}
|
||||
{% block innercontent %}
|
||||
<table class="pure-table pure-table-striped watch-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>#</th>
|
||||
<th></th>
|
||||
<th>Last Checked</th>
|
||||
<th>Last Changed</th>
|
||||
<th></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
|
||||
|
||||
{% for watch in watches %}
|
||||
<tr id="{{ watch.uuid }}"
|
||||
class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}
|
||||
{% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
|
||||
{% if watch.newest_history_key| int > watch.last_viewed| int %}unviewed{% endif %}">
|
||||
<td>{{ loop.index }}</td>
|
||||
<td class="title-col">{{watch.title if watch.title is not none else watch.url}}
|
||||
<a class="external" target=_blank href="{{ watch.url }}"></a>
|
||||
{% if watch.last_error is defined and watch.last_error != False %}
|
||||
<div class="fetch-error">{{ watch.last_error }}</div>
|
||||
{% endif %}
|
||||
{% if not active_tag %}
|
||||
<span class="watch-tag-list">{{ watch.tag}}</span>
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>{{watch|format_last_checked_time}}</td>
|
||||
<td>{% if watch.history|length >= 2 and watch.last_changed %}
|
||||
{{watch.last_changed|format_timestamp_timeago}}
|
||||
{% else %}
|
||||
Not yet
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
<a href="/api/checknow?uuid={{ watch.uuid}}{% if request.args.get('tag') %}&tag={{request.args.get('tag')}}{% endif %}"
|
||||
class="pure-button button-small pure-button-primary">Recheck</a>
|
||||
<a href="/edit/{{ watch.uuid}}" class="pure-button button-small pure-button-primary">Edit</a>
|
||||
{% if watch.history|length >= 2 %}
|
||||
<a href="/diff/{{ watch.uuid}}" class="pure-button button-small pure-button-primary">Diff</a>
|
||||
{% endif %}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
<ul id="post-list-buttons">
|
||||
{% if has_unviewed %}
|
||||
<li>
|
||||
<a href="/api/mark-all-viewed" class="pure-button button-tag ">Mark all viewed</a>
|
||||
</li>
|
||||
{% endif %}
|
||||
<li>
|
||||
<a href="/api/checknow{% if active_tag%}?tag={{active_tag}}{%endif%}" class="pure-button button-tag ">Recheck
|
||||
all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
|
||||
</li>
|
||||
<li>
|
||||
<a href="{{ url_for('index', tag=active_tag , rss=true)}}"><img id="feed-icon" src="/static/images/Generic_Feed-icon.svg" height="15px"></a>
|
||||
</li>
|
||||
</ul>
|
||||
{% endblock %}
|
||||
Reference in New Issue
Block a user