changedetection.io/changedetectionio/tests/unit/test_thread_supervisor.py

#!/usr/bin/env python3

"""
Tests for thread_supervisor.supervise().

Context: the ticker thread had `return False` buried inside its per-watch loop.
One watch with an unresolvable timezone made the whole loop return, the thread
ended, and no watch was ever checked again - with the process still up and
looking healthy. threading.Thread cannot be restarted (start() twice raises
RuntimeError), so recovery has to live inside the thread.

The two behaviours that matter:
  - a target that returns or raises is restarted, loudly
  - a target that exits during shutdown is NOT restarted

Run:  python3 -m unittest changedetectionio.tests.unit.test_thread_supervisor
"""

import threading
import time
import unittest

from changedetectionio.thread_supervisor import supervise, start_supervised_thread


class TestRestartsOnDeath(unittest.TestCase):

    def test_target_that_returns_is_restarted(self):
        """The exact ticker bug: a stray `return` must not end the thread."""
        exit_event = threading.Event()
        calls = []

        def flaky():
            calls.append('run')
            if len(calls) >= 3:
                exit_event.set()      # let the supervisor finish
            return                     # simulates `return False` in the loop

        supervise(flaky, 'flaky', exit_event, max_backoff=0.01, healthy_after=9999)
        self.assertEqual(len(calls), 3, "target should have been restarted twice")

    def test_target_that_raises_is_restarted(self):
        exit_event = threading.Event()
        calls = []

        def crashy():
            calls.append('run')
            if len(calls) >= 3:
                exit_event.set()
                return
            raise RuntimeError("boom")

        supervise(crashy, 'crashy', exit_event, max_backoff=0.01, healthy_after=9999)
        self.assertEqual(len(calls), 3)

    def test_restart_reason_is_reported(self):
        exit_event = threading.Event()
        reasons = []
        calls = []

        def mixed():
            calls.append('run')
            if len(calls) == 1:
                return                        # clean but wrong
            if len(calls) == 2:
                raise ValueError("nope")      # crash
            exit_event.set()

        supervise(mixed, 'mixed', exit_event, max_backoff=0.01, healthy_after=9999,
                  _on_restart=lambda n, reason: reasons.append(reason))

        self.assertEqual(len(reasons), 2)
        self.assertIn('returned unexpectedly', reasons[0])
        self.assertIn('ValueError: nope', reasons[1])

    def test_backoff_grows_then_resets_after_healthy_run(self):
        exit_event = threading.Event()
        stamps = []

        def failing():
            stamps.append(time.monotonic())
            if len(stamps) >= 4:
                exit_event.set()
                return
            raise RuntimeError("boom")

        supervise(failing, 'failing', exit_event, max_backoff=0.08, healthy_after=9999)
        gaps = [stamps[i + 1] - stamps[i] for i in range(len(stamps) - 1)]
        # 1s doubling is capped at max_backoff, so every gap should hit the cap
        for gap in gaps:
            self.assertGreaterEqual(gap, 0.05, f"gaps={gaps} - backoff not applied")

    def test_does_not_spin_hot(self):
        """A target failing instantly must not be restarted in a tight loop."""
        exit_event = threading.Event()
        calls = []
        start = time.monotonic()

        def instant():
            calls.append(1)
            if len(calls) >= 3:
                exit_event.set()
                return
            raise RuntimeError("instant")

        supervise(instant, 'instant', exit_event, max_backoff=0.05, healthy_after=9999)
        self.assertGreater(time.monotonic() - start, 0.05,
                           "supervisor restarted with no delay - would burn CPU")


class TestDoesNotRestartDuringShutdown(unittest.TestCase):

    def test_exit_event_stops_restarting(self):
        exit_event = threading.Event()
        calls = []

        def target():
            calls.append('run')
            exit_event.set()          # shutdown requested, then return normally

        supervise(target, 'target', exit_event, max_backoff=0.01)
        self.assertEqual(len(calls), 1, "must not restart once exit_event is set")

    def test_exit_event_set_before_start_never_runs_target(self):
        exit_event = threading.Event()
        exit_event.set()
        calls = []
        supervise(lambda: calls.append('run'), 'target', exit_event)
        self.assertEqual(calls, [])

    def test_secondary_shutdown_flag_stops_restarting(self):
        """sigshutdown_handler also sets datastore.stop_thread - honour it."""
        exit_event = threading.Event()
        state = {'stop_thread': False}
        calls = []

        def target():
            calls.append('run')
            state['stop_thread'] = True    # only the secondary flag is set
            return

        supervise(target, 'target', exit_event,
                  is_shutting_down=lambda: state['stop_thread'],
                  max_backoff=0.01)
        self.assertEqual(len(calls), 1,
                         "must not restart when the secondary shutdown flag is set")
        self.assertFalse(exit_event.is_set(), "primary flag was never set in this test")

    def test_raising_shutdown_predicate_does_not_block_restart(self):
        """A broken predicate must not silently disable recovery."""
        exit_event = threading.Event()
        calls = []

        def target():
            calls.append('run')
            if len(calls) >= 2:
                exit_event.set()
            return

        def broken():
            raise RuntimeError("predicate is broken")

        supervise(target, 'target', exit_event, is_shutting_down=broken,
                  max_backoff=0.01, healthy_after=9999)
        self.assertEqual(len(calls), 2, "should still have restarted once")


class TestStartSupervisedThread(unittest.TestCase):

    def test_returns_a_real_thread_object(self):
        """Thread(...).start() returns None - that is why ticker_thread was
        always None and nothing could check is_alive()."""
        exit_event = threading.Event()
        started = threading.Event()

        def target():
            started.set()
            exit_event.wait(5)

        t = start_supervised_thread(target, 'TestThread', exit_event)
        self.addCleanup(exit_event.set)

        self.assertIsInstance(t, threading.Thread)
        self.assertTrue(started.wait(5), "target did not run")
        self.assertTrue(t.is_alive())
        self.assertTrue(t.daemon)
        self.assertEqual(t.name, 'TestThread')

        exit_event.set()
        t.join(timeout=5)
        self.assertFalse(t.is_alive(), "thread did not exit on shutdown")

    def test_thread_survives_a_dying_target(self):
        """End to end: the thread outlives a target that keeps returning."""
        exit_event = threading.Event()
        calls = []
        done = threading.Event()

        def target():
            calls.append(1)
            if len(calls) >= 3:
                done.set()
                exit_event.wait(5)
            return

        t = start_supervised_thread(target, 'Dying', exit_event, max_backoff=0.01,
                                    healthy_after=9999)
        self.addCleanup(exit_event.set)

        self.assertTrue(done.wait(10), f"target was not restarted (calls={len(calls)})")
        self.assertTrue(t.is_alive(), "thread died despite supervision")
        exit_event.set()
        t.join(timeout=5)


if __name__ == '__main__':
    unittest.main()