send an email when updating the SpaceAI consistently fails

diff --git a/spaceapi.py b/spaceapi.py
index 8e36b0fe21a3d372e0b8e03f443f80baf60365c8..7f788ab4b32aee8897b48f60e2f944139c28069f 100644 (file)
--- a/spaceapi.py
+++ b/spaceapi.py
@@ -13,6 +13,7 @@ class SpaceApi:
                self._remote_state = None
                self._last_set_at = 0
                self._running = True
+               self._fail_count = 0 # number of consecutive fails
                self.set_state = ThreadFunction(self._set_state, "Space API")
                waker.register(self.set_state, RETRY_TIME)
        
@@ -44,7 +45,11 @@ class SpaceApi:
                if self._local_state is not None and (self._local_state != self._remote_state or now > self._last_set_at+HEARTBEAT_TIME):
                        # take action!
                        success = self._do_request(self._local_state)
-                       # TODO error too often -> log critical to send mails
                        if success:
                                self._remote_state = self._local_state
                                self._last_set_at = now
+                               self._fail_count = 0
+                       else:
+                               self._fail_count += 1
+                               if self._fail_count in (5, 50, 500):
+                                       logger.critical("Updating the SpaceAPI failed %d times in a row" % self._fail_count)

diff --git a/tuerd b/tuerd
index c0b4413957d4af20c9543106dcd73de7253bdc6a..942c495267403d957e1e01ee579a3d88414e4a47 100755 (executable)
--- a/tuerd
+++ b/tuerd
@@ -43,7 +43,7 @@ except KeyboardInterrupt:
        # this is what we waited for!
        pass
 
-logger.info("Terminating...")
+logger.info("Terminating...") # somehow this does not arrive in the syslog
 
 # bring 'em all down
 the_waker.stop() # this one first, it "randomly" calls other threads