send an email when updating the SpaceAI consistently fails

[saartuer.git] / spaceapi.py

diff --git a/spaceapi.py b/spaceapi.py
index 8e36b0fe21a3d372e0b8e03f443f80baf60365c8..7f788ab4b32aee8897b48f60e2f944139c28069f 100644 (file)
--- a/spaceapi.py
+++ b/spaceapi.py
@@ -13,6 +13,7 @@ class SpaceApi:
                self._remote_state = None
                self._last_set_at = 0
                self._running = True
+               self._fail_count = 0 # number of consecutive fails
                self.set_state = ThreadFunction(self._set_state, "Space API")
                waker.register(self.set_state, RETRY_TIME)
        
@@ -44,7 +45,11 @@ class SpaceApi:
                if self._local_state is not None and (self._local_state != self._remote_state or now > self._last_set_at+HEARTBEAT_TIME):
                        # take action!
                        success = self._do_request(self._local_state)
-                       # TODO error too often -> log critical to send mails
                        if success:
                                self._remote_state = self._local_state
                                self._last_set_at = now
+                               self._fail_count = 0
+                       else:
+                               self._fail_count += 1
+                               if self._fail_count in (5, 50, 500):
+                                       logger.critical("Updating the SpaceAPI failed %d times in a row" % self._fail_count)