add retries on exceptions

Author: mredolatti

splitio/push/manager.py

@@ -66,7 +66,6 @@ def update_workers_status(self, enabled):
         """
         self._processor.update_workers_status(enabled)
 
-
     def start(self):
         """Start a new connection if not already running."""
         if self._running:

splitio/sync/synchronizer.py

@@ -235,29 +235,51 @@ def synchronize_segment(self, segment_name, till):
         :type till: int
         """
         _LOGGER.debug('Synchronizing segment %s', segment_name)
-        return self._split_synchronizers.segment_sync.synchronize_segment(segment_name, till)
+        success = self._split_synchronizers.segment_sync.synchronize_segment(segment_name, till)
+        if not success:
+            _LOGGER.error('Failed to sync some segments.')
+        return success
 
     def synchronize_splits(self, till):
         """
         Synchronize all splits.
 
         :param till: to fetch
         :type till: int
+
+        :returns: whether the synchronization was successful or not.
+        :rtype: bool
         """
         _LOGGER.debug('Starting splits synchronization')
-        return self._split_synchronizers.split_sync.synchronize_splits(till)
-
-    def sync_all(self):
-        """Synchronize all split data."""
         try:
-            self.synchronize_splits(None)
+            self._split_synchronizers.split_sync.synchronize_splits(till)
+            return True
         except APIException:
             _LOGGER.error('Failed syncing splits')
             _LOGGER.debug('Error: ', exc_info=True)
+            return False
 
-        if not self._synchronize_segments():
-            _LOGGER.error('Failed syncing segments')
-            _LOGGER.debug('Error: ', exc_info=True)
+    def sync_all(self):
+        """Synchronize all split data."""
+        attempts = 3
+        while attempts > 0:
+            try:
+                if not self.synchronize_splits(None):
+                    attempts -= 1
+                    continue
+
+                if not self._synchronize_segments():
+                    attempts -= 1
+                    continue
+
+                # All is good
+                return
+            except Exception as exc:  # pylint:disable=broad-except
+                attempts -= 1
+                _LOGGER.error("Exception caught when trying to sync all data: %s", str(exc))
+                _LOGGER.debug('Error: ', exc_info=True)
+
+        _LOGGER.error("Could not correctly synchronize splits and segments after 3 attempts.")
 
     def shutdown(self, blocking):
         """

tests/sync/test_synchronizer.py

@@ -33,8 +33,7 @@ def run(x):
                                                  mocker.Mock(), mocker.Mock(), mocker.Mock())
         sychronizer = Synchronizer(split_synchronizers, mocker.Mock(spec=SplitTasks))
 
-        with pytest.raises(APIException):
-            sychronizer.synchronize_splits(None)
+        sychronizer.synchronize_splits(None)  # APIExceptions are handled locally and should not be propagated!
 
         sychronizer.sync_all()  # sync_all should not throw!
 
@@ -209,3 +208,59 @@ def stop_mock_2():
         assert len(impression_count_task.stop.mock_calls) == 1
         assert len(event_task.stop.mock_calls) == 1
         assert len(telemetry_task.stop.mock_calls) == 1
+
+    def test_sync_all_ok(self, mocker):
+        """Test that 3 attempts are done before failing."""
+        split_synchronizers = mocker.Mock(spec=SplitSynchronizers)
+        counts = {'splits': 0, 'segments': 0}
+
+        def sync_splits(*_):
+            """Sync Splits."""
+            counts['splits'] += 1
+            return True
+
+        def sync_segments(*_):
+            """Sync Segments."""
+            counts['segments'] += 1
+            return True
+
+        split_synchronizers.split_sync.synchronize_splits.side_effect = sync_splits
+        split_synchronizers.segment_sync.synchronize_segments.side_effect = sync_segments
+        split_tasks = mocker.Mock(spec=SplitTasks)
+        synchronizer = Synchronizer(split_synchronizers, split_tasks)
+
+        synchronizer.sync_all()
+        assert counts['splits'] == 1
+        assert counts['segments'] == 1
+
+    def test_sync_all_attempts(self, mocker):
+        """Test that 3 attempts are done before failing."""
+        split_synchronizers = mocker.Mock(spec=SplitSynchronizers)
+        counts = {'splits': 0, 'segments': 0}
+        def sync_splits(*_):
+            """Sync Splits."""
+            counts['splits'] += 1
+            raise Exception('sarasa')
+
+        split_synchronizers.split_sync.synchronize_splits.side_effect = sync_splits
+        split_tasks = mocker.Mock(spec=SplitTasks)
+        synchronizer = Synchronizer(split_synchronizers, split_tasks)
+
+        synchronizer.sync_all()
+        assert counts['splits'] == 3
+
+    def test_sync_all_attempts(self, mocker):
+        """Test that 3 attempts are done before failing."""
+        split_synchronizers = mocker.Mock(spec=SplitSynchronizers)
+        counts = {'splits': 0, 'segments': 0}
+        def sync_segments(*_):
+            """Sync Splits."""
+            counts['segments'] += 1
+            raise Exception('sarasa')
+
+        split_synchronizers.segment_sync.synchronize_segments.side_effect = sync_segments
+        split_tasks = mocker.Mock(spec=SplitTasks)
+        synchronizer = Synchronizer(split_synchronizers, split_tasks)
+
+        synchronizer.sync_all()
+        assert counts['segments'] == 3