Bug fix/wait for hot backup (#389)

* workaround: give the cluster time to restore the hot backup

* implement waiting for cluster resillience after hotbackup restore

* Adapting the wait function

* Adapting the wait function

* Adapting the wait function

* Adapting the wait function

---------

Co-authored-by: Alex Petenchea <alex.petenchea@gmail.com>

Author: dothebart

tests/test_backup.py

@@ -11,10 +11,48 @@
     BackupGetError,
     BackupRestoreError,
     BackupUploadError,
+    ReplicationClusterInventoryError,
 )
 from tests.helpers import assert_raises
 
 
+def wait_for_cluster_resilient(sys_db):
+    collections_in_sync = False
+    max_attempts = 100
+
+    while not collections_in_sync and max_attempts > 0:
+        count_in_sync = 0
+        count_still_waiting = 0
+
+        try:
+            inventory = sys_db.replication.cluster_inventory(include_system=True)
+        except ReplicationClusterInventoryError:
+            print("Failed to get cluster inventory, retrying...")
+            time.sleep(1)
+            max_attempts -= 1
+            continue
+
+        collections_in_sync = True
+        for col in inventory["collections"]:
+            if not col["all_in_sync"]:
+                count_still_waiting += 1
+                collections_in_sync = False
+            else:
+                count_in_sync += 1
+
+        if not collections_in_sync:
+            if max_attempts % 50 == 0:
+                print(inventory)
+                print(f"In sync: {count_in_sync}")
+                print(f"Still not in sync: {count_still_waiting}")
+            time.sleep(1)
+
+        max_attempts -= 1
+
+    if not collections_in_sync:
+        raise Exception("Collections didn't come in sync!")
+
+
 def test_backup_management(sys_db, bad_db, cluster, skip_tests, db_version):
     if "enterprise" in skip_tests:
         pytest.skip("Only for ArangoDB enterprise edition")
@@ -112,8 +150,7 @@ def test_backup_management(sys_db, bad_db, cluster, skip_tests, db_version):
     result = sys_db.backup.restore(backup_id_foo)
     assert isinstance(result, dict)
 
-    # Wait for restore to complete
-    time.sleep(10)
+    wait_for_cluster_resilient(sys_db)
 
     # Test restore backup with bad database.
     with assert_raises(BackupRestoreError) as err: