IGNITE-14245 Fix infinite loop while trying to get affinity mapping on failed node (#30)

(cherry picked from commit 6f56a3b)

Co-authored-by: Ivan Dashchinskiy <ivandasch@gmail.com>

Author: isapego

.travis.yml

@@ -0,0 +1,52 @@
+#
+# Copyright 2021 GridGain Systems, Inc. and Contributors.
+#
+# Licensed under the GridGain Community Edition License (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.gridgain.com/products/software/community-edition/gridgain-community-edition-license
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+language: python
+sudo: required
+
+addons:
+  apt:
+    packages:
+      - openjdk-8-jdk
+
+env:
+  global:
+    - IGNITE_VERSION=2.9.1
+    - IGNITE_HOME=/opt/ignite
+
+before_install:
+  - curl -L https://apache-mirror.rbc.ru/pub/apache/ignite/${IGNITE_VERSION}/apache-ignite-slim-${IGNITE_VERSION}-bin.zip > ignite.zip
+  - unzip ignite.zip -d /opt
+  - mv /opt/apache-ignite-slim-${IGNITE_VERSION}-bin /opt/ignite
+  - mv /opt/ignite/libs/optional/ignite-log4j2 /opt/ignite/libs/
+
+jobs:
+  include:
+    - python: '3.6'
+      arch: amd64
+      env: TOXENV=py36
+    - python: '3.7'
+      arch: amd64
+      env: TOXENV=py37
+    - python: '3.8'
+      arch: amd64
+      env: TOXENV=py38
+    - python: '3.9'
+      arch: amd64
+      env: TOXENV=py39
+
+install: pip install tox
+script: tox

pygridgain/cache.py

@@ -264,6 +264,7 @@ def get_best_node(
                         break
                     except connection_errors:
                         # retry if connection failed
+                        conn = self._client.random_node
                         pass
                     except CacheError:
                         # server did not create mapping in time

tests/affinity/conftest.py

@@ -19,6 +19,10 @@
 from pygridgain.api import cache_create, cache_destroy
 from tests.util import start_ignite_gen
 
+# Sometimes on slow testing servers and unstable topology
+# default timeout is not enough for cache ops.
+CLIENT_SOCKET_TIMEOUT = 20.0
+
 
 @pytest.fixture(scope='module', autouse=True)
 def server1():
@@ -37,7 +41,7 @@ def server3():
 
 @pytest.fixture
 def client():
-    client = Client(partition_aware=True)
+    client = Client(partition_aware=True, timeout=CLIENT_SOCKET_TIMEOUT)
 
     client.connect([('127.0.0.1', 10800 + i) for i in range(1, 4)])
 
@@ -48,7 +52,7 @@ def client():
 
 @pytest.fixture
 def client_not_connected():
-    client = Client(partition_aware=True)
+    client = Client(partition_aware=True, timeout=CLIENT_SOCKET_TIMEOUT)
     yield client
     client.close()
 

tests/affinity/test_affinity_bad_servers.py

@@ -16,45 +16,61 @@
 import pytest
 
 from pygridgain.exceptions import ReconnectError
-from tests.util import start_ignite, kill_process_tree
+from tests.affinity.conftest import CLIENT_SOCKET_TIMEOUT
+from tests.util import start_ignite, kill_process_tree, get_client
 
+@pytest.fixture(params=['with-partition-awareness', 'without-partition-awareness'])
+def with_partition_awareness(request):
+    yield request.param == 'with-partition-awareness'
 
-def test_client_with_multiple_bad_servers(client_not_connected):
+
+def test_client_with_multiple_bad_servers(with_partition_awareness):
     with pytest.raises(ReconnectError) as e_info:
-        client_not_connected.connect([("127.0.0.1", 10900), ("127.0.0.1", 10901)])
+        with get_client(partition_aware=with_partition_awareness) as client:
+            client.connect([("127.0.0.1", 10900), ("127.0.0.1", 10901)])
     assert str(e_info.value) == "Can not connect."
 
 
-def test_client_with_failed_server(request, client_not_connected):
+def test_client_with_failed_server(request, with_partition_awareness):
     srv = start_ignite(idx=4)
     try:
-        client_not_connected.connect([("127.0.0.1", 10804)])
-        cache = client_not_connected.get_or_create_cache(request.node.name)
-        cache.put(1, 1)
-        kill_process_tree(srv.pid)
-        with pytest.raises(ConnectionResetError):
-            cache.get(1)
+        with get_client(partition_aware=with_partition_awareness) as client:
+            client.connect([("127.0.0.1", 10804)])
+            cache = client.get_or_create_cache(request.node.name)
+            cache.put(1, 1)
+            kill_process_tree(srv.pid)
+
+            if with_partition_awareness:
+                ex_class = (ReconnectError, ConnectionResetError)
+            else:
+                ex_class = ConnectionResetError
+
+            with pytest.raises(ex_class):
+                cache.get(1)
     finally:
         kill_process_tree(srv.pid)
 
 
-def test_client_with_recovered_server(request, client_not_connected):
+def test_client_with_recovered_server(request, with_partition_awareness):
     srv = start_ignite(idx=4)
     try:
-        client_not_connected.connect([("127.0.0.1", 10804)])
-        cache = client_not_connected.get_or_create_cache(request.node.name)
-        cache.put(1, 1)
+        with get_client(partition_aware=with_partition_awareness, timeout=CLIENT_SOCKET_TIMEOUT) as client:
+            client.connect([("127.0.0.1", 10804)])
+            cache = client.get_or_create_cache(request.node.name)
+            cache.put(1, 1)
 
-        # Kill and restart server
-        kill_process_tree(srv.pid)
-        srv = start_ignite(idx=4)
+            # Kill and restart server
+            kill_process_tree(srv.pid)
+            srv = start_ignite(idx=4)
 
-        # First request fails
-        with pytest.raises(Exception):
-            cache.put(1, 2)
+            # First request may fail.
+            try:
+                cache.put(1, 2)
+            except:
+                pass
 
-        # Retry succeeds
-        cache.put(1, 2)
-        assert cache.get(1) == 2
+            # Retry succeeds
+            cache.put(1, 2)
+            assert cache.get(1) == 2
     finally:
         kill_process_tree(srv.pid)