Merge pull request #449 from splitio/sse-retries

Retry with backoff if get an HTTP error opening the SSE connection

Author: sanzmauro

lib/splitclient-rb/engine/auth_api_client.rb

@@ -59,11 +59,7 @@ def process_error(response)
 
       def process_success(response, start)
         @config.logger.debug("Success connection to: #{@config.auth_service_url}") if @config.debug_enabled
-
-        bucket = BinarySearchLatencyTracker.get_bucket((Time.now - start) * 1000.0)
-        @telemetry_runtime_producer.record_sync_latency(Telemetry::Domain::Constants::TOKEN_SYNC, bucket)
-        timestamp = (Time.now.to_f * 1000.0).to_i
-        @telemetry_runtime_producer.record_successful_sync(Telemetry::Domain::Constants::TOKEN_SYNC, timestamp)
+        record_telemetry(start)
 
         body_json = JSON.parse(response.body, symbolize_names: true)
         push_enabled = body_json[:pushEnabled]
@@ -77,7 +73,7 @@ def process_success(response, start)
           @telemetry_runtime_producer.record_token_refreshes
         end
 
-        { push_enabled: push_enabled, token: token, channels: channels, exp: exp, retry: false }
+        { push_enabled: push_enabled, token: token, channels: channels, exp: exp, retry: true }
       end
 
       def control_channels(channels_string)
@@ -88,6 +84,13 @@ def control_channels(channels_string)
 
         channels_string.gsub(control_sec, "#{prefix}#{control_sec}")
       end
+
+      def record_telemetry(start)
+        bucket = BinarySearchLatencyTracker.get_bucket((Time.now - start) * 1000.0)
+        @telemetry_runtime_producer.record_sync_latency(Telemetry::Domain::Constants::TOKEN_SYNC, bucket)
+        timestamp = (Time.now.to_f * 1000.0).to_i
+        @telemetry_runtime_producer.record_successful_sync(Telemetry::Domain::Constants::TOKEN_SYNC, timestamp)
+      end
     end
   end
 end

lib/splitclient-rb/engine/back_off.rb

@@ -2,7 +2,7 @@
 
 module SplitIoClient
   module Engine
-    BACKOFF_MAX_ALLOWED = 1.8
+    BACKOFF_MAX_ALLOWED = 1800
     class BackOff
       def initialize(back_off_base, attempt = 0, max_allowed = BACKOFF_MAX_ALLOWED)
         @attempt = attempt

lib/splitclient-rb/engine/push_manager.rb

@@ -14,7 +14,6 @@ def initialize(config, sse_handler, api_key, telemetry_runtime_producer)
 
       def start_sse
         response = @auth_api_client.authenticate(@api_key)
-
         @config.logger.debug("Auth service response push_enabled: #{response[:push_enabled]}") if @config.debug_enabled
 
         if response[:push_enabled] && @sse_handler.start(response[:token], response[:channels])
@@ -26,7 +25,6 @@ def start_sse
         end
 
         stop_sse
-
         schedule_next_token_refresh(@back_off.interval) if response[:retry]
         false
       rescue StandardError => e
@@ -35,7 +33,8 @@ def start_sse
 
       def stop_sse
         @sse_handler.stop
-        SplitIoClient::Helpers::ThreadHelper.stop(:schedule_next_token_refresh, @config)
+      rescue StandardError => e
+        @config.logger.error(e.inspect)
       end
 
       private

lib/splitclient-rb/engine/synchronizer.rb

@@ -6,8 +6,6 @@ class Synchronizer
       include SplitIoClient::Cache::Fetchers
       include SplitIoClient::Cache::Senders
 
-      ON_DEMAND_FETCH_BACKOFF_BASE_SECONDS = 10
-      ON_DEMAND_FETCH_BACKOFF_MAX_WAIT_SECONDS = 60
       ON_DEMAND_FETCH_BACKOFF_MAX_RETRIES = 10
 
       def initialize(
@@ -27,6 +25,9 @@ def initialize(
         @telemetry_synchronizer = params[:telemetry_synchronizer]
         @impressions_sender_adapter = params[:impressions_sender_adapter]
         @unique_keys_tracker = params[:unique_keys_tracker]
+
+        @splits_sync_backoff = Engine::BackOff.new(10, 0, 60)
+        @segments_sync_backoff = Engine::BackOff.new(10, 0, 60)
       end
 
       def sync_all(asynchronous = true)
@@ -140,7 +141,7 @@ def fetch_segment(name, target_change_number)
 
       def attempt_segment_sync(name, target_cn, fetch_options, max_retries, retry_delay_seconds, with_backoff)
         remaining_attempts = max_retries
-        backoff = Engine::BackOff.new(ON_DEMAND_FETCH_BACKOFF_BASE_SECONDS, 0, ON_DEMAND_FETCH_BACKOFF_MAX_WAIT_SECONDS) if with_backoff
+        @segments_sync_backoff.reset
 
         loop do
           remaining_attempts -= 1
@@ -150,14 +151,14 @@ def attempt_segment_sync(name, target_cn, fetch_options, max_retries, retry_dela
           return sync_result(true, remaining_attempts) if target_cn <= @segments_repository.get_change_number(name).to_i
           return sync_result(false, remaining_attempts) if remaining_attempts <= 0
 
-          delay = with_backoff ? backoff.interval : retry_delay_seconds
+          delay = with_backoff ? @segments_sync_backoff.interval : retry_delay_seconds
           sleep(delay)
         end
       end
 
       def attempt_splits_sync(target_cn, fetch_options, max_retries, retry_delay_seconds, with_backoff)
         remaining_attempts = max_retries
-        backoff = Engine::BackOff.new(ON_DEMAND_FETCH_BACKOFF_BASE_SECONDS, 0, ON_DEMAND_FETCH_BACKOFF_MAX_WAIT_SECONDS) if with_backoff
+        @splits_sync_backoff.reset
 
         loop do
           remaining_attempts -= 1
@@ -167,7 +168,7 @@ def attempt_splits_sync(target_cn, fetch_options, max_retries, retry_delay_secon
           return sync_result(true, remaining_attempts, result[:segment_names]) if target_cn <= @splits_repository.get_change_number
           return sync_result(false, remaining_attempts, result[:segment_names]) if remaining_attempts <= 0
 
-          delay = with_backoff ? backoff.interval : retry_delay_seconds
+          delay = with_backoff ? @splits_sync_backoff.interval : retry_delay_seconds
           sleep(delay)
         end
       end

spec/engine/auth_api_client_spec.rb

@@ -22,7 +22,7 @@
 
     expect(response[:push_enabled]).to eq(true)
     expect(response[:channels]).to eq('xxxx_xxxx_segments%2Cxxxx_xxxx_splits%2C%5B%3Foccupancy%3Dmetrics.publishers%5Dcontrol_pri%2C%5B%3Foccupancy%3Dmetrics.publishers%5Dcontrol_sec')
-    expect(response[:retry]).to eq(false)
+    expect(response[:retry]).to eq(true)
   end
 
   it 'auth server return 500' do

spec/engine/sync_manager_spec.rb

@@ -103,7 +103,7 @@
       sleep(2)
       expect(a_request(:get, 'https://sdk.split.io/api/splitChanges?since=-1')).to have_been_made.once
 
-      expect(config.threads.size).to eq(7)
+      expect(config.threads.size).to eq(8)
     end
   end