Merge pull request #440 from splitio/development

[8.1.0] Development into master

Author: sanzmauro

CHANGES.txt

@@ -1,5 +1,8 @@
 CHANGES
 
+8.1.0 (Oct 5, 2022)
+- Added a new impressions mode for the SDK called NONE , to be used in factory when there is no desire to capture impressions on an SDK factory to feed Split's analytics engine. Running NONE mode, the SDK will only capture unique keys evaluated for a particular feature flag instead of full blown impressions.
+
 8.0.1 (Jul 20, 2022)
 - Updated dependencies to support faraday > 2.0
 
@@ -49,9 +52,10 @@ CHANGES
 - Updated ably error handling. 
 
 7.2.0 (Sep 25, 2020)
-- Added deduplication logic for impressions data.
-  - Now there are two modes for Impressions when the SDK is in standalone mode, OPTIMIZED (default) that only ships unique impressions and DEBUG for times where you need to send ALL impressions to debug an integration.
-  - Impression listener remains unchanged and will still get all impressions.
+- Added impressions dedupe logic to avoid sending duplicated impressions:  
+  - Added `OPTIMIZED` and `DEBUG` modes in order to enabling/disabling how impressions are going to be sent into Split servers,
+    - `OPTIMIZED`: will send unique impressions in a timeframe in order to reduce how many times impressions are posted to Split.
+    - `DEBUG`: will send every impression generated to Split.
 
 7.1.3 (Jul 31, 2020)
 - Updated rake development dependency to ~> 12.3.3.

lib/splitclient-rb/cache/senders/impressions_adapter/redis_sender.rb

@@ -12,7 +12,9 @@ def initialize(config)
         end
 
         def record_uniques_key(uniques)
-          formatted = uniques_formatter(uniques)
+          return if uniques.nil? || uniques == {}
+
+          formatted = uniques_formatter(uniques).to_json
 
           unless formatted.nil?
             size = @adapter.add_to_queue(unique_keys_key, formatted)

lib/splitclient-rb/engine/common/impressions_manager.rb

@@ -30,7 +30,8 @@ def build_impression(matching_key, bucketing_key, split_name, treatment, params
               @unique_keys_tracker.track(split_name, matching_key)
             else # In OPTIMIZED mode we should track the total amount of evaluations and deduplicate the impressions.
               impression_data[:pt] = @impression_observer.test_and_set(impression_data)
-              @impression_counter.inc(split_name, impression_data[:m])
+
+              @impression_counter.inc(split_name, impression_data[:m]) unless impression_data[:pt].nil?
             end
           rescue StandardError => e
             @config.log_found_exception(__method__.to_s, e)

lib/splitclient-rb/split_config.rb

@@ -310,23 +310,18 @@ def self.default_on_demand_fetch_max_retries
       10
     end
 
-    def self.default_impressions_mode
-      :optimized
-    end
-
     def init_impressions_mode(impressions_mode, adapter)
-      impressions_mode ||= SplitConfig.default_impressions_mode
-
-      return :debug if adapter == :redis
-
       case impressions_mode
+      when :optimized
+        return :optimized
+      when :none
+       return :none
       when :debug
         return :debug
-      # when :none  // we not support :none impression mode yet. Defaulting to :optimized mode
-      #  return :none
       else
-        @logger.error('You passed an invalid impressions_mode, impressions_mode should be one of the following values: :debug or :optimized. Defaulting to :optimized mode') unless impressions_mode == :optimized
-        return :optimized
+        default = adapter == :redis ? :debug : :optimized
+        @logger.error("You passed an invalid impressions_mode, impressions_mode should be one of the following values: :debug, :optimized or :none. Defaulting to #{default} mode")
+        return default
       end
     end
 
@@ -455,7 +450,7 @@ def self.default_connection_timeout
     end
 
     def self.default_features_refresh_rate
-      5
+      60
     end
 
     def self.default_segments_refresh_rate

lib/splitclient-rb/version.rb

@@ -1,3 +1,3 @@
 module SplitIoClient
-  VERSION = '8.0.1'
+  VERSION = '8.1.0'
 end

spec/cache/senders/impressions_sender_adapter_spec.rb

@@ -22,10 +22,18 @@
       sender.record_uniques_key(uniques)
 
       result = config.cache_adapter.get_from_queue(unique_keys_key, 0)
-      expect(result.size).to eq(3)
-      expect(result[0]).to eq('{:f=>"feature-name-1", :k=>["key-1", "key-2", "key-3", "key-4"]}')
-      expect(result[1]).to eq('{:f=>"feature-name-2", :k=>["key-1", "key-2", "key-3", "key-4"]}')
-      expect(result[2]).to eq('{:f=>"feature-name-3", :k=>["key-1", "key-2", "key-3", "key-4"]}')
+
+      expect(result.size).to eq(1)
+      data = JSON.parse(result[0], symbolize_names: true)
+
+      expect(data[0][:f]).to eq('feature-name-1')
+      expect(data[0][:k].to_s).to eq('["key-1", "key-2", "key-3", "key-4"]')
+
+      expect(data[1][:f]).to eq('feature-name-2')
+      expect(data[1][:k].to_s).to eq('["key-1", "key-2", "key-3", "key-4"]')
+
+      expect(data[2][:f]).to eq('feature-name-3')
+      expect(data[2][:k].to_s).to eq('["key-1", "key-2", "key-3", "key-4"]')
     end
 
     it 'record_uniques_key when uniques is nil or empty' do

spec/engine/impressions/redis_unique_keys_tracker_spec.rb

@@ -35,7 +35,13 @@
       expect(tracker.track("feature-test-#{i}", 'key_test-2')).to eq(true)
     end
 
-    expect(config.cache_adapter.get_from_queue(key, 0).size).to eq(20)
+    result = config.cache_adapter.get_from_queue(key, 0)
+    expect(result.size).to eq(10)
+
+    10.times do |i|
+      data = JSON.parse(result[i], symbolize_names: true)
+      expect(data.size).to eq(2)
+    end
 
     cache.clear
   end
@@ -56,7 +62,13 @@
 
     sleep 1
 
-    expect(config.cache_adapter.get_from_queue(key, 0).size).to eq(10)
+    result = config.cache_adapter.get_from_queue(key, 0)
+    expect(result.size).to eq(1)
+
+    10.times do |i|
+      data = JSON.parse(result[0], symbolize_names: true)
+      expect(data.size).to eq(10)
+    end
 
     cache.clear
   end

spec/integrations/dedupe_impression_spec.rb

@@ -98,8 +98,8 @@
       .with(
         body: {
           pf: [
-            { f: 'FACUNDO_TEST', m: time_frame, rc: 3 },
-            { f: 'Test_Save_1', m: time_frame, rc: 2 }
+            { f: 'FACUNDO_TEST', m: time_frame, rc: 1 },
+            { f: 'Test_Save_1', m: time_frame, rc: 1 }
           ]
         }.to_json
       )).to have_been_made
@@ -120,6 +120,7 @@
       client.get_treatments('admin', %w[FACUNDO_TEST MAURO_TEST Test_Save_1])
       client.get_treatments('maldo', %w[FACUNDO_TEST Test_Save_1])
       client.get_treatments('nico_test', %w[FACUNDO_TEST MAURO_TEST Test_Save_1])
+      client.get_treatments('nico_test', %w[FACUNDO_TEST MAURO_TEST Test_Save_1])
 
       time_frame = SplitIoClient::Engine::Common::ImpressionCounter.truncate_time_frame((Time.now.to_f * 1000.0).to_i)
 
@@ -130,9 +131,9 @@
       .with(
         body: {
           pf: [
-            { f: 'FACUNDO_TEST', m: time_frame, rc: 4 },
-            { f: 'MAURO_TEST', m: time_frame, rc: 3 },
-            { f: 'Test_Save_1', m: time_frame, rc: 4 }
+            { f: 'FACUNDO_TEST', m: time_frame, rc: 2 },
+            { f: 'MAURO_TEST', m: time_frame, rc: 2 },
+            { f: 'Test_Save_1', m: time_frame, rc: 2 }
           ]
         }.to_json
       )).to have_been_made

spec/splitclient/split_config_spec.rb

@@ -54,7 +54,7 @@
 
       expect(configs.connection_timeout).to eq 5
       expect(configs.read_timeout).to eq 5
-      expect(configs.features_refresh_rate).to eq 5
+      expect(configs.features_refresh_rate).to eq 60
       expect(configs.segments_refresh_rate).to eq 60
       expect(configs.impressions_refresh_rate).to eq 300
       expect(configs.impressions_queue_size).to eq 5000
@@ -65,7 +65,7 @@
 
       expect(configs.connection_timeout).to eq 5
       expect(configs.read_timeout).to eq 5
-      expect(configs.features_refresh_rate).to eq 5
+      expect(configs.features_refresh_rate).to eq 60
       expect(configs.segments_refresh_rate).to eq 60
       expect(configs.impressions_refresh_rate).to eq 60
       expect(configs.impressions_queue_size).to eq 5000
@@ -113,6 +113,16 @@
       configs3 = SplitIoClient::SplitConfig.new(options3)
 
       expect(configs3.impressions_mode).to eq(:optimized)
+
+      options4 = { impressions_mode: :sarasa, cache_adapter: :redis }
+      configs4 = SplitIoClient::SplitConfig.new(options4)
+
+      expect(configs4.impressions_mode).to eq(:debug)
+
+      options5 = { impressions_mode: :optimized, cache_adapter: :redis }
+      configs5 = SplitIoClient::SplitConfig.new(options5)
+
+      expect(configs5.impressions_mode).to eq(:optimized)
     end
 
     it 'set impressions refresh rate with impressions optimized mode' do

spec/telemetry/synchronizer_spec.rb

@@ -45,8 +45,8 @@
     let(:body_usage) { "{\"lS\":{\"sp\":111111222,\"se\":111111222,\"im\":111111222,\"ic\":111111222,\"ev\":111111222,\"te\":111111222,\"to\":111111222},\"mL\":{\"t\":[0,2,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"ts\":[0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"tc\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"tcs\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"tr\":[0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},\"mE\":{\"t\":2,\"ts\":1,\"tc\":1,\"tcs\":0,\"tr\":1},\"hE\":{\"sp\":{},\"se\":{\"400\":1},\"im\":{},\"ic\":{},\"ev\":{\"500\":2,\"501\":1},\"te\":{},\"to\":{}},\"hL\":{\"sp\":[0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"se\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"im\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"ic\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"ev\":[0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"te\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"to\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},\"tR\":1,\"aR\":1,\"iQ\":3,\"iDe\":1,\"iDr\":2,\"spC\":3,\"seC\":3,\"skC\":7,\"sL\":444555,\"eQ\":4,\"eD\":1,\"sE\":[{\"e\":50,\"d\":222222333,\"t\":222222333},{\"e\":70,\"d\":0,\"t\":222222333},{\"e\":70,\"d\":1,\"t\":222222333}],\"t\":[\"tag-1\",\"tag-2\"]}" }
     let(:empty_body_usage) { "{\"lS\":{\"sp\":0,\"se\":0,\"im\":0,\"ic\":0,\"ev\":0,\"te\":0,\"to\":0},\"mL\":{\"t\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"ts\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"tc\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"tcs\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"tr\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},\"mE\":{\"t\":0,\"ts\":0,\"tc\":0,\"tcs\":0,\"tr\":0},\"hE\":{\"sp\":{},\"se\":{},\"im\":{},\"ic\":{},\"ev\":{},\"te\":{},\"to\":{}},\"hL\":{\"sp\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"se\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"im\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"ic\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"ev\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"te\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],\"to\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},\"tR\":0,\"aR\":0,\"iQ\":0,\"iDe\":0,\"iDr\":0,\"spC\":0,\"seC\":0,\"skC\":0,\"sL\":0,\"eQ\":0,\"eD\":0,\"sE\":[],\"t\":[]}" }    
     let(:body_custom_config)  { "{\"oM\":0,\"sE\":true,\"st\":\"memory\",\"rR\":{\"sp\":100,\"se\":110,\"im\":120,\"ev\":130,\"te\":140},\"iQ\":5000,\"eQ\":500,\"iM\":0,\"uO\":{\"s\":true,\"e\":true,\"a\":true,\"st\":false,\"t\":false},\"iL\":false,\"hP\":false,\"aF\":1,\"rF\":1,\"tR\":100,\"bT\":2,\"nR\":1,\"t\":[],\"i\":null}" }
-    let(:body_default_config) { "{\"oM\":0,\"sE\":true,\"st\":\"memory\",\"rR\":{\"sp\":5,\"se\":60,\"im\":300,\"ev\":60,\"te\":3600},\"iQ\":5000,\"eQ\":500,\"iM\":0,\"uO\":{\"s\":false,\"e\":false,\"a\":false,\"st\":false,\"t\":false},\"iL\":false,\"hP\":false,\"aF\":1,\"rF\":1,\"tR\":500,\"bT\":0,\"nR\":0,\"t\":[],\"i\":null}" }
-    let(:body_proxy_config)   { "{\"oM\":0,\"sE\":true,\"st\":\"memory\",\"rR\":{\"sp\":5,\"se\":60,\"im\":300,\"ev\":60,\"te\":3600},\"iQ\":5000,\"eQ\":500,\"iM\":0,\"uO\":{\"s\":false,\"e\":false,\"a\":false,\"st\":false,\"t\":false},\"iL\":false,\"hP\":true,\"aF\":1,\"rF\":1,\"tR\":500,\"bT\":0,\"nR\":0,\"t\":[],\"i\":null}" }
+    let(:body_default_config) { "{\"oM\":0,\"sE\":true,\"st\":\"memory\",\"rR\":{\"sp\":60,\"se\":60,\"im\":300,\"ev\":60,\"te\":3600},\"iQ\":5000,\"eQ\":500,\"iM\":0,\"uO\":{\"s\":false,\"e\":false,\"a\":false,\"st\":false,\"t\":false},\"iL\":false,\"hP\":false,\"aF\":1,\"rF\":1,\"tR\":500,\"bT\":0,\"nR\":0,\"t\":[],\"i\":null}" }
+    let(:body_proxy_config)   { "{\"oM\":0,\"sE\":true,\"st\":\"memory\",\"rR\":{\"sp\":60,\"se\":60,\"im\":300,\"ev\":60,\"te\":3600},\"iQ\":5000,\"eQ\":500,\"iM\":0,\"uO\":{\"s\":false,\"e\":false,\"a\":false,\"st\":false,\"t\":false},\"iL\":false,\"hP\":true,\"aF\":1,\"rF\":1,\"tR\":500,\"bT\":0,\"nR\":0,\"t\":[],\"i\":null}" }
 
     context 'synchronize_stats' do
       before do