Added tests for binlog_server.

mtr/binlog_streaming/include/diff_with_storage_object.inc

@@ -33,7 +33,7 @@ if ($storage_backend == file)
 if ($storage_backend == s3)
 {
   --let $downloaded_file_path = $MYSQL_TMP_DIR/diff_with_storage_object.downloaded
-  --exec $aws_cli s3 cp s3://$aws_s3_bucket$storage_object $downloaded_file_path > /dev/null
+  --exec $aws_cli s3 cp s3://$aws_s3_bucket/$storage_object $downloaded_file_path > /dev/null
   --diff_files $local_file $downloaded_file_path
   --remove_file $downloaded_file_path
 }

mtr/binlog_streaming/include/set_up_binsrv_environment.inc

@@ -38,11 +38,11 @@ if ($storage_backend == file)
 if ($storage_backend == s3)
 {
   --let $binsrv_buffer_path = $MYSQL_TMP_DIR/buffer
-  --let $binsrv_storage_path = `SELECT CONCAT('/mtr-', UUID())`
+  --let $binsrv_storage_path = `SELECT CONCAT('mtr-', UUID())`
   --let $aws_s3_bucket = $MTR_BINSRV_AWS_S3_BUCKET
   if ($MTR_BINSRV_AWS_S3_ENDPOINT != '')
   {
-    eval SET @storage_uri = CONCAT('http://', '$MTR_BINSRV_AWS_ACCESS_KEY_ID', ':', '$MTR_BINSRV_AWS_SECRET_ACCESS_KEY', '@', '$MTR_BINSRV_AWS_S3_ENDPOINT', '/$MTR_BINSRV_AWS_S3_BUCKET', '$binsrv_storage_path');
+    eval SET @storage_uri = CONCAT('http://', '$MTR_BINSRV_AWS_ACCESS_KEY_ID', ':', '$MTR_BINSRV_AWS_SECRET_ACCESS_KEY', '@', '$MTR_BINSRV_AWS_S3_ENDPOINT', '/$MTR_BINSRV_AWS_S3_BUCKET', '/$binsrv_storage_path');
   }
   if ($MTR_BINSRV_AWS_S3_ENDPOINT == '')
   {
@@ -51,7 +51,7 @@ if ($storage_backend == s3)
     {
       --let $qualified_bucket = $qualified_bucket.$MTR_BINSRV_AWS_S3_REGION
     }
-    eval SET @storage_uri = CONCAT('s3://', '$MTR_BINSRV_AWS_ACCESS_KEY_ID', ':', '$MTR_BINSRV_AWS_SECRET_ACCESS_KEY', '@', '$qualified_bucket',  '$binsrv_storage_path');
+    eval SET @storage_uri = CONCAT('s3://', '$MTR_BINSRV_AWS_ACCESS_KEY_ID', ':', '$MTR_BINSRV_AWS_SECRET_ACCESS_KEY', '@', '$qualified_bucket',  '/$binsrv_storage_path');
   }
 }
 

mtr/binlog_streaming/include/tear_down_binsrv_environment.inc

@@ -6,7 +6,7 @@ if ($storage_backend == file)
 }
 if ($storage_backend == s3)
 {
-  --exec $aws_cli s3 rm s3://$aws_s3_bucket$binsrv_storage_path/ --recursive > /dev/null
+  --exec $aws_cli s3 rm s3://$aws_s3_bucket/$binsrv_storage_path/ --recursive > /dev/null
   --force-rmdir $binsrv_buffer_path
 }
 

mtr/binlog_streaming/include/v80_v84_compatibility_defines.inc

@@ -16,9 +16,13 @@ if ($lts_series == v80)
 {
   --let $stmt_reset_binary_logs_and_gtids = RESET MASTER
   --let $stmt_show_binary_log_status = SHOW MASTER STATUS
+  --let $init_rpl_inc = include/master-slave.inc
+  --let $deinit_rpl_inc = include/rpl_end.inc
 }
 if ($lts_series == v84)
 {
   --let $stmt_reset_binary_logs_and_gtids = RESET BINARY LOGS AND GTIDS
   --let $stmt_show_binary_log_status = SHOW BINARY LOG STATUS
+  --let $init_rpl_inc = include/rpl/init_source_replica.inc
+  --let $deinit_rpl_inc = include/rpl/deinit.inc
 }

mtr/binlog_streaming/r/binsrv.result

@@ -29,6 +29,10 @@ INSERT INTO t1 VALUES(DEFAULT);
 *** Checking that the Binlog Server utility detected an empty storage
 include/assert_grep.inc [Binlog storage must be initialized on an empty directory]
 
+*** Verifying that binlog files were written to storage
+include/assert_grep.inc [First binlog should exist in storage]
+include/assert_grep.inc [Second binlog also hould exist in storage]
+
 *** Comparing server and downloaded versions of the first binlog file.
 
 *** Patching the server version of the second binlog file to clear the

mtr/binlog_streaming/r/checkpointing_verification.result

@@ -0,0 +1,116 @@
+*** Testing checkpointing verification
+*** For S3 backend, each checkpoint triggers a full object re-upload.
+
+*** Resetting replication at the very beginning of the test.
+
+*** Creating a table and generating large binlog events
+CREATE TABLE t1(
+id SERIAL,
+val CHAR(64) CHARACTER SET ascii NOT NULL,
+PRIMARY KEY(id)
+) ENGINE=InnoDB;
+
+*** Generating data to trigger checkpointing
+
+*** Determining current open binlog file name
+
+*** Setting up checkpointing configuration
+
+*** Setting up configuration
+SET @save_binsrv_checkpoint_size = 'binsrv_checkpoint_size';
+SET @save_binsrv_checkpoint_interval = 'binsrv_checkpoint_interval';
+
+*** Test 1: Test checkpointing with existing open binary log
+*** Expectation:
+*** - Without checkpointing: storage object stays at magic payload size while binsrv runs
+*** - With size checkpointing: storage object grows beyond magic payload while binsrv runs
+*** Note: Checkpointing only works with S3 backend, not with file backend
+
+Executing binlog_server in pull mode without checkpointing
+
+*** Generating a configuration file in JSON format for the Binlog
+*** Server utility.
+
+*** Determining binlog file directory from the server.
+
+*** Creating a temporary directory <BINSRV_STORAGE_PATH> for storing
+*** binlog files downloaded via the Binlog Server utility.
+include/read_file_to_var.inc
+include/read_file_to_var.inc
+
+*** Removing the Binlog Server utility storage directory.
+
+*** Removing the Binlog Server utility log file.
+
+*** Removing the Binlog Server utility configuration file.
+
+Executing binlog_server in pull mode with checkpointing
+
+*** Generating a configuration file in JSON format for the Binlog
+*** Server utility.
+
+*** Determining binlog file directory from the server.
+
+*** Creating a temporary directory <BINSRV_STORAGE_PATH> for storing
+*** binlog files downloaded via the Binlog Server utility.
+include/read_file_to_var.inc
+include/read_file_to_var.inc
+
+*** Removing the Binlog Server utility storage directory.
+
+*** Removing the Binlog Server utility log file.
+
+*** Removing the Binlog Server utility configuration file.
+*** Test 2: Test that new updates in binary log on server will be flushed
+*** to the binlog_server storage directory due to checkpointing.
+
+
+*** Starting pull mode in background to monitor checkpointing
+
+*** Generating a configuration file in JSON format for the Binlog
+*** Server utility.
+
+*** Determining binlog file directory from the server.
+
+*** Creating a temporary directory <BINSRV_STORAGE_PATH> for storing
+*** binlog files downloaded via the Binlog Server utility.
+include/read_file_to_var.inc
+
+*** Capturing current storage object size (before workload)
+include/read_file_to_var.inc
+
+*** Generating more data to trigger checkpointing during pull
+
+*** Waiting a bit to allow checkpointing to occur
+
+*** Capturing storage object size after checkpointing workload
+include/read_file_to_var.inc
+
+*** Verifying checkpointing occurred (by storage object size change)
+
+*** Patching the server version of the binlog file to clear the
+*** LOG_EVENT_BINLOG_IN_USE_F (currently in use) flag.
+*** (The binlog is still open on the server, so we patch it instead of flushing
+*** to avoid adding a ROTATE event that wouldn't exist in the storage object)
+
+*** Terminating pull mode
+
+*** Verifying binlog file integrity after checkpointing
+
+*** Starting pull mode again to verify it resumes correctly without any issues.
+include/read_file_to_var.inc
+
+*** Adding more data
+INSERT INTO t1(val) VALUES(SHA2(LAST_INSERT_ID(), 256));
+FLUSH BINARY LOGS;
+
+*** Waiting a bit then terminating
+
+*** Verifying that resume worked correctly (no duplicate data)
+
+*** Removing the Binlog Server utility storage directory.
+
+*** Removing the Binlog Server utility log file.
+
+*** Removing the Binlog Server utility configuration file.
+DROP TABLE t1;

mtr/binlog_streaming/r/cli_validation.result

@@ -0,0 +1,64 @@
+
+*** Testing command line argument validation
+
+*** Test 1: No arguments (should fail)
+
+*** Test 2: Invalid operation mode (should fail)
+
+*** Test 3: Version mode without config file (should succeed)
+
+*** Test 3a: Verify version output format (semantic version)
+include/assert_grep.inc [Version output should match semantic version format (e.g., 0.1.0)]
+
+*** Test 3b: Verify version output is only on stdout (not stderr)
+include/read_file_to_var.inc
+
+*** Test 3c: Verify version output is a single line
+include/read_file_to_var.inc
+
+*** Test 4: Fetch mode without config file (should fail)
+
+*** Test 5: Pull mode without config file (should fail)
+
+*** Test 6: Fetch mode with non-existent config file (should fail)
+include/assert_grep.inc [Fetch failed since the config file specified doesn't exist.]
+
+*** Test 7: Pull mode with non-existent config file (should fail)
+include/assert_grep.inc [Pull failed since the config file specified doesn't exist.]
+
+*** Test 8: Too many arguments (should fail)
+
+*** Test 9: Version mode with extra argument (should fail)
+
+*** Test 10: Version mode with multiple extra arguments (should fail)
+
+*** Test 11: Version mode case sensitivity - uppercase (should fail)
+
+*** Test 12: Version mode case sensitivity - mixed case (should fail)
+
+*** Test 13: Version mode case sensitivity - VeRsIoN (should fail)
+
+*** Test 14: Verify version command exit code is 0
+include/read_file_to_var.inc
+
+*** Test 15: Invalid replication mode value (should fail)
+
+*** Generating a configuration file in JSON format for the Binlog
+*** Server utility.
+
+*** Determining binlog file directory from the server.
+
+*** Creating a temporary directory <BINSRV_STORAGE_PATH> for storing
+*** binlog files downloaded via the Binlog Server utility.
+include/assert_grep.inc [Config validation should fail with invalid replication mode]
+
+*** Test 16: Missing replication mode (should fail)
+
+*** Generating a configuration file in JSON format for the Binlog
+*** Server utility.
+
+*** Determining binlog file directory from the server.
+
+*** Creating a temporary directory <BINSRV_STORAGE_PATH> for storing
+*** binlog files downloaded via the Binlog Server utility.
+include/assert_grep.inc [Config validation should fail with missing replication mode]

mtr/binlog_streaming/r/config_validation.result

@@ -0,0 +1,117 @@
+
+*** Testing configuration parsing and validation failures
+
+*** Test 1: Empty configuration file
+include/assert_grep.inc [Empty config file should be rejected]
+
+*** Test 2: Malformed JSON configuration file
+include/assert_grep.inc [Malformed JSON config should be rejected]
+
+*** Test 3: Oversized configuration file
+include/assert_grep.inc [Oversized config file should be rejected]
+
+*** Preparing reusable local storage directory and config values
+SET @config_validation_log_path = 'MYSQL_TMP_DIR/binsrv_config_validation.log';
+SET @config_validation_storage_uri = CONCAT('file://', 'MYSQL_TMP_DIR/config_validation_storage');
+
+*** Test 4: Rewrite mode cannot be used with position replication
+SET @binsrv_config_json = JSON_OBJECT(
+'logger', JSON_OBJECT('level', 'error', 'file', @config_validation_log_path),
+'connection', JSON_OBJECT(
+'host', '127.0.0.1',
+'port', @@global.port,
+'user', 'root',
+'password', '',
+'connect_timeout', 20,
+'read_timeout', 60,
+'write_timeout', 60
+),
+'replication', JSON_OBJECT(
+'server_id', @@server_id + 1,
+'idle_time', 10,
+'verify_checksum', TRUE,
+'mode', 'position',
+'rewrite', JSON_OBJECT('base_file_name', 'rewritten', 'file_size', '1K')
+),
+'storage', JSON_OBJECT(
+'backend', 'file',
+'uri', @config_validation_storage_uri
+)
+);
+include/assert_grep.inc [Rewrite mode in position replication should fail validation]
+
+*** Test 5: Rewrite file size below minimum
+SET @binsrv_config_json = JSON_SET(@binsrv_config_json,
+'$.replication.mode', 'gtid',
+'$.replication.rewrite.file_size', '1023'
+);
+include/assert_grep.inc [Rewrite file size below 1024 bytes should fail validation]
+
+*** Test 6: Invalid checkpoint_size unit
+SET @binsrv_config_json = JSON_REMOVE(@binsrv_config_json, '$.replication.rewrite');
+SET @binsrv_config_json = JSON_SET(@binsrv_config_json,
+'$.replication.mode', 'position',
+'$.storage.checkpoint_size', 'abc'
+);
+include/assert_grep.inc [Invalid checkpoint size should fail parsing]
+
+*** Test 7: Invalid checkpoint_interval unit
+SET @binsrv_config_json = JSON_REMOVE(@binsrv_config_json, '$.storage.checkpoint_size');
+SET @binsrv_config_json = JSON_SET(@binsrv_config_json,
+'$.storage.checkpoint_interval', '5w'
+);
+include/assert_grep.inc [Invalid checkpoint interval should fail parsing]
+
+*** Test 8: Connection config without host/port or DNS SRV name
+SET @binsrv_config_json = JSON_REMOVE(@binsrv_config_json, '$.storage.checkpoint_interval');
+SET @binsrv_config_json = JSON_REMOVE(@binsrv_config_json, '$.connection.host');
+SET @binsrv_config_json = JSON_REMOVE(@binsrv_config_json, '$.connection.port');
+include/assert_grep.inc [Connection config must specify DNS SRV or host and port]
+
+*** Test 9: File URI with userinfo
+SET @file_uri_with_userinfo = CONCAT('file://user:pass@', 'MYSQL_TMP_DIR/config_validation_storage');
+SET @binsrv_config_json = JSON_SET(@binsrv_config_json,
+'$.connection.host', '127.0.0.1',
+'$.connection.port', @@global.port,
+'$.storage.uri', @file_uri_with_userinfo
+);
+include/assert_grep.inc [File URI with userinfo should be rejected]
+
+*** Test 10: File URI with query string
+SET @file_uri_with_query = CONCAT('file://', 'MYSQL_TMP_DIR/config_validation_storage', '?x=1');
+SET @binsrv_config_json = JSON_SET(@binsrv_config_json,
+'$.storage.uri', @file_uri_with_query
+);
+include/assert_grep.inc [File URI with query string should be rejected]
+
+*** Test 11: File URI with missing path
+SET @binsrv_config_json = JSON_SET(@binsrv_config_json,
+'$.storage.uri', 'file://'
+);
+include/assert_grep.inc [File URI with missing path should be rejected]
+
+*** Test 12: File URI with host
+SET @binsrv_config_json = JSON_SET(@binsrv_config_json,
+'$.storage.uri', 'file://host/path'
+);
+include/assert_grep.inc [File URI with host should be rejected]
+
+*** Test 13: Storage directory does not exist
+SET @missing_file_uri = CONCAT('file://', 'MYSQL_TMP_DIR/nonexistent_config_validation_storage');
+SET @binsrv_config_json = JSON_SET(@binsrv_config_json,
+'$.storage.uri', @missing_file_uri
+);
+include/assert_grep.inc [Missing storage directory should be rejected]
+
+*** Test 14: Storage path is a file, not a directory
+SET @storage_file_uri = CONCAT('file://', 'MYSQL_TMP_DIR/config_validation_storage_file');
+SET @binsrv_config_json = JSON_SET(@binsrv_config_json,
+'$.storage.uri', @storage_file_uri
+);
+include/assert_grep.inc [Storage path that is a file should be rejected]
+
+*** Test 15: Storage directory containing a subdirectory
+SET @binsrv_config_json = JSON_SET(@binsrv_config_json,
+'$.storage.uri', @config_validation_storage_uri
+);
+include/assert_grep.inc [Storage directory entries must be regular files]