test: support retries in test methods

Author: jaredhendrickson13

pfSense-pkg-RESTAPI/files/usr/local/pkg/RESTAPI/Core/TestCase.inc

@@ -68,20 +68,36 @@ class TestCase {
 
                 # Set the current method undergoing testing
                 $this->method = $method;
+                $ref_method = new ReflectionMethod($this, $method);
 
                 # Gather the description for this test method from its doc comment
-                $this->method_docstring = (new ReflectionMethod($this, $method))->getDocComment();
+                $this->method_docstring = $ref_method->getDocComment();
                 $this->method_docstring = str_replace([PHP_EOL, '/**', '*/', '*', '  '], '', $this->method_docstring);
 
-                # Try to run the test. On failure, restore the original config and throw the exception
-                try {
-                    $this->$method();
-                } catch (Error | Exception $e) {
-                    # Restore the original configuration, teardown the TestCase and throw the encountered error
-                    $config = $original_config;
-                    write_config("Restored config after API test '$method'");
-                    $this->teardown();
-                    throw $e;
+                # Get the retry settings for this method
+                $retry_settings = $this->get_method_retry_settings($ref_method);
+
+                # Always attempt once, then add retries if configured
+                for ($attempt = 0; $attempt <= $retry_settings->retries; $attempt++) {
+                    try {
+                        # Try to run the test
+                        $this->$method();
+                        break;
+                    } catch (Error | Exception $e) {
+                        # Tear down any resources created by this test before retrying or exiting
+                        $config = $original_config;
+                        write_config("Restored config after API test '$method'");
+                        $this->teardown();
+
+                        # If we have retries left, wait the configured delay and try again
+                        if ($attempt < $retry_settings->retries) {
+                            sleep($retry_settings->delay);
+                            continue;
+                        }
+
+                        # If we made it here, we have no retries left. Throw the exception to the caller.
+                        throw $e;
+                    }
                 }
 
                 # Restore the config as it was when the test began.
@@ -137,6 +153,23 @@ class TestCase {
      */
     public function teardown(): void {}
 
+    /**
+     * @param ReflectionMethod $method The method to check for a TestCaseRetry attribute.
+     * @return TestCaseRetry The TestCaseRetry attribute found on the method, or a default TestCaseRetry object
+     */
+    protected function get_method_retry_settings(ReflectionMethod $method): TestCaseRetry {
+        # Use default retry settings if no attribute is found
+        $retries = new TestCaseRetry();
+
+        # If this method has a TestCaseRetry attribute, use it
+        $attributes = $method->getAttributes(TestCaseRetry::class);
+        if (count($attributes) > 0) {
+            $retries = $attributes[0]->newInstance();
+        }
+
+        return $retries;
+    }
+
     /**
      * Obtains the environment variable with a given name.
      */

pfSense-pkg-RESTAPI/files/usr/local/pkg/RESTAPI/Core/TestCaseRetry.inc

@@ -0,0 +1,25 @@
+<?php
+
+namespace RESTAPI\Core;
+
+use Attribute;
+
+/**
+ * A class to represent retry settings for test cases.  This is intended to be defined in methods as a method attribute.
+ */
+#[Attribute(Attribute::TARGET_METHOD)]
+class TestCaseRetry
+{
+
+    /**
+     * @param int $retries The number of retries to attempt
+     * @param int $delay The delay in seconds between retries
+     */
+    public function __construct(public int $retries = 0, public int $delay = 0)
+    {
+        # Ensure values are non-negative
+        $this->retries = max(0, $retries);
+        $this->delay = max(0, $delay);
+    }
+
+}