Skip to content

Commit a89a89e

Browse files
memleakdmemleakd
committed
feat(http): add CURLRequest retry option
- Add retry support for failed HTTP responses - Support configurable attempts, delays, status codes, and Retry-After - Add opt-in retries for transient cURL errors - Document retry behavior and safety considerations - Add focused CURLRequest retry tests Signed-off-by: memleakd <121398829+memleakd@users.noreply.github.com>
1 parent 1efb85b commit a89a89e

6 files changed

Lines changed: 707 additions & 3 deletions

File tree

‎system/HTTP/CURLRequest.php‎

Lines changed: 251 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,33 @@ class CURLRequest extends OutgoingRequest
8282
],
8383
];
8484

85+
/**
86+
* Default values for when 'retry' is enabled.
87+
*
88+
* @var array<string, bool|int|list<int>>
89+
*/
90+
protected $retryDefaults = [
91+
'max_retries' => 3,
92+
'delay' => 1000,
93+
'max_delay' => 30_000,
94+
'status_codes' => [429, 503],
95+
'curl_errors' => false,
96+
'respect_retry_after' => true,
97+
];
98+
99+
/**
100+
* cURL error numbers that may succeed on another attempt.
101+
*
102+
* @var list<int>
103+
*/
104+
protected $transientCurlErrors = [
105+
CURLE_COULDNT_RESOLVE_HOST,
106+
CURLE_COULDNT_CONNECT,
107+
CURLE_OPERATION_TIMEDOUT,
108+
CURLE_SEND_ERROR,
109+
CURLE_RECV_ERROR,
110+
];
111+
85112
/**
86113
* The number of milliseconds to delay before
87114
* sending the request.
@@ -90,6 +117,11 @@ class CURLRequest extends OutgoingRequest
90117
*/
91118
protected $delay = 0.0;
92119

120+
/**
121+
* The last cURL error number.
122+
*/
123+
protected int $lastCurlError = 0;
124+
93125
/**
94126
* The default options from the constructor. Applied to all requests.
95127
*/
@@ -374,6 +406,8 @@ public function send(string $method, string $url)
374406
{
375407
// Reset our curl options so we're on a fresh slate.
376408
$curlOptions = [];
409+
$config = $this->config;
410+
$retry = $this->normalizeRetryOption($config['retry'] ?? false);
377411

378412
if (! empty($this->config['query']) && is_array($this->config['query'])) {
379413
// This is likely too naive a solution.
@@ -394,14 +428,75 @@ public function send(string $method, string $url)
394428
// Disable @file uploads in post data.
395429
$curlOptions[CURLOPT_SAFE_UPLOAD] = true;
396430

397-
$curlOptions = $this->setCURLOptions($curlOptions, $this->config);
431+
$curlOptions = $this->setCURLOptions($curlOptions, $config);
398432
$curlOptions = $this->applyMethod($method, $curlOptions);
399433
$curlOptions = $this->applyRequestHeaders($curlOptions);
400434

435+
if ($retry !== null) {
436+
$curlOptions[CURLOPT_FAILONERROR] = false;
437+
}
438+
401439
// Do we need to delay this request?
402440
if ($this->delay > 0) {
403-
usleep((int) $this->delay * 1_000_000);
441+
$this->sleep($this->delay);
442+
}
443+
444+
if ($retry === null) {
445+
return $this->sendAttempt($curlOptions);
446+
}
447+
448+
$httpErrors = array_key_exists('http_errors', $config) ? (bool) $config['http_errors'] : true;
449+
450+
return $this->sendWithRetries($curlOptions, $retry, $httpErrors);
451+
}
452+
453+
/**
454+
* Sends the request until it succeeds or retry attempts are exhausted.
455+
*
456+
* @param array<int, mixed> $curlOptions
457+
* @param array<string, bool|int|list<int>> $retry
458+
*/
459+
protected function sendWithRetries(array $curlOptions, array $retry, bool $httpErrors): ResponseInterface
460+
{
461+
$attempt = 0;
462+
463+
while (true) {
464+
$this->response = clone $this->responseOrig;
465+
466+
try {
467+
$response = $this->sendAttempt($curlOptions);
468+
} catch (HTTPException $e) {
469+
if (! $this->shouldRetryCurlError($retry, $attempt)) {
470+
throw $e;
471+
}
472+
473+
$this->sleep($this->getRetryDelay($retry, $attempt) / 1000);
474+
$attempt++;
475+
476+
continue;
477+
}
478+
479+
if (! $this->shouldRetryResponse($response, $retry, $attempt)) {
480+
if ($httpErrors && $response->getStatusCode() >= 400) {
481+
throw HTTPException::forCurlError('22', 'The requested URL returned error: ' . $response->getStatusCode());
482+
}
483+
484+
return $response;
485+
}
486+
487+
$this->sleep($this->getRetryDelay($retry, $attempt, $response) / 1000);
488+
$attempt++;
404489
}
490+
}
491+
492+
/**
493+
* Sends a single cURL request attempt and populates the response.
494+
*
495+
* @param array<int, mixed> $curlOptions
496+
*/
497+
protected function sendAttempt(array $curlOptions): ResponseInterface
498+
{
499+
$this->lastCurlError = 0;
405500

406501
$output = $this->sendRequest($curlOptions);
407502

@@ -430,6 +525,158 @@ public function send(string $method, string $url)
430525
return $this->response;
431526
}
432527

528+
/**
529+
* Normalizes the retry option into retry settings.
530+
*
531+
* @return array<string, bool|int|list<int>>|null
532+
*/
533+
protected function normalizeRetryOption(mixed $retry): ?array
534+
{
535+
if (in_array($retry, [false, null, 0], true)) {
536+
return null;
537+
}
538+
539+
$config = $this->retryDefaults;
540+
541+
if (is_int($retry)) {
542+
$config['max_retries'] = $retry;
543+
} elseif (is_array($retry)) {
544+
$config = array_merge($config, $retry);
545+
} else {
546+
return null;
547+
}
548+
549+
$config['max_retries'] = max(0, (int) $config['max_retries']);
550+
551+
if ($config['max_retries'] === 0) {
552+
return null;
553+
}
554+
555+
$config['delay'] = $this->normalizeRetryDelay($config['delay']);
556+
$config['max_delay'] = max(0, (int) $config['max_delay']);
557+
$config['status_codes'] = array_map(intval(...), (array) $config['status_codes']);
558+
$config['curl_errors'] = (bool) $config['curl_errors'];
559+
$config['respect_retry_after'] = (bool) $config['respect_retry_after'];
560+
561+
return $config;
562+
}
563+
564+
/**
565+
* Normalizes the retry delay setting.
566+
*
567+
* @return int|list<int>
568+
*/
569+
protected function normalizeRetryDelay(mixed $delay): array|int
570+
{
571+
if (is_array($delay)) {
572+
return array_map(static fn ($value): int => max(0, (int) $value), $delay);
573+
}
574+
575+
return max(0, (int) $delay);
576+
}
577+
578+
/**
579+
* Determines whether a response should be retried.
580+
*
581+
* @param array<string, bool|int|list<int>> $retry
582+
*/
583+
protected function shouldRetryResponse(ResponseInterface $response, array $retry, int $attempt): bool
584+
{
585+
if ($attempt >= $retry['max_retries']) {
586+
return false;
587+
}
588+
589+
return in_array($response->getStatusCode(), $retry['status_codes'], true);
590+
}
591+
592+
/**
593+
* Determines whether a cURL error should be retried.
594+
*
595+
* @param array<string, bool|int|list<int>> $retry
596+
*/
597+
protected function shouldRetryCurlError(array $retry, int $attempt): bool
598+
{
599+
if ($attempt >= $retry['max_retries'] || $retry['curl_errors'] === false) {
600+
return false;
601+
}
602+
603+
return in_array($this->lastCurlError, $this->transientCurlErrors, true);
604+
}
605+
606+
/**
607+
* Returns the delay before the next retry attempt.
608+
*
609+
* @param array<string, bool|int|list<int>> $retry
610+
*/
611+
protected function getRetryDelay(array $retry, int $attempt, ?ResponseInterface $response = null): int
612+
{
613+
if ($response instanceof ResponseInterface && $retry['respect_retry_after'] === true) {
614+
$retryAfter = $this->getRetryAfterDelay($response);
615+
616+
if ($retryAfter !== null) {
617+
return $this->limitRetryDelay($retryAfter * 1000, $retry);
618+
}
619+
}
620+
621+
$delay = $retry['delay'];
622+
623+
if (is_array($delay)) {
624+
$lastDelay = end($delay);
625+
626+
return $this->limitRetryDelay((int) ($delay[$attempt] ?? ($lastDelay !== false ? $lastDelay : 0)), $retry);
627+
}
628+
629+
return $this->limitRetryDelay((int) $delay, $retry);
630+
}
631+
632+
/**
633+
* Caps the retry delay when configured.
634+
*
635+
* @param array<string, bool|int|list<int>> $retry
636+
*/
637+
protected function limitRetryDelay(int $delay, array $retry): int
638+
{
639+
$maxDelay = (int) $retry['max_delay'];
640+
641+
if ($maxDelay === 0) {
642+
return $delay;
643+
}
644+
645+
return min($delay, $maxDelay);
646+
}
647+
648+
/**
649+
* Returns the delay from a Retry-After header in seconds.
650+
*/
651+
protected function getRetryAfterDelay(ResponseInterface $response): ?int
652+
{
653+
$retryAfter = $response->getHeaderLine('Retry-After');
654+
655+
if ($retryAfter === '') {
656+
return null;
657+
}
658+
659+
if (ctype_digit($retryAfter)) {
660+
return (int) $retryAfter;
661+
}
662+
663+
$timestamp = strtotime($retryAfter);
664+
665+
if ($timestamp === false) {
666+
return null;
667+
}
668+
669+
return max(0, $timestamp - time());
670+
}
671+
672+
/**
673+
* Sleeps for the configured number of seconds.
674+
*/
675+
protected function sleep(float $seconds): void
676+
{
677+
usleep((int) ($seconds * 1_000_000));
678+
}
679+
433680
/**
434681
* Adds $this->headers to the cURL request.
435682
*/
@@ -731,6 +978,8 @@ protected function sendRequest(array $curlOptions = []): string
731978
$output = curl_exec($ch);
732979

733980
if ($output === false) {
981+
$this->lastCurlError = curl_errno($ch);
982+
734983
throw HTTPException::forCurlError((string) curl_errno($ch), curl_error($ch));
735984
}
736985

‎system/Test/Mock/MockCURLRequest.php‎

Lines changed: 62 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
namespace CodeIgniter\Test\Mock;
1515

1616
use CodeIgniter\HTTP\CURLRequest;
17+
use CodeIgniter\HTTP\Exceptions\HTTPException;
1718
use CodeIgniter\HTTP\URI;
1819

1920
/**
@@ -33,6 +34,21 @@ class MockCURLRequest extends CURLRequest
3334
*/
3435
protected $output = '';
3536

37+
/**
38+
* @var list<string>
39+
*/
40+
protected array $outputs = [];
41+
42+
/**
43+
* @var list<array{0: int, 1: string}>
44+
*/
45+
protected array $curlErrors = [];
46+
47+
/**
48+
* @var list<float>
49+
*/
50+
protected array $sleeps = [];
51+
3652
/**
3753
* @param string $output
3854
*
@@ -45,6 +61,30 @@ public function setOutput($output)
4561
return $this;
4662
}
4763

64+
/**
65+
* @param list<string> $outputs
66+
*
67+
* @return $this
68+
*/
69+
public function setOutputs(array $outputs)
70+
{
71+
$this->outputs = $outputs;
72+
73+
return $this;
74+
}
75+
76+
/**
77+
* @param list<array{0: int, 1: string}> $curlErrors
78+
*
79+
* @return $this
80+
*/
81+
public function setCurlErrors(array $curlErrors)
82+
{
83+
$this->curlErrors = $curlErrors;
84+
85+
return $this;
86+
}
87+
4888
/**
4989
* @param array<int, mixed> $curlOptions
5090
*/
@@ -54,7 +94,28 @@ protected function sendRequest(array $curlOptions = []): string
5494

5595
$this->curl_options = $curlOptions;
5696

57-
return $this->output;
97+
if ($this->curlErrors !== []) {
98+
[$this->lastCurlError, $message] = array_shift($this->curlErrors);
99+
100+
throw HTTPException::forCurlError((string) $this->lastCurlError, $message);
101+
}
102+
103+
return $this->outputs !== [] ? array_shift($this->outputs) : $this->output;
104+
}
105+
106+
protected function sleep(float $seconds): void
107+
{
108+
$this->sleeps[] = $seconds;
109+
}
110+
111+
/**
112+
* for testing purposes only
113+
*
114+
* @return list<float>
115+
*/
116+
public function getSleeps(): array
117+
{
118+
return $this->sleeps;
58119
}
59120

60121
/**

0 commit comments

Comments
 (0)