diff --git a/client/client-core/src/main/java/software/amazon/smithy/java/client/core/ClientPipeline.java b/client/client-core/src/main/java/software/amazon/smithy/java/client/core/ClientPipeline.java index 016d210680..a2bb8f684c 100644 --- a/client/client-core/src/main/java/software/amazon/smithy/java/client/core/ClientPipeline.java +++ b/client/client-core/src/main/java/software/amazon/smithy/java/client/core/ClientPipeline.java @@ -395,6 +395,10 @@ private O deseriali return retry(call, request, acquireResult.token(), acquireResult.delay()); } catch (TokenAcquisitionFailedException tafe) { // 9.b If InterceptorContext.response() is an unretryable failure, continue to step 10. + // For long-polling operations, backoff before returning. + if (tafe.delay() != null && !tafe.delay().isZero()) { + sleep(tafe.delay()); + } LOGGER.debug("Cannot acquire a retry token: {}", tafe); } } diff --git a/client/client-http/src/main/java/software/amazon/smithy/java/client/http/plugins/ApplyHttpRetryInfoPlugin.java b/client/client-http/src/main/java/software/amazon/smithy/java/client/http/plugins/ApplyHttpRetryInfoPlugin.java index 3fe96b051f..f65f5e73e6 100644 --- a/client/client-http/src/main/java/software/amazon/smithy/java/client/http/plugins/ApplyHttpRetryInfoPlugin.java +++ b/client/client-http/src/main/java/software/amazon/smithy/java/client/http/plugins/ApplyHttpRetryInfoPlugin.java @@ -6,16 +6,11 @@ package software.amazon.smithy.java.client.http.plugins; import java.time.Duration; -import java.time.Instant; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.time.temporal.ChronoUnit; import software.amazon.smithy.java.client.core.AutoClientPlugin; import software.amazon.smithy.java.client.core.CallContext; import software.amazon.smithy.java.client.core.ClientConfig; import software.amazon.smithy.java.client.core.interceptors.ClientInterceptor; import software.amazon.smithy.java.client.core.interceptors.OutputHook; -import software.amazon.smithy.java.client.core.settings.ClockSetting; import software.amazon.smithy.java.client.http.HttpMessageExchange; import software.amazon.smithy.java.context.Context; import software.amazon.smithy.java.core.error.CallException; @@ -33,6 +28,8 @@ */ @SmithyInternalApi public final class ApplyHttpRetryInfoPlugin implements AutoClientPlugin { + private static final HeaderName X_AMZ_RETRY_AFTER = HeaderName.of("x-amz-retry-after"); + @Override public void configureClient(ClientConfig.Builder config) { // We can conditionally add the interceptor here because client transport can't change after construction. @@ -60,7 +57,7 @@ public O modifyBeforeAttemptCompletion( static void applyRetryInfo(HttpResponse response, CallException exception, Context context) { // (1) Check with the protocol if the server explicitly wants a retry. - if (!applyRetryAfterHeader(response, exception, context)) { + if (!applyRetryAfterHeader(response, exception)) { if (!applyThrottlingStatusCodes(response, exception)) { // (2) If no retry was detected so far, is it safe to retry because of a 5XX error + idempotency token? if (exception.isRetrySafe() == RetrySafety.MAYBE) { @@ -85,29 +82,19 @@ private static boolean applyThrottlingStatusCodes(HttpResponse response, CallExc return false; } - // If there's a retry-after header, then the server is telling us it's retryable. - private static boolean applyRetryAfterHeader(HttpResponse response, CallException exception, Context context) { - var retryAfter = response.headers().firstValue(HeaderName.RETRY_AFTER); - if (retryAfter != null) { - exception.isThrottle(true); - exception.isRetrySafe(RetrySafety.YES); - exception.retryAfter(parseRetryAfter(retryAfter, context)); - return true; + // Per SEP: use x-amz-retry-after (integer milliseconds). Ignore standard Retry-After header. + private static boolean applyRetryAfterHeader(HttpResponse response, CallException exception) { + var xAmzRetryAfter = response.headers().firstValue(X_AMZ_RETRY_AFTER); + if (xAmzRetryAfter != null) { + try { + var millis = Long.parseLong(xAmzRetryAfter); + exception.isRetrySafe(RetrySafety.YES); + exception.retryAfter(Duration.ofMillis(millis)); + return true; + } catch (NumberFormatException e) { + // Invalid value — ignore, fall back to exponential backoff + } } - return false; } - - private static Duration parseRetryAfter(String retryAfter, Context context) { - try { - return Duration.of(Integer.parseInt(retryAfter), ChronoUnit.SECONDS); - } catch (NumberFormatException e) { - // It's not a number, so it must be a http-date like "Wed, 21 Oct 2015 07:28:00 GMT". - var date = ZonedDateTime.parse(retryAfter, DateTimeFormatter.RFC_1123_DATE_TIME); - // Use the Clock associated with the context, if any, to account for things like clock skew. - var clock = context.get(ClockSetting.CLOCK); - var now = clock == null ? Instant.now() : clock.instant(); - return Duration.between(now, date); - } - } } diff --git a/client/client-http/src/test/java/software/amazon/smithy/java/client/http/plugins/ApplyHttpRetryInfoPluginTest.java b/client/client-http/src/test/java/software/amazon/smithy/java/client/http/plugins/ApplyHttpRetryInfoPluginTest.java index f3aa6cff99..1d02d3f0cc 100644 --- a/client/client-http/src/test/java/software/amazon/smithy/java/client/http/plugins/ApplyHttpRetryInfoPluginTest.java +++ b/client/client-http/src/test/java/software/amazon/smithy/java/client/http/plugins/ApplyHttpRetryInfoPluginTest.java @@ -10,15 +10,11 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; -import java.time.Clock; import java.time.Duration; -import java.time.Instant; -import java.time.ZoneId; import java.util.List; import java.util.Map; import org.junit.jupiter.api.Test; import software.amazon.smithy.java.client.core.CallContext; -import software.amazon.smithy.java.client.core.settings.ClockSetting; import software.amazon.smithy.java.context.Context; import software.amazon.smithy.java.core.error.CallException; import software.amazon.smithy.java.http.api.HttpHeaders; @@ -26,11 +22,12 @@ import software.amazon.smithy.java.retries.api.RetrySafety; public class ApplyHttpRetryInfoPluginTest { + @Test - public void appliesRetryAfterHeader() { + public void appliesXAmzRetryAfterHeader() { var response = HttpResponse.create() .setStatusCode(500) - .setHeaders(HttpHeaders.of(Map.of("retry-after", List.of("10")))) + .setHeaders(HttpHeaders.of(Map.of("x-amz-retry-after", List.of("1500")))) .toUnmodifiable(); var e = new CallException("err"); var context = Context.create(); @@ -38,25 +35,41 @@ public void appliesRetryAfterHeader() { ApplyHttpRetryInfoPlugin.applyRetryInfo(response, e, context); assertThat(e.isRetrySafe(), is(RetrySafety.YES)); - assertThat(e.isThrottle(), is(true)); - assertThat(e.retryAfter(), equalTo(Duration.ofSeconds(10))); + assertThat(e.retryAfter(), equalTo(Duration.ofMillis(1500))); } @Test - public void appliesRetryAfterHeaderDate() { + public void ignoresInvalidXAmzRetryAfterHeader() { var response = HttpResponse.create() .setStatusCode(500) - .setHeaders(HttpHeaders.of(Map.of("retry-after", List.of("Wed, 21 Oct 2015 07:28:00 GMT")))) + .setHeaders(HttpHeaders.of(Map.of("x-amz-retry-after", List.of("invalid")))) .toUnmodifiable(); var e = new CallException("err"); var context = Context.create(); - context.put(ClockSetting.CLOCK, Clock.fixed(Instant.parse("2015-10-21T05:28:00Z"), ZoneId.of("UTC"))); + context.put(CallContext.IDEMPOTENCY_TOKEN, "foo"); ApplyHttpRetryInfoPlugin.applyRetryInfo(response, e, context); + // Falls through to normal 5xx + idempotency handling assertThat(e.isRetrySafe(), is(RetrySafety.YES)); - assertThat(e.isThrottle(), is(true)); - assertThat(e.retryAfter(), equalTo(Duration.ofHours(2))); + assertThat(e.retryAfter(), nullValue()); + } + + @Test + public void ignoresStandardRetryAfterHeader() { + // SEP: SDKs MUST ignore the standard HTTP Retry-After header + var response = HttpResponse.create() + .setStatusCode(500) + .setHeaders(HttpHeaders.of(Map.of("retry-after", List.of("10")))) + .toUnmodifiable(); + var e = new CallException("err"); + var context = Context.create(); + + ApplyHttpRetryInfoPlugin.applyRetryInfo(response, e, context); + + // Standard Retry-After is ignored, falls through to non-retryable (no idempotency token) + assertThat(e.isRetrySafe(), is(RetrySafety.NO)); + assertThat(e.retryAfter(), nullValue()); } @Test diff --git a/retries-api/src/main/java/software/amazon/smithy/java/retries/api/TokenAcquisitionFailedException.java b/retries-api/src/main/java/software/amazon/smithy/java/retries/api/TokenAcquisitionFailedException.java index c46eab074c..e203e7fab3 100644 --- a/retries-api/src/main/java/software/amazon/smithy/java/retries/api/TokenAcquisitionFailedException.java +++ b/retries-api/src/main/java/software/amazon/smithy/java/retries/api/TokenAcquisitionFailedException.java @@ -5,25 +5,37 @@ package software.amazon.smithy.java.retries.api; +import java.time.Duration; + /** * Exception thrown by {@link RetryStrategy} when a new token cannot be acquired. */ public final class TokenAcquisitionFailedException extends RuntimeException { private final transient RetryToken token; + private final transient Duration delay; public TokenAcquisitionFailedException(String msg) { super(msg); token = null; + delay = Duration.ZERO; } public TokenAcquisitionFailedException(String msg, Throwable cause) { super(msg, cause); token = null; + delay = Duration.ZERO; } public TokenAcquisitionFailedException(String msg, RetryToken token, Throwable cause) { super(msg, cause); this.token = token; + this.delay = Duration.ZERO; + } + + public TokenAcquisitionFailedException(String msg, RetryToken token, Throwable cause, Duration delay) { + super(msg, cause); + this.token = token; + this.delay = delay != null ? delay : Duration.ZERO; } /** @@ -33,4 +45,13 @@ public TokenAcquisitionFailedException(String msg, RetryToken token, Throwable c public RetryToken token() { return token; } + + /** + * Returns the delay to wait before returning the error to the caller. + * This is non-zero for long-polling operations when retry quota is exhausted. + * @return the delay. + */ + public Duration delay() { + return delay; + } } diff --git a/retries/src/main/java/software/amazon/smithy/java/retries/AdaptiveRetryStrategy.java b/retries/src/main/java/software/amazon/smithy/java/retries/AdaptiveRetryStrategy.java index cd73deb3fc..fcba303217 100644 --- a/retries/src/main/java/software/amazon/smithy/java/retries/AdaptiveRetryStrategy.java +++ b/retries/src/main/java/software/amazon/smithy/java/retries/AdaptiveRetryStrategy.java @@ -69,6 +69,7 @@ public static Builder builder() { return new Builder() .maxAttempts(Constants.Adaptive.MAX_ATTEMPTS) .backoffBaseDelay(Constants.Adaptive.BASE_DELAY) + .throttlingBackoffBaseDelay(Constants.Standard.THROTTLING_BASE_DELAY) .backoffMaxBackoff(Constants.Adaptive.MAX_BACKOFF) .retryCost(Constants.Adaptive.RETRY_COST) .throttlingRetryCost(Constants.Adaptive.THROTTLING_RETRY_COST) @@ -118,6 +119,17 @@ public Builder backoffBaseDelay(Duration baseDelay) { return this; } + /** + * Set the base delay for exponential backoff on throttling errors. + * + * @param baseDelay the throttling base delay. + * @return the builder. + */ + public Builder throttlingBackoffBaseDelay(Duration baseDelay) { + setThrottlingBackoffBaseDelay(baseDelay); + return this; + } + /** * Set the maximum backoff delay. * diff --git a/retries/src/main/java/software/amazon/smithy/java/retries/BaseRetryStrategy.java b/retries/src/main/java/software/amazon/smithy/java/retries/BaseRetryStrategy.java index c3b3e7dd86..72af1202e3 100644 --- a/retries/src/main/java/software/amazon/smithy/java/retries/BaseRetryStrategy.java +++ b/retries/src/main/java/software/amazon/smithy/java/retries/BaseRetryStrategy.java @@ -9,7 +9,9 @@ import java.time.Duration; import java.util.Objects; +import java.util.Random; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; import software.amazon.smithy.java.logging.InternalLogger; import software.amazon.smithy.java.retries.api.AcquireInitialTokenRequest; import software.amazon.smithy.java.retries.api.AcquireInitialTokenResponse; @@ -29,6 +31,7 @@ abstract class BaseRetryStrategy implements RetryStrategy, Claimable { protected final InternalLogger log; protected final int maxAttempts; protected final ExponentialDelayWithJitter backoffStrategy; + protected final ExponentialDelayWithJitter throttlingBackoffStrategy; protected final int retryCost; protected final int throttlingRetryCost; protected final int initialRetryTokens; @@ -38,7 +41,21 @@ abstract class BaseRetryStrategy implements RetryStrategy, Claimable { BaseRetryStrategy(InternalLogger log, Builder builder) { this.log = log; this.maxAttempts = validateIsPositive(builder.maxAttempts, "maxAttempts"); - this.backoffStrategy = new ExponentialDelayWithJitter(builder.backoffBaseDelay, builder.backoffMaxDelay); + if (builder.randomSupplier != null) { + this.backoffStrategy = new ExponentialDelayWithJitter( + builder.randomSupplier, + builder.backoffBaseDelay, + builder.backoffMaxDelay); + this.throttlingBackoffStrategy = new ExponentialDelayWithJitter( + builder.randomSupplier, + builder.throttlingBackoffBaseDelay, + builder.backoffMaxDelay); + } else { + this.backoffStrategy = new ExponentialDelayWithJitter(builder.backoffBaseDelay, builder.backoffMaxDelay); + this.throttlingBackoffStrategy = new ExponentialDelayWithJitter( + builder.throttlingBackoffBaseDelay, + builder.backoffMaxDelay); + } this.retryCost = Objects.requireNonNull(builder.retryCost, "retryCost"); this.throttlingRetryCost = Objects.requireNonNull(builder.throttlingRetryCost, "throttlingRetryCost"); this.initialRetryTokens = builder.initialRetryTokens; @@ -140,12 +157,16 @@ protected Duration computeInitialBackoff(AcquireInitialTokenRequest request) { * method to compute a different backoff depending on their logic. */ protected Duration computeBackoff(RefreshRetryTokenRequest request, DefaultRetryToken token) { - var backoff = backoffStrategy.computeDelay(token.attempt()); + var isThrottle = treatAsThrottling(request.failure()); + var strategy = isThrottle ? throttlingBackoffStrategy : backoffStrategy; + var backoff = strategy.computeDelay(token.attempt()); var suggested = request.suggestedDelay(); if (suggested == null) { return backoff; } - return maxOf(suggested, backoff); + // Clamp suggested delay: min bound is t_i, max bound is 5s + t_i + var maxBound = backoff.plus(Duration.ofSeconds(5)); + return minOf(maxOf(suggested, backoff), maxBound); } /** @@ -183,7 +204,7 @@ private DefaultRetryToken refreshToken(RefreshRetryTokenRequest request, Acquire private AcquireResponse requestAcquireCapacity(RefreshRetryTokenRequest request, DefaultRetryToken token) { var tokenBucket = tokenBucketStore.tokenBucketForScope(token.scope()); - return tokenBucket.tryAcquire(retryCost(request), token); + return tokenBucket.tryAcquire(retryCost(request)); } private ReleaseResponse releaseTokenBucketCapacity(DefaultRetryToken token) { @@ -238,6 +259,19 @@ private void throwOnAcquisitionFailure(RefreshRetryTokenRequest request, Acquire var token = asDefaultRetryToken(request.token()); if (acquireResponse.acquisitionFailed()) { var failure = request.failure(); + // For long-polling operations, compute backoff delay before returning + if (token.isLongPolling()) { + var refreshedToken = token.toBuilder() + .increaseAttempt() + .capacityRemaining(acquireResponse.capacityRemaining()) + .capacityAcquired(acquireResponse.capacityAcquired()) + .addFailure(failure) + .build(); + var backoff = computeBackoff(request, refreshedToken); + var message = acquisitionFailedForLongPollOperationMessage(acquireResponse); + log.debug(message, failure); + throw new TokenAcquisitionFailedException(message, refreshedToken, failure, backoff); + } var refreshedToken = token.toBuilder() .capacityRemaining(acquireResponse.capacityRemaining()) .capacityAcquired(acquireResponse.capacityAcquired()) @@ -336,6 +370,13 @@ static Duration maxOf(Duration left, Duration right) { return right; } + static Duration minOf(Duration left, Duration right) { + if (left.compareTo(right) <= 0) { + return left; + } + return right; + } + static DefaultRetryToken asDefaultRetryToken(RetryToken token) { if (token instanceof DefaultRetryToken t) { return t; @@ -363,7 +404,9 @@ public abstract static class Builder implements RetryStrategy.Builder { protected int initialRetryTokens; protected int maxScopes; protected Duration backoffBaseDelay; + protected Duration throttlingBackoffBaseDelay; protected Duration backoffMaxDelay; + private Supplier randomSupplier; Builder() { this.initialRetryTokens = Constants.Standard.INITIAL_RETRY_TOKENS; @@ -377,6 +420,7 @@ public abstract static class Builder implements RetryStrategy.Builder { this.initialRetryTokens = strategy.initialRetryTokens; this.maxScopes = strategy.maxScopes; this.backoffBaseDelay = strategy.backoffStrategy.baseDelay(); + this.throttlingBackoffBaseDelay = strategy.throttlingBackoffStrategy.baseDelay(); this.backoffMaxDelay = strategy.backoffStrategy.maxDelay(); } @@ -404,8 +448,16 @@ void setBackoffBaseDelay(Duration backoffBaseDelay) { this.backoffBaseDelay = backoffBaseDelay; } + void setThrottlingBackoffBaseDelay(Duration throttlingBackoffBaseDelay) { + this.throttlingBackoffBaseDelay = throttlingBackoffBaseDelay; + } + void setBackoffMaxDelay(Duration backoffMaxDelay) { this.backoffMaxDelay = backoffMaxDelay; } + + void setRandomSupplier(Supplier randomSupplier) { + this.randomSupplier = randomSupplier; + } } } diff --git a/retries/src/main/java/software/amazon/smithy/java/retries/Constants.java b/retries/src/main/java/software/amazon/smithy/java/retries/Constants.java index 9a79ea9993..3d48abad8b 100644 --- a/retries/src/main/java/software/amazon/smithy/java/retries/Constants.java +++ b/retries/src/main/java/software/amazon/smithy/java/retries/Constants.java @@ -24,6 +24,7 @@ final class Constants { */ static final class Standard { static final Duration BASE_DELAY = Duration.ofMillis(50); + static final Duration THROTTLING_BASE_DELAY = Duration.ofMillis(1000); static final Duration MAX_BACKOFF = Duration.ofSeconds(20); static final int MAX_ATTEMPTS = 3; static final int RETRY_COST = 14; diff --git a/retries/src/main/java/software/amazon/smithy/java/retries/ExponentialDelayWithJitter.java b/retries/src/main/java/software/amazon/smithy/java/retries/ExponentialDelayWithJitter.java index 0b437517b1..03d0d2a774 100644 --- a/retries/src/main/java/software/amazon/smithy/java/retries/ExponentialDelayWithJitter.java +++ b/retries/src/main/java/software/amazon/smithy/java/retries/ExponentialDelayWithJitter.java @@ -48,8 +48,8 @@ Duration computeDelay(int attempt) { if (delay <= 0) { return Duration.ZERO; } - var randInt = randomSupplier.get().nextInt(delay); - return Duration.ofMillis(randInt); + var jitter = randomSupplier.get().nextDouble(); + return Duration.ofMillis((long) (jitter * delay)); } int calculateExponentialDelay(int attempt) { diff --git a/retries/src/main/java/software/amazon/smithy/java/retries/RateLimiterTokenBucket.java b/retries/src/main/java/software/amazon/smithy/java/retries/RateLimiterTokenBucket.java index f206b76619..f6b0bb906a 100644 --- a/retries/src/main/java/software/amazon/smithy/java/retries/RateLimiterTokenBucket.java +++ b/retries/src/main/java/software/amazon/smithy/java/retries/RateLimiterTokenBucket.java @@ -226,15 +226,28 @@ void calculateTimeWindow() { } double cubicSuccess(double timestamp) { - var delta = timestamp - this.lastThrottleTime; - return (SCALE_CONSTANT * Math.pow(delta - this.timeWindow, 3)) + this.lastMaxRate; + return computeCubicSuccess(this.lastMaxRate, this.lastThrottleTime, timestamp); } double cubicThrottle(double rateToUse) { - return rateToUse * BETA; + return computeCubicThrottle(rateToUse); } } + // visible for testing + static double computeCubicSuccess(double lastMaxRate, double lastThrottleTime, double timestamp) { + double timeWindow = Math.pow( + (lastMaxRate * (1 - TransientState.BETA)) / TransientState.SCALE_CONSTANT, + 1.0 / 3); + double delta = timestamp - lastThrottleTime; + return (TransientState.SCALE_CONSTANT * Math.pow(delta - timeWindow, 3)) + lastMaxRate; + } + + // visible for testing + static double computeCubicThrottle(double rateToUse) { + return rateToUse * TransientState.BETA; + } + /** * Immutable snapshot of the rate limiter state stored in an {@link AtomicReference}. */ diff --git a/retries/src/main/java/software/amazon/smithy/java/retries/StandardRetryStrategy.java b/retries/src/main/java/software/amazon/smithy/java/retries/StandardRetryStrategy.java index 7363ae81b0..664cf3ecb3 100644 --- a/retries/src/main/java/software/amazon/smithy/java/retries/StandardRetryStrategy.java +++ b/retries/src/main/java/software/amazon/smithy/java/retries/StandardRetryStrategy.java @@ -46,6 +46,7 @@ public static Builder builder() { return new Builder() .maxAttempts(Constants.Standard.MAX_ATTEMPTS) .backoffBaseDelay(Constants.Standard.BASE_DELAY) + .throttlingBackoffBaseDelay(Constants.Standard.THROTTLING_BASE_DELAY) .backoffMaxBackoff(Constants.Standard.MAX_BACKOFF) .retryCost(Constants.Standard.RETRY_COST) .throttlingRetryCost(Constants.Standard.THROTTLING_RETRY_COST) @@ -118,6 +119,17 @@ public Builder backoffBaseDelay(Duration baseDelay) { return this; } + /** + * Set the base delay for exponential backoff on throttling errors. + * + * @param baseDelay the throttling base delay. + * @return the builder. + */ + public Builder throttlingBackoffBaseDelay(Duration baseDelay) { + setThrottlingBackoffBaseDelay(baseDelay); + return this; + } + /** * Set the maximum backoff delay. * diff --git a/retries/src/main/java/software/amazon/smithy/java/retries/TokenBucket.java b/retries/src/main/java/software/amazon/smithy/java/retries/TokenBucket.java index f3ddbf7ae3..1ce2caccdb 100644 --- a/retries/src/main/java/software/amazon/smithy/java/retries/TokenBucket.java +++ b/retries/src/main/java/software/amazon/smithy/java/retries/TokenBucket.java @@ -27,7 +27,7 @@ final class TokenBucket { * Try to acquire a certain number of tokens from this bucket. If there aren't sufficient tokens in this bucket then * {@link AcquireResponse#acquisitionFailed()} returns {@code true}. */ - AcquireResponse tryAcquire(int amountToAcquire, DefaultRetryToken token) { + AcquireResponse tryAcquire(int amountToAcquire) { if (amountToAcquire < 0) { throw new IllegalArgumentException("amountToAcquire cannot be negative"); } @@ -42,8 +42,7 @@ AcquireResponse tryAcquire(int amountToAcquire, DefaultRetryToken token) { currentCapacity = capacity.get(); newCapacity = currentCapacity - amountToAcquire; if (newCapacity < 0) { - var acquisitionFailed = !token.isLongPolling(); - return new AcquireResponse(initialRetryTokens, amountToAcquire, 0, currentCapacity, acquisitionFailed); + return new AcquireResponse(initialRetryTokens, amountToAcquire, 0, currentCapacity, true); } } while (!capacity.compareAndSet(currentCapacity, newCapacity)); diff --git a/retries/src/test/java/software/amazon/smithy/java/retries/ExponentialDelayWithJitterTest.java b/retries/src/test/java/software/amazon/smithy/java/retries/ExponentialDelayWithJitterTest.java index caeeeb6c85..f5baf2c0a7 100644 --- a/retries/src/test/java/software/amazon/smithy/java/retries/ExponentialDelayWithJitterTest.java +++ b/retries/src/test/java/software/amazon/smithy/java/retries/ExponentialDelayWithJitterTest.java @@ -11,14 +11,13 @@ import java.util.Arrays; import java.util.Collection; import java.util.Random; -import java.util.function.Function; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; class ExponentialDelayWithJitterTest { - static final ComputedNextInt MIN_VALUE_RND = new ComputedNextInt(bound -> 0); - static final ComputedNextInt MID_VALUE_RND = new ComputedNextInt(bound -> bound / 2); - static final ComputedNextInt MAX_VALUE_RND = new ComputedNextInt(bound -> bound - 1); + static final ComputedNextDouble MIN_VALUE_RND = new ComputedNextDouble(0.0); + static final ComputedNextDouble MID_VALUE_RND = new ComputedNextDouble(0.5); + static final ComputedNextDouble MAX_VALUE_RND = new ComputedNextDouble(1.0); static final Duration BASE_DELAY = Duration.ofMillis(23); static final Duration MAX_DELAY = Duration.ofSeconds(20); @@ -30,7 +29,7 @@ void testCase(TestCase testCase) { static Collection parameters() { return Arrays.asList( - // --- Using random that returns: bound - 1 + // --- Using random that returns: 1.0 (max jitter) new TestCase() .configureRandom(MAX_VALUE_RND) .givenAttempt(1) @@ -38,28 +37,28 @@ static Collection parameters() { new TestCase() .configureRandom(MAX_VALUE_RND) .givenAttempt(2) - .expectDelayInMs(22), + .expectDelayInMs(23), new TestCase() .configureRandom(MAX_VALUE_RND) .givenAttempt(3) - .expectDelayInMs(45), + .expectDelayInMs(46), new TestCase() .configureRandom(MAX_VALUE_RND) .givenAttempt(5) - .expectDelayInMs(183), + .expectDelayInMs(184), new TestCase() .configureRandom(MAX_VALUE_RND) .givenAttempt(7) - .expectDelayInMs(735), + .expectDelayInMs(736), new TestCase() .configureRandom(MAX_VALUE_RND) .givenAttempt(11) - .expectDelayInMs(11775), + .expectDelayInMs(11776), new TestCase() .configureRandom(MAX_VALUE_RND) .givenAttempt(13) - .expectDelayInMs(19999), - // --- Using random that returns: bound / 2 + .expectDelayInMs(20000), + // --- Using random that returns: 0.5 (mid jitter) new TestCase() .configureRandom(MID_VALUE_RND) .givenAttempt(1) @@ -88,7 +87,7 @@ static Collection parameters() { .configureRandom(MID_VALUE_RND) .givenAttempt(13) .expectDelayInMs(10000), - // --- Using random that returns: 0 + // --- Using random that returns: 0.0 (no jitter) new TestCase() .configureRandom(MIN_VALUE_RND) .givenAttempt(1) @@ -149,16 +148,16 @@ Duration expected() { } } - static class ComputedNextInt extends Random { - final Function compute; + static class ComputedNextDouble extends Random { + final double value; - ComputedNextInt(Function compute) { - this.compute = compute; + ComputedNextDouble(double value) { + this.value = value; } @Override - public int nextInt(int bound) { - return compute.apply(bound); + public double nextDouble() { + return value; } } } diff --git a/retries/src/test/java/software/amazon/smithy/java/retries/RetryStrategyTestCommon.java b/retries/src/test/java/software/amazon/smithy/java/retries/RetryStrategyTestCommon.java index 2365e9b527..1f3fdcad49 100644 --- a/retries/src/test/java/software/amazon/smithy/java/retries/RetryStrategyTestCommon.java +++ b/retries/src/test/java/software/amazon/smithy/java/retries/RetryStrategyTestCommon.java @@ -80,12 +80,13 @@ public static Collection parameters() { .expectedLastCapacityAcquired(0) .expectedCapacityRemaining(10) .build(), - builder("Exhausts tokens with retryable yet succeeds for long polling") - .statuses(createExhaustingRetryableCallStatusesWithFinalSuccess()) - .expectSuccess(true) + builder("Exhausts tokens with retryable yet backs off for long polling") + .statuses(createExhaustingRetryableCallStatuses()) + .expectSuccess(false) .isLongPolling(true) .expectedLastCapacityAcquired(0) - .expectedCapacityRemaining(11) // returns 1 upon success + .expectedCapacityRemaining(10) + .expectDelay(true) .build(), builder("Exhausts tokens with throttling") .statuses(createExhaustingThrottlingCallStatuses()) @@ -93,12 +94,13 @@ public static Collection parameters() { .expectedLastCapacityAcquired(5) .expectedCapacityRemaining(0) .build(), - builder("Exhausts tokens with throttling yet succeeds for long polling") - .statuses(createExhaustingThrottlingCallStatusesWithFinalSuccess()) - .expectSuccess(true) + builder("Exhausts tokens with throttling yet backs off for long polling") + .statuses(createExhaustingThrottlingCallStatusesForLongPolling()) + .expectSuccess(false) .isLongPolling(true) .expectedLastCapacityAcquired(0) - .expectedCapacityRemaining(1) // returns 1 upon success + .expectedCapacityRemaining(0) + .expectDelay(true) .build()); } @@ -119,12 +121,6 @@ static List createExhaustingRetryableCallStatuses() { return result; } - static List createExhaustingRetryableCallStatusesWithFinalSuccess() { - var result = createExhaustingRetryableCallStatuses(); - result.add(CallStatus.SUCCESS); - return result; - } - static List createExhaustingThrottlingCallStatuses() { var result = new ArrayList(); for (var idx = 0; idx < 99; idx++) { @@ -136,10 +132,18 @@ static List createExhaustingThrottlingCallStatuses() { return result; } - static List createExhaustingThrottlingCallStatusesWithFinalSuccess() { - var result = createExhaustingThrottlingCallStatuses(); + // After 99 rounds of (THROTTLED, THROTTLED, SUCCESS), net = 500 - 99*5 = 5. + // Then one more THROTTLED acquires 5 → 0 remaining. + // Then one more THROTTLED cannot be acquired (0 < 5). + static List createExhaustingThrottlingCallStatusesForLongPolling() { + var result = new ArrayList(); + for (var idx = 0; idx < 99; idx++) { + result.add(CallStatus.THROTTLED); + result.add(CallStatus.THROTTLED); + result.add(CallStatus.SUCCESS); + } + result.add(CallStatus.THROTTLED); result.add(CallStatus.THROTTLED); - result.add(CallStatus.SUCCESS); return result; } @@ -150,6 +154,7 @@ static class TestCase { final Integer expectedCapacityAcquired; final Integer expectedCapacityRemaining; final int flags; + final boolean expectDelay; TestCase(TestCaseBuilder builder) { this.name = builder.name; @@ -158,6 +163,7 @@ static class TestCase { this.expectedCapacityAcquired = builder.expectedCapacityAcquired; this.expectedCapacityRemaining = builder.expectedCapacityRemaining; this.flags = builder.flags; + this.expectDelay = builder.expectDelay; } public void run(BaseRetryStrategy strategy) { @@ -182,6 +188,11 @@ public void run(BaseRetryStrategy strategy) { throw new AssertionError("Expected token acquisition failed", e); } token = e.token(); + if (expectDelay) { + if (e.delay() == null || e.delay().isZero()) { + throw new AssertionError("Expected non-zero delay for long-polling backoff"); + } + } if (idx != statuses.size() - 1) { throw new AssertionError("Test case not setup correctly, " + "not all statuses were covered, remaining: " + @@ -226,6 +237,7 @@ static class TestCaseBuilder { Integer expectedCapacityAcquired; Integer expectedCapacityRemaining; int flags = 0; + boolean expectDelay = false; TestCaseBuilder(String name) { this.name = name; @@ -263,6 +275,11 @@ public TestCaseBuilder isLongPolling(boolean isLongPolling) { return this; } + public TestCaseBuilder expectDelay(boolean expectDelay) { + this.expectDelay = expectDelay; + return this; + } + public TestCase build() { return new TestCase(this); } diff --git a/retries/src/test/java/software/amazon/smithy/java/retries/SepCubicTest.java b/retries/src/test/java/software/amazon/smithy/java/retries/SepCubicTest.java new file mode 100644 index 0000000000..7cfdde0186 --- /dev/null +++ b/retries/src/test/java/software/amazon/smithy/java/retries/SepCubicTest.java @@ -0,0 +1,84 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.java.retries; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.within; + +import java.util.List; +import org.junit.jupiter.api.Test; + +/** + * SEP Appendix A: CUBIC Testcases. + * + *

Verifies _CUBICSuccess() and _CUBICThrottle() calculations with BETA=0.7 and SCALE_CONSTANT=0.4. + */ +class SepCubicTest { + private static final double EPSILON = 0.001; + + // SEP test set 1: all success responses, last_max_rate=10, last_throttle_time=5 + @Test + void cubicAllSuccess() { + double lastMaxRate = 10; + double lastThrottleTime = 5; + + assertRate(lastMaxRate, lastThrottleTime, 5, 7.0); + assertRate(lastMaxRate, lastThrottleTime, 6, 9.64893600966); + assertRate(lastMaxRate, lastThrottleTime, 7, 10.000030849917364); + assertRate(lastMaxRate, lastThrottleTime, 8, 10.453284520772092); + assertRate(lastMaxRate, lastThrottleTime, 9, 13.408697022224185); + assertRate(lastMaxRate, lastThrottleTime, 10, 21.26626835427364); + assertRate(lastMaxRate, lastThrottleTime, 11, 36.425998516920465); + } + + // SEP test set 2: mixed success/throttle, last_max_rate=10, last_throttle_time=5 + // State evolves through the sequence. + @Test + void cubicMixedSuccessAndThrottle() { + record Step(String response, double timestamp, double expectedRate) {} + + var steps = List.of( + new Step("success", 5, 7.0), + new Step("success", 6, 9.64893600966), + new Step("throttle", 7, 6.754255206761999), + new Step("throttle", 8, 4.727978644733399), + new Step("success", 9, 6.606547753887045), + new Step("success", 10, 6.763279816944947), + new Step("success", 11, 7.598174833907107), + new Step("success", 12, 11.511232804773524)); + + double lastMaxRate = 10; + double lastThrottleTime = 5; + double previousCalculatedRate = 0; + + for (var step : steps) { + double calculatedRate; + if ("throttle".equals(step.response)) { + // _CUBICThrottle uses the previous calculated_rate as input + calculatedRate = RateLimiterTokenBucket.computeCubicThrottle(previousCalculatedRate); + // After throttle: last_max_rate = previous calculated_rate, last_throttle_time = now + lastMaxRate = previousCalculatedRate; + lastThrottleTime = step.timestamp; + } else { + calculatedRate = RateLimiterTokenBucket.computeCubicSuccess( + lastMaxRate, + lastThrottleTime, + step.timestamp); + } + assertThat(calculatedRate) + .as("response=%s, timestamp=%s", step.response, step.timestamp) + .isCloseTo(step.expectedRate, within(EPSILON)); + previousCalculatedRate = calculatedRate; + } + } + + private void assertRate(double lastMaxRate, double lastThrottleTime, double timestamp, double expected) { + double rate = RateLimiterTokenBucket.computeCubicSuccess(lastMaxRate, lastThrottleTime, timestamp); + assertThat(rate) + .as("timestamp=%s", timestamp) + .isCloseTo(expected, within(EPSILON)); + } +} diff --git a/retries/src/test/java/software/amazon/smithy/java/retries/SepRetryStrategyTest.java b/retries/src/test/java/software/amazon/smithy/java/retries/SepRetryStrategyTest.java new file mode 100644 index 0000000000..a1aaa441c7 --- /dev/null +++ b/retries/src/test/java/software/amazon/smithy/java/retries/SepRetryStrategyTest.java @@ -0,0 +1,390 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package software.amazon.smithy.java.retries; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.time.Duration; +import java.util.Random; +import org.junit.jupiter.api.Test; +import software.amazon.smithy.java.retries.api.AcquireInitialTokenFlags; +import software.amazon.smithy.java.retries.api.AcquireInitialTokenRequest; +import software.amazon.smithy.java.retries.api.RecordSuccessRequest; +import software.amazon.smithy.java.retries.api.RefreshRetryTokenRequest; +import software.amazon.smithy.java.retries.api.RetryInfo; +import software.amazon.smithy.java.retries.api.RetrySafety; +import software.amazon.smithy.java.retries.api.RetryToken; +import software.amazon.smithy.java.retries.api.TokenAcquisitionFailedException; + +/** + * Tests from the SEP Appendix A: Standard Mode Testcases. + * All tests use exponential_base=1 (jitter always returns max value). + */ +class SepRetryStrategyTest { + + // Fixed jitter that always returns 1.0 (exponential_base: 1) + private static final Random FIXED_JITTER = new Random() { + @Override + public double nextDouble() { + return 1.0; + } + }; + + private StandardRetryStrategy strategy() { + return strategy(500, 3, 20.0); + } + + private StandardRetryStrategy strategy(int initialTokens, int maxAttempts, double maxBackoffSeconds) { + var builder = StandardRetryStrategy.builder() + .initialRetryTokens(initialTokens) + .maxAttempts(maxAttempts) + .backoffMaxBackoff(Duration.ofMillis((long) (maxBackoffSeconds * 1000))); + builder.setRandomSupplier(() -> FIXED_JITTER); + return builder.build(); + } + + private RetryToken acquireToken(StandardRetryStrategy s) { + return acquireToken(s, 0); + } + + private RetryToken acquireToken(StandardRetryStrategy s, int flags) { + return s.acquireInitialToken(new AcquireInitialTokenRequest("scope", flags)).token(); + } + + // --- SEP Test: Retry eventually succeeds --- + @Test + void retryEventuallySucceeds() { + var s = strategy(); + var token = acquireToken(s); + + // 500 → retry, quota=486, delay=0.05 + var r1 = s.refreshRetryToken(new RefreshRetryTokenRequest(token, transientError(), null)); + assertThat(r1.delay()).isEqualTo(Duration.ofMillis(50)); + assertQuota(r1.token(), 486); + + // 500 → retry, quota=472, delay=0.1 + var r2 = s.refreshRetryToken(new RefreshRetryTokenRequest(r1.token(), transientError(), null)); + assertThat(r2.delay()).isEqualTo(Duration.ofMillis(100)); + assertQuota(r2.token(), 472); + + // 200 → success, quota=486 + var success = s.recordSuccess(new RecordSuccessRequest(r2.token())); + assertQuota(success.token(), 486); + } + + // --- SEP Test: Fail due to max attempts reached --- + @Test + void failDueToMaxAttemptsReached() { + var s = strategy(); + var token = acquireToken(s); + + // 502 → retry, quota=486, delay=0.05 + var r1 = s.refreshRetryToken(new RefreshRetryTokenRequest(token, transientError(), null)); + assertThat(r1.delay()).isEqualTo(Duration.ofMillis(50)); + assertQuota(r1.token(), 486); + + // 502 → retry, quota=472, delay=0.1 + var r2 = s.refreshRetryToken(new RefreshRetryTokenRequest(r1.token(), transientError(), null)); + assertThat(r2.delay()).isEqualTo(Duration.ofMillis(100)); + assertQuota(r2.token(), 472); + + // 502 → max_attempts_exceeded, quota=472 + assertThatThrownBy(() -> s.refreshRetryToken(new RefreshRetryTokenRequest(r2.token(), transientError(), null))) + .isInstanceOf(TokenAcquisitionFailedException.class) + .satisfies(e -> assertQuota(((TokenAcquisitionFailedException) e).token(), 472)); + } + + // --- SEP Test: Retry Quota reached after a single retry --- + @Test + void retryQuotaReachedAfterSingleRetry() { + var s = strategy(14, 3, 20.0); + var token = acquireToken(s); + + // 500 → retry, quota=0, delay=0.05 + var r1 = s.refreshRetryToken(new RefreshRetryTokenRequest(token, transientError(), null)); + assertThat(r1.delay()).isEqualTo(Duration.ofMillis(50)); + assertQuota(r1.token(), 0); + + // 500 → retry_quota_exceeded, quota=0 + assertThatThrownBy(() -> s.refreshRetryToken(new RefreshRetryTokenRequest(r1.token(), transientError(), null))) + .isInstanceOf(TokenAcquisitionFailedException.class) + .satisfies(e -> assertQuota(((TokenAcquisitionFailedException) e).token(), 0)); + } + + // --- SEP Test: No retries at all if retry quota is 0 --- + @Test + void noRetriesIfRetryQuotaIsZero() { + var s = strategy(0, 3, 20.0); + var token = acquireToken(s); + + // 500 → retry_quota_exceeded, quota=0 + assertThatThrownBy(() -> s.refreshRetryToken(new RefreshRetryTokenRequest(token, transientError(), null))) + .isInstanceOf(TokenAcquisitionFailedException.class) + .satisfies(e -> assertQuota(((TokenAcquisitionFailedException) e).token(), 0)); + } + + // --- SEP Test: Verifying exponential backoff timing --- + @Test + void exponentialBackoffTiming() { + var s = strategy(500, 5, 20.0); + var token = acquireToken(s); + + // i=0: delay=0.05 + var r1 = s.refreshRetryToken(new RefreshRetryTokenRequest(token, transientError(), null)); + assertThat(r1.delay()).isEqualTo(Duration.ofMillis(50)); + assertQuota(r1.token(), 486); + + // i=1: delay=0.1 + var r2 = s.refreshRetryToken(new RefreshRetryTokenRequest(r1.token(), transientError(), null)); + assertThat(r2.delay()).isEqualTo(Duration.ofMillis(100)); + assertQuota(r2.token(), 472); + + // i=2: delay=0.2 + var r3 = s.refreshRetryToken(new RefreshRetryTokenRequest(r2.token(), transientError(), null)); + assertThat(r3.delay()).isEqualTo(Duration.ofMillis(200)); + assertQuota(r3.token(), 458); + + // i=3: delay=0.4 + var r4 = s.refreshRetryToken(new RefreshRetryTokenRequest(r3.token(), transientError(), null)); + assertThat(r4.delay()).isEqualTo(Duration.ofMillis(400)); + assertQuota(r4.token(), 444); + + // max_attempts_exceeded + assertThatThrownBy(() -> s.refreshRetryToken(new RefreshRetryTokenRequest(r4.token(), transientError(), null))) + .isInstanceOf(TokenAcquisitionFailedException.class) + .satisfies(e -> assertQuota(((TokenAcquisitionFailedException) e).token(), 444)); + } + + // --- SEP Test: Verify max backoff time --- + @Test + void maxBackoffTime() { + var s = strategy(500, 5, 0.2); + var token = acquireToken(s); + + // i=0: delay=0.05 + var r1 = s.refreshRetryToken(new RefreshRetryTokenRequest(token, transientError(), null)); + assertThat(r1.delay()).isEqualTo(Duration.ofMillis(50)); + + // i=1: delay=0.1 + var r2 = s.refreshRetryToken(new RefreshRetryTokenRequest(r1.token(), transientError(), null)); + assertThat(r2.delay()).isEqualTo(Duration.ofMillis(100)); + + // i=2: delay=0.2 (capped by max_backoff) + var r3 = s.refreshRetryToken(new RefreshRetryTokenRequest(r2.token(), transientError(), null)); + assertThat(r3.delay()).isEqualTo(Duration.ofMillis(200)); + + // i=3: delay=0.2 (still capped) + var r4 = s.refreshRetryToken(new RefreshRetryTokenRequest(r3.token(), transientError(), null)); + assertThat(r4.delay()).isEqualTo(Duration.ofMillis(200)); + } + + // --- SEP Test: Retry Stops After Retry Quota Exhaustion --- + @Test + void retryStopsAfterRetryQuotaExhaustion() { + var s = strategy(20, 5, 20.0); + var token = acquireToken(s); + + // 500 → retry, quota=6, delay=0.05 + var r1 = s.refreshRetryToken(new RefreshRetryTokenRequest(token, transientError(), null)); + assertThat(r1.delay()).isEqualTo(Duration.ofMillis(50)); + assertQuota(r1.token(), 6); + + // 502 → retry_quota_exceeded, quota=6 + assertThatThrownBy(() -> s.refreshRetryToken(new RefreshRetryTokenRequest(r1.token(), transientError(), null))) + .isInstanceOf(TokenAcquisitionFailedException.class) + .satisfies(e -> assertQuota(((TokenAcquisitionFailedException) e).token(), 6)); + } + + // --- SEP Test: Retry quota Recovery After Successful Responses --- + @Test + void retryQuotaRecoveryAfterSuccess() { + var s = strategy(30, 5, 20.0); + + // First invocation: 500 → 502 → 200 + var token = acquireToken(s); + var r1 = s.refreshRetryToken(new RefreshRetryTokenRequest(token, transientError(), null)); + assertThat(r1.delay()).isEqualTo(Duration.ofMillis(50)); + assertQuota(r1.token(), 16); + + var r2 = s.refreshRetryToken(new RefreshRetryTokenRequest(r1.token(), transientError(), null)); + assertThat(r2.delay()).isEqualTo(Duration.ofMillis(100)); + assertQuota(r2.token(), 2); + + var success1 = s.recordSuccess(new RecordSuccessRequest(r2.token())); + assertQuota(success1.token(), 16); // returns 14 + + // Second invocation: 500 → 200 + var token2 = acquireToken(s); + var r3 = s.refreshRetryToken(new RefreshRetryTokenRequest(token2, transientError(), null)); + assertThat(r3.delay()).isEqualTo(Duration.ofMillis(50)); + assertQuota(r3.token(), 2); + + var success2 = s.recordSuccess(new RecordSuccessRequest(r3.token())); + assertQuota(success2.token(), 16); // returns 14 + } + + // --- SEP Test: Throttling Error Token Bucket Drain (5 tokens) and Backoff Duration (1000ms) --- + @Test + void throttlingErrorUsesHigherBaseDelay() { + var s = strategy(); + var token = acquireToken(s); + + // Throttling → retry, quota=495, delay=1.0 (1000ms base) + var r1 = s.refreshRetryToken(new RefreshRetryTokenRequest(token, throttlingError(), null)); + assertThat(r1.delay()).isEqualTo(Duration.ofMillis(1000)); + assertQuota(r1.token(), 495); + + // 200 → success, quota=500 + var success = s.recordSuccess(new RecordSuccessRequest(r1.token())); + assertQuota(success.token(), 500); + } + + // --- SEP Test: Long-Polling Backoff After Transient Error When Token Bucket Empty --- + @Test + void longPollingBackoffAfterTransientErrorWhenTokenBucketEmpty() { + var s = strategy(0, 3, 20.0); + var token = acquireToken(s, AcquireInitialTokenFlags.IS_LONG_POLLING); + + // 500 → retry_quota_exceeded with delay=0.05 + assertThatThrownBy(() -> s.refreshRetryToken(new RefreshRetryTokenRequest(token, transientError(), null))) + .isInstanceOf(TokenAcquisitionFailedException.class) + .satisfies(e -> { + var tafe = (TokenAcquisitionFailedException) e; + assertQuota(tafe.token(), 0); + assertThat(tafe.delay()).isEqualTo(Duration.ofMillis(50)); + }); + } + + // --- SEP Test: Long-Polling Backoff After Throttling Error When Token Bucket Empty --- + @Test + void longPollingBackoffAfterThrottlingErrorWhenTokenBucketEmpty() { + var s = strategy(0, 3, 20.0); + var token = acquireToken(s, AcquireInitialTokenFlags.IS_LONG_POLLING); + + // Throttling → retry_quota_exceeded with delay=1.0 (throttling base) + assertThatThrownBy(() -> s.refreshRetryToken(new RefreshRetryTokenRequest(token, throttlingError(), null))) + .isInstanceOf(TokenAcquisitionFailedException.class) + .satisfies(e -> { + var tafe = (TokenAcquisitionFailedException) e; + assertQuota(tafe.token(), 0); + assertThat(tafe.delay()).isEqualTo(Duration.ofMillis(1000)); + }); + } + + // --- SEP Test: Long-Polling Max Attempts Exceeded Must NOT Delay --- + @Test + void longPollingMaxAttemptsExceededMustNotDelay() { + var s = strategy(500, 2, 20.0); + var token = acquireToken(s, AcquireInitialTokenFlags.IS_LONG_POLLING); + + // 500 → retry, delay=0.05 + var r1 = s.refreshRetryToken(new RefreshRetryTokenRequest(token, transientError(), null)); + assertThat(r1.delay()).isEqualTo(Duration.ofMillis(50)); + + // 500 → max_attempts_exceeded, no delay + assertThatThrownBy(() -> s.refreshRetryToken(new RefreshRetryTokenRequest(r1.token(), transientError(), null))) + .isInstanceOf(TokenAcquisitionFailedException.class) + .satisfies(e -> { + var tafe = (TokenAcquisitionFailedException) e; + assertThat(tafe.delay()).isEqualTo(Duration.ZERO); + }); + } + + // --- SEP Test: Honor x-amz-retry-after Header --- + @Test + void honorXAmzRetryAfterHeader() { + var s = strategy(); + var token = acquireToken(s); + + // 500 with x-amz-retry-after: 1500 → delay=1.5s + var r1 = s.refreshRetryToken( + new RefreshRetryTokenRequest(token, transientError(), Duration.ofMillis(1500))); + assertThat(r1.delay()).isEqualTo(Duration.ofMillis(1500)); + assertQuota(r1.token(), 486); + + var success = s.recordSuccess(new RecordSuccessRequest(r1.token())); + assertQuota(success.token(), 500); + } + + // --- SEP Test: x-amz-retry-after minimum is exponential backoff duration --- + @Test + void xAmzRetryAfterMinimumIsExponentialBackoff() { + var s = strategy(); + var token = acquireToken(s); + + // x-amz-retry-after: 0 → clamped to t_i=0.05 + var r1 = s.refreshRetryToken( + new RefreshRetryTokenRequest(token, transientError(), Duration.ofMillis(0))); + assertThat(r1.delay()).isEqualTo(Duration.ofMillis(50)); + assertQuota(r1.token(), 486); + } + + // --- SEP Test: x-amz-retry-after maximum is 5+exponential backoff duration --- + @Test + void xAmzRetryAfterMaximumIs5PlusExponentialBackoff() { + var s = strategy(); + var token = acquireToken(s); + + // x-amz-retry-after: 10000 → clamped to 5+t_i = 5.05s + var r1 = s.refreshRetryToken( + new RefreshRetryTokenRequest(token, transientError(), Duration.ofMillis(10000))); + assertThat(r1.delay()).isEqualTo(Duration.ofMillis(5050)); + assertQuota(r1.token(), 486); + } + + // --- SEP Test: Invalid x-amz-retry-after Falls Back to Exponential Backoff --- + // (Invalid values are handled at the HTTP layer; at the strategy level, null suggestedDelay = fallback) + @Test + void invalidRetryAfterFallsBackToExponentialBackoff() { + var s = strategy(); + var token = acquireToken(s); + + // null suggestedDelay → uses exponential backoff = 0.05 + var r1 = s.refreshRetryToken(new RefreshRetryTokenRequest(token, transientError(), null)); + assertThat(r1.delay()).isEqualTo(Duration.ofMillis(50)); + assertQuota(r1.token(), 486); + } + + private void assertQuota(RetryToken token, int expected) { + var t = (DefaultRetryToken) token; + assertThat(t.capacityRemaining()).isEqualTo(expected); + } + + private static Throwable transientError() { + return new TestException(RetrySafety.YES, false); + } + + private static Throwable throttlingError() { + return new TestException(RetrySafety.YES, true); + } + + private static class TestException extends RuntimeException implements RetryInfo { + private final RetrySafety safety; + private final boolean throttle; + + TestException(RetrySafety safety, boolean throttle) { + super("test"); + this.safety = safety; + this.throttle = throttle; + } + + @Override + public RetrySafety isRetrySafe() { + return safety; + } + + @Override + public boolean isThrottle() { + return throttle; + } + + @Override + public Duration retryAfter() { + return null; + } + } +}