fix(retry): start refill loop when budget enabled via UpdateConfig

The refill goroutine started only when the budget was enabled at
construction. Enabling it later through UpdateConfig left a bucket that
never refilled.

Start the refill loop from UpdateConfig, guarded by sync.Once so it
starts at most once, sharing the existing shutdown lifecycle.
This commit is contained in:
2026-08-14 01:42:04 +01:00
parent 6226717aad
commit 77447bd353
2 changed files with 94 additions and 1 deletions
+24 -1
View File
@@ -21,6 +21,8 @@ type RetryBudget struct {
logger *libpack_logger.Logger logger *libpack_logger.Logger
ctx context.Context ctx context.Context
cancel context.CancelFunc cancel context.CancelFunc
refillOnce sync.Once // ensures refillLoop starts at most once, whether from construction or a later UpdateConfig
refillStarts atomic.Int32 // counts startRefillLoop executions; used by tests to assert no duplicate refiller
// Statistics // Statistics
totalAttempts atomic.Int64 totalAttempts atomic.Int64
@@ -58,12 +60,25 @@ func NewRetryBudgetWithContext(ctx context.Context, config RetryBudgetConfig, lo
// Start refill goroutine // Start refill goroutine
if rb.enabled { if rb.enabled {
go rb.refillLoop() rb.startRefillLoop()
} }
return rb return rb
} }
// startRefillLoop starts the background refill goroutine, at most once,
// regardless of how many times it is called from construction or UpdateConfig.
// Shutdown is terminal: once rb.ctx is cancelled, a later call here still
// starts refillLoop, but it exits immediately on the cancelled context
// without ever refilling, so enabling the budget again after Shutdown does
// not bring back a working refiller.
func (rb *RetryBudget) startRefillLoop() {
rb.refillOnce.Do(func() {
rb.refillStarts.Add(1)
go rb.refillLoop()
})
}
// AllowRetry checks if a retry is allowed based on the current budget // AllowRetry checks if a retry is allowed based on the current budget
func (rb *RetryBudget) AllowRetry() bool { func (rb *RetryBudget) AllowRetry() bool {
rb.totalAttempts.Add(1) rb.totalAttempts.Add(1)
@@ -210,6 +225,14 @@ func (rb *RetryBudget) UpdateConfig(config RetryBudgetConfig) {
rb.currentTokens.Store(rb.maxTokens) rb.currentTokens.Store(rb.maxTokens)
rb.configMu.Unlock() rb.configMu.Unlock()
// Enabling via UpdateConfig must (re)start the refill loop, otherwise a
// budget constructed disabled and later enabled would never refill.
// startRefillLoop is idempotent, so re-enabling an already-running budget
// is a no-op here.
if config.Enabled {
rb.startRefillLoop()
}
if rb.logger != nil { if rb.logger != nil {
rb.logger.Info(&libpack_logger.LogMessage{ rb.logger.Info(&libpack_logger.LogMessage{
Message: "Retry budget configuration updated", Message: "Retry budget configuration updated",
+70
View File
@@ -360,3 +360,73 @@ func TestRetryBudget_ConcurrentUpdateConfig(t *testing.T) {
close(stop) close(stop)
wg.Wait() wg.Wait()
} }
// TestRetryBudget_UpdateConfigEnablesRefill covers the latent defect where a
// budget constructed disabled, then enabled via UpdateConfig, never started
// its refill goroutine: tokens would be handed out once and never replenish.
func TestRetryBudget_UpdateConfigEnablesRefill(t *testing.T) {
rb := NewRetryBudget(RetryBudgetConfig{
TokensPerSecond: 100.0, // fast refill so the poll loop below stays short
MaxTokens: 10,
Enabled: false,
}, nil)
defer rb.Shutdown()
// No refill goroutine should be running yet: enabling later must start it.
rb.UpdateConfig(RetryBudgetConfig{
TokensPerSecond: 100.0,
MaxTokens: 10,
Enabled: true,
})
// Drain the bucket to zero.
for i := 0; i < 10; i++ {
assert.True(t, rb.AllowRetry())
}
assert.False(t, rb.AllowRetry(), "bucket should be exhausted after draining")
assert.Equal(t, int64(0), rb.currentTokens.Load())
// Poll with a deadline instead of a fixed sleep: refill runs every 100ms,
// so tokens must appear well within the deadline if the loop is running.
deadline := time.Now().Add(2 * time.Second)
refilled := false
for time.Now().Before(deadline) {
if rb.currentTokens.Load() > 0 {
refilled = true
break
}
time.Sleep(10 * time.Millisecond)
}
assert.True(t, refilled, "tokens never refilled after UpdateConfig enabled the budget")
}
// TestRetryBudget_UpdateConfigDoesNotLeakRefiller asserts that enabling a
// budget multiple times (once at construction, then repeatedly via
// concurrent UpdateConfig calls) starts at most one refill goroutine. It
// uses the package-internal refillStarts counter, incremented exactly once
// inside the sync.Once guarded by startRefillLoop, giving a deterministic
// assertion instead of a goroutine-count heuristic.
func TestRetryBudget_UpdateConfigDoesNotLeakRefiller(t *testing.T) {
rb := NewRetryBudget(RetryBudgetConfig{
TokensPerSecond: 50.0,
MaxTokens: 10,
Enabled: true, // refill goroutine already started at construction
}, nil)
defer rb.Shutdown()
assert.Equal(t, int32(1), rb.refillStarts.Load(), "construction with Enabled: true should start exactly one refiller")
var wg sync.WaitGroup
for i := 0; i < 20; i++ {
wg.Add(1)
go func() {
defer wg.Done()
// Re-enabling an already-enabled budget must not start a second
// refiller.
rb.UpdateConfig(RetryBudgetConfig{TokensPerSecond: 50.0, MaxTokens: 10, Enabled: true})
}()
}
wg.Wait()
assert.Equal(t, int32(1), rb.refillStarts.Load(), "concurrent re-enables must not start a second refiller")
}