mirror of
https://github.com/lukaszraczylo/traefikoidc.git
synced 2026-11-18 10:16:05 +00:00
Compare commits
1
Commits
984fd1c08f
...
v1.0.14
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ae4ccaa89d |
+77
-43
@@ -15,7 +15,13 @@ import (
|
|||||||
// It implements request coalescing, rate limiting, and circuit breaking
|
// It implements request coalescing, rate limiting, and circuit breaking
|
||||||
// specifically for token refresh operations.
|
// specifically for token refresh operations.
|
||||||
type RefreshCoordinator struct {
|
type RefreshCoordinator struct {
|
||||||
inFlightRefreshes map[string]*refreshOperation
|
// inFlightRefreshes maps tokenHash -> *refreshOperation. sync.Map is used
|
||||||
|
// instead of a plain map + RWMutex so concurrent refreshes do not
|
||||||
|
// serialize on a single global lock. Under Yaegi the previous
|
||||||
|
// refreshMutex.Lock() was held for tens of milliseconds per request due
|
||||||
|
// to interpreter overhead on the work inside the critical section,
|
||||||
|
// causing dozens of goroutines to stack up on it and pin one CPU core.
|
||||||
|
inFlightRefreshes sync.Map
|
||||||
cleanupTimers map[string]*time.Timer
|
cleanupTimers map[string]*time.Timer
|
||||||
sessionRefreshAttempts map[string]*refreshAttemptTracker
|
sessionRefreshAttempts map[string]*refreshAttemptTracker
|
||||||
circuitBreaker *RefreshCircuitBreaker
|
circuitBreaker *RefreshCircuitBreaker
|
||||||
@@ -25,7 +31,6 @@ type RefreshCoordinator struct {
|
|||||||
config RefreshCoordinatorConfig
|
config RefreshCoordinatorConfig
|
||||||
wg sync.WaitGroup
|
wg sync.WaitGroup
|
||||||
attemptsMutex sync.RWMutex
|
attemptsMutex sync.RWMutex
|
||||||
refreshMutex sync.RWMutex
|
|
||||||
cleanupTimerMu sync.Mutex
|
cleanupTimerMu sync.Mutex
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -130,7 +135,7 @@ func NewRefreshCoordinator(config RefreshCoordinatorConfig, logger *Logger) *Ref
|
|||||||
}
|
}
|
||||||
|
|
||||||
rc := &RefreshCoordinator{
|
rc := &RefreshCoordinator{
|
||||||
inFlightRefreshes: make(map[string]*refreshOperation),
|
// inFlightRefreshes is a sync.Map; zero value is ready to use.
|
||||||
sessionRefreshAttempts: make(map[string]*refreshAttemptTracker),
|
sessionRefreshAttempts: make(map[string]*refreshAttemptTracker),
|
||||||
config: config,
|
config: config,
|
||||||
metrics: &RefreshMetrics{},
|
metrics: &RefreshMetrics{},
|
||||||
@@ -227,13 +232,28 @@ func (rc *RefreshCoordinator) getOrCreateOperation(
|
|||||||
tokenHash string,
|
tokenHash string,
|
||||||
refreshToken string,
|
refreshToken string,
|
||||||
) (*refreshOperation, bool, error) {
|
) (*refreshOperation, bool, error) {
|
||||||
rc.refreshMutex.Lock()
|
// Speculatively construct the operation we WOULD register if we win the
|
||||||
defer rc.refreshMutex.Unlock()
|
// race. Allocating here keeps the LoadOrStore call below atomic and
|
||||||
|
// avoids any global lock — under Yaegi the previous map+RWMutex design
|
||||||
|
// held the write lock long enough (tens of ms per call) that concurrent
|
||||||
|
// refreshes on the same coordinator serialized into a queue that grew
|
||||||
|
// without bound. See struct comment on inFlightRefreshes.
|
||||||
|
candidate := &refreshOperation{
|
||||||
|
refreshToken: refreshToken,
|
||||||
|
done: make(chan struct{}),
|
||||||
|
startTime: time.Now(),
|
||||||
|
waiterCount: 1,
|
||||||
|
}
|
||||||
|
|
||||||
// Check for existing operation while holding the lock
|
if existing, loaded := rc.inFlightRefreshes.LoadOrStore(tokenHash, candidate); loaded {
|
||||||
if existingOp, exists := rc.inFlightRefreshes[tokenHash]; exists {
|
existingOp, ok := existing.(*refreshOperation)
|
||||||
|
if !ok {
|
||||||
|
// Defensive: anything stored here is always *refreshOperation, but
|
||||||
|
// keep the typed assert so a programming error elsewhere doesn't
|
||||||
|
// surface as a confusing panic in an interpreter frame.
|
||||||
|
return nil, false, fmt.Errorf("inFlightRefreshes corrupt: unexpected type %T", existing)
|
||||||
|
}
|
||||||
if existingOp.refreshToken == refreshToken {
|
if existingOp.refreshToken == refreshToken {
|
||||||
// Join existing operation
|
|
||||||
atomic.AddInt32(&existingOp.waiterCount, 1)
|
atomic.AddInt32(&existingOp.waiterCount, 1)
|
||||||
return existingOp, false, nil
|
return existingOp, false, nil
|
||||||
}
|
}
|
||||||
@@ -241,41 +261,60 @@ func (rc *RefreshCoordinator) getOrCreateOperation(
|
|||||||
return nil, false, fmt.Errorf("refresh token mismatch")
|
return nil, false, fmt.Errorf("refresh token mismatch")
|
||||||
}
|
}
|
||||||
|
|
||||||
// No existing operation - check if we can create a new one
|
// We won the race and registered `candidate`. Apply gates now. If any
|
||||||
// All checks happen while holding the lock to prevent races
|
// gate fails we must remove our entry from the map and signal failure
|
||||||
|
// to any joiners that snuck in between LoadOrStore and now.
|
||||||
|
if err := rc.applyLeaderGates(sessionID); err != nil {
|
||||||
|
rc.failCandidate(tokenHash, candidate, err)
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
|
||||||
// Check and record refresh attempt for rate limiting
|
// Reserve concurrent slot via CAS — without the old global lock we can
|
||||||
|
// no longer rely on mutex-mediated check-then-increment. If we lose the
|
||||||
|
// CAS race we retry; if the limit has since been reached we back out.
|
||||||
|
for {
|
||||||
|
current := atomic.LoadInt32(&rc.metrics.currentInFlightRefreshes)
|
||||||
|
if int(current) >= rc.config.MaxConcurrentRefreshes {
|
||||||
|
err := fmt.Errorf("maximum concurrent refresh operations reached")
|
||||||
|
rc.failCandidate(tokenHash, candidate, err)
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
if atomic.CompareAndSwapInt32(&rc.metrics.currentInFlightRefreshes, current, current+1) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return candidate, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// applyLeaderGates runs the rate-limit, cooldown, and memory-pressure checks
|
||||||
|
// that previously ran under the global refreshMutex. Only the leader (the
|
||||||
|
// goroutine that just registered the operation) runs them; joiners share the
|
||||||
|
// leader's outcome via operation.done.
|
||||||
|
func (rc *RefreshCoordinator) applyLeaderGates(sessionID string) error {
|
||||||
rc.recordRefreshAttempt(sessionID)
|
rc.recordRefreshAttempt(sessionID)
|
||||||
if rc.isInCooldown(sessionID) {
|
if rc.isInCooldown(sessionID) {
|
||||||
atomic.AddInt64(&rc.metrics.cooldownsTriggered, 1)
|
atomic.AddInt64(&rc.metrics.cooldownsTriggered, 1)
|
||||||
return nil, false, fmt.Errorf("refresh attempts exceeded for session, in cooldown period")
|
return fmt.Errorf("refresh attempts exceeded for session, in cooldown period")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check memory pressure
|
|
||||||
if rc.config.EnableMemoryPressureDetection && rc.isUnderMemoryPressure() {
|
if rc.config.EnableMemoryPressureDetection && rc.isUnderMemoryPressure() {
|
||||||
atomic.AddInt64(&rc.metrics.memoryPressureEvents, 1)
|
atomic.AddInt64(&rc.metrics.memoryPressureEvents, 1)
|
||||||
return nil, false, fmt.Errorf("system under memory pressure, refresh denied")
|
return fmt.Errorf("system under memory pressure, refresh denied")
|
||||||
}
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// Check and reserve concurrent refresh slot atomically
|
// failCandidate removes the leader's just-registered operation from the
|
||||||
current := atomic.LoadInt32(&rc.metrics.currentInFlightRefreshes)
|
// in-flight map and signals the error to any joiners by recording the result
|
||||||
if int(current) >= rc.config.MaxConcurrentRefreshes {
|
// and closing the done channel. This keeps the (nil, false, err) return path
|
||||||
return nil, false, fmt.Errorf("maximum concurrent refresh operations reached")
|
// equivalent to the pre-sync.Map version: callers see the error directly,
|
||||||
}
|
// joiners see it via operation.done.
|
||||||
|
func (rc *RefreshCoordinator) failCandidate(tokenHash string, op *refreshOperation, err error) {
|
||||||
// Reserve the slot - we're still holding the lock so this is safe
|
rc.inFlightRefreshes.Delete(tokenHash)
|
||||||
atomic.AddInt32(&rc.metrics.currentInFlightRefreshes, 1)
|
op.mutex.Lock()
|
||||||
|
op.result = &refreshResult{err: err}
|
||||||
// Create and register new operation
|
op.mutex.Unlock()
|
||||||
operation := &refreshOperation{
|
close(op.done)
|
||||||
refreshToken: refreshToken,
|
|
||||||
done: make(chan struct{}),
|
|
||||||
startTime: time.Now(),
|
|
||||||
waiterCount: 1,
|
|
||||||
}
|
|
||||||
rc.inFlightRefreshes[tokenHash] = operation
|
|
||||||
|
|
||||||
return operation, true, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// executeRefreshAsync performs the actual refresh operation asynchronously
|
// executeRefreshAsync performs the actual refresh operation asynchronously
|
||||||
@@ -367,16 +406,11 @@ func (rc *RefreshCoordinator) scheduleDelayedCleanup(tokenHash string) {
|
|||||||
|
|
||||||
// performCleanup removes the operation from the in-flight map.
|
// performCleanup removes the operation from the in-flight map.
|
||||||
// Idempotent: only decrements the in-flight counter if an entry was actually
|
// Idempotent: only decrements the in-flight counter if an entry was actually
|
||||||
// removed. This guards against any future path accidentally calling cleanup
|
// removed. LoadAndDelete is atomic so any concurrent failCandidate or repeat
|
||||||
// twice for the same tokenHash (which would corrupt the refresh budget).
|
// cleanup call will see exactly one removal — the budget cannot be corrupted
|
||||||
|
// by double-decrement.
|
||||||
func (rc *RefreshCoordinator) performCleanup(tokenHash string) {
|
func (rc *RefreshCoordinator) performCleanup(tokenHash string) {
|
||||||
rc.refreshMutex.Lock()
|
if _, existed := rc.inFlightRefreshes.LoadAndDelete(tokenHash); existed {
|
||||||
_, existed := rc.inFlightRefreshes[tokenHash]
|
|
||||||
if existed {
|
|
||||||
delete(rc.inFlightRefreshes, tokenHash)
|
|
||||||
}
|
|
||||||
rc.refreshMutex.Unlock()
|
|
||||||
if existed {
|
|
||||||
atomic.AddInt32(&rc.metrics.currentInFlightRefreshes, -1)
|
atomic.AddInt32(&rc.metrics.currentInFlightRefreshes, -1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user