package auth import ( "context" "encoding/json" "errors" "net/http" "strconv" "strings" "sync" "time" internalconfig "github.com/router-for-me/CLIProxyAPI/v7/internal/config" "github.com/router-for-me/CLIProxyAPI/v7/internal/registry" log "github.com/sirupsen/logrus" ) // RefreshEvaluator allows runtime state to override refresh decisions. type RefreshEvaluator interface { ShouldRefresh(now time.Time, auth *Auth) bool } const ( refreshCheckInterval = 5 * time.Second refreshMaxConcurrency = 16 refreshPendingBackoff = time.Minute refreshFailureBackoff = 5 * time.Minute // refreshIneffectiveBackoff throttles refresh attempts when an executor returns // success but the auth still evaluates as needing refresh (e.g. token expiry // wasn't updated). Without this guard, the auto-refresh loop can tight-loop and // burn CPU at idle. refreshIneffectiveBackoff = 30 * time.Second quotaBackoffBase = time.Second quotaBackoffMax = 30 * time.Minute transientErrorCooldown = time.Minute ) // StartAutoRefresh launches a background loop that evaluates auth freshness // every few seconds and triggers refresh operations when required. // Only one loop is kept alive; starting a new one cancels the previous run. func (m *Manager) StartAutoRefresh(parent context.Context, interval time.Duration) { if interval <= 0 { interval = refreshCheckInterval } m.mu.Lock() cancelPrev := m.refreshCancel m.refreshCancel = nil m.refreshLoop = nil m.mu.Unlock() if cancelPrev != nil { cancelPrev() } ctx, cancelCtx := context.WithCancel(parent) workers := refreshMaxConcurrency if cfg, ok := m.runtimeConfig.Load().(*internalconfig.Config); ok && cfg != nil && cfg.AuthAutoRefreshWorkers > 0 { workers = cfg.AuthAutoRefreshWorkers } loop := newAuthAutoRefreshLoop(m, interval, workers) m.mu.Lock() m.refreshCancel = cancelCtx m.refreshLoop = loop m.mu.Unlock() loop.rebuild(time.Now()) go loop.run(ctx) } // StopAutoRefresh cancels the background refresh loop, if running. // It also stops the selector if it implements StoppableSelector. func (m *Manager) StopAutoRefresh() { m.mu.Lock() cancel := m.refreshCancel m.refreshCancel = nil m.refreshLoop = nil m.mu.Unlock() if cancel != nil { cancel() } // Stop selector if it implements StoppableSelector (e.g., SessionAffinitySelector) if stoppable, ok := m.selector.(StoppableSelector); ok { stoppable.Stop() } } func (m *Manager) queueRefreshReschedule(authID string) { if m == nil || authID == "" { return } m.mu.RLock() loop := m.refreshLoop m.mu.RUnlock() if loop == nil { return } loop.queueReschedule(authID) } func (m *Manager) queueRefreshUnschedule(authID string) { if m == nil || authID == "" { return } m.mu.RLock() loop := m.refreshLoop m.mu.RUnlock() if loop == nil { return } loop.remove(authID) } func (m *Manager) shouldRefresh(a *Auth, now time.Time) bool { if a == nil { return false } if hasUnauthorizedAuthFailure(a) { return false } if !a.NextRefreshAfter.IsZero() && now.Before(a.NextRefreshAfter) { return false } if evaluator, ok := a.Runtime.(RefreshEvaluator); ok && evaluator != nil { return evaluator.ShouldRefresh(now, a) } lastRefresh := a.LastRefreshedAt if lastRefresh.IsZero() { if ts, ok := authLastRefreshTimestamp(a); ok { lastRefresh = ts } } expiry, hasExpiry := a.ExpirationTime() if interval := authPreferredInterval(a); interval > 0 { if hasExpiry && !expiry.IsZero() { if !expiry.After(now) { return true } if expiry.Sub(now) <= interval { return true } } if lastRefresh.IsZero() { return true } return now.Sub(lastRefresh) >= interval } provider := strings.ToLower(a.Provider) lead := ProviderRefreshLead(provider, a.Runtime) if lead == nil { return false } if *lead <= 0 { if hasExpiry && !expiry.IsZero() { return now.After(expiry) } return false } if hasExpiry && !expiry.IsZero() { return time.Until(expiry) <= *lead } if !lastRefresh.IsZero() { return now.Sub(lastRefresh) >= *lead } return true } func authPreferredInterval(a *Auth) time.Duration { if a == nil { return 0 } if d := durationFromMetadata(a.Metadata, "refresh_interval_seconds", "refreshIntervalSeconds", "refresh_interval", "refreshInterval"); d > 0 { return d } if d := durationFromAttributes(a.Attributes, "refresh_interval_seconds", "refreshIntervalSeconds", "refresh_interval", "refreshInterval"); d > 0 { return d } return 0 } func durationFromMetadata(meta map[string]any, keys ...string) time.Duration { if len(meta) == 0 { return 0 } for _, key := range keys { if val, ok := meta[key]; ok { if dur := parseDurationValue(val); dur > 0 { return dur } } } return 0 } func durationFromAttributes(attrs map[string]string, keys ...string) time.Duration { if len(attrs) == 0 { return 0 } for _, key := range keys { if val, ok := attrs[key]; ok { if dur := parseDurationString(val); dur > 0 { return dur } } } return 0 } func parseDurationValue(val any) time.Duration { switch v := val.(type) { case time.Duration: if v <= 0 { return 0 } return v case int: if v <= 0 { return 0 } return time.Duration(v) * time.Second case int32: if v <= 0 { return 0 } return time.Duration(v) * time.Second case int64: if v <= 0 { return 0 } return time.Duration(v) * time.Second case uint: if v == 0 { return 0 } return time.Duration(v) * time.Second case uint32: if v == 0 { return 0 } return time.Duration(v) * time.Second case uint64: if v == 0 { return 0 } return time.Duration(v) * time.Second case float32: if v <= 0 { return 0 } return time.Duration(float64(v) * float64(time.Second)) case float64: if v <= 0 { return 0 } return time.Duration(v * float64(time.Second)) case json.Number: if i, err := v.Int64(); err == nil { if i <= 0 { return 0 } return time.Duration(i) * time.Second } if f, err := v.Float64(); err == nil && f > 0 { return time.Duration(f * float64(time.Second)) } case string: return parseDurationString(v) } return 0 } func parseDurationString(raw string) time.Duration { s := strings.TrimSpace(raw) if s == "" { return 0 } if dur, err := time.ParseDuration(s); err == nil && dur > 0 { return dur } if secs, err := strconv.ParseFloat(s, 64); err == nil && secs > 0 { return time.Duration(secs * float64(time.Second)) } return 0 } func authLastRefreshTimestamp(a *Auth) (time.Time, bool) { if a == nil { return time.Time{}, false } if a.Metadata != nil { if ts, ok := lookupMetadataTime(a.Metadata, "last_refresh", "lastRefresh", "last_refreshed_at", "lastRefreshedAt"); ok { return ts, true } } if a.Attributes != nil { for _, key := range []string{"last_refresh", "lastRefresh", "last_refreshed_at", "lastRefreshedAt"} { if val := strings.TrimSpace(a.Attributes[key]); val != "" { if ts, ok := parseTimeValue(val); ok { return ts, true } } } } return time.Time{}, false } func lookupMetadataTime(meta map[string]any, keys ...string) (time.Time, bool) { for _, key := range keys { if val, ok := meta[key]; ok { if ts, ok1 := parseTimeValue(val); ok1 { return ts, true } } } return time.Time{}, false } func (m *Manager) markRefreshPending(id string, now time.Time) bool { m.mu.Lock() auth, ok := m.auths[id] if !ok || auth == nil { m.mu.Unlock() return false } if !auth.NextRefreshAfter.IsZero() && now.Before(auth.NextRefreshAfter) { m.mu.Unlock() return false } auth.NextRefreshAfter = now.Add(refreshPendingBackoff) m.auths[id] = auth m.mu.Unlock() m.queueRefreshReschedule(id) return true } type authRefreshLock struct { mu sync.Mutex } func authAccessToken(auth *Auth) string { if token := authMetadataString(auth, "access_token"); token != "" { return token } return authMetadataString(auth, "accessToken") } func authHasRefreshCredential(auth *Auth) bool { if authMetadataString(auth, "refresh_token") != "" { return true } return authMetadataString(auth, "refreshToken") != "" } func clearUnauthorizedModelStates(auth *Auth, now time.Time) []string { if auth == nil || len(auth.ModelStates) == 0 { return nil } var resumed []string for model, state := range auth.ModelStates { if state == nil || state.LastError == nil { continue } if state.LastError.StatusCode() != http.StatusUnauthorized && !strings.EqualFold(state.LastError.Code, "unauthorized") { continue } resetModelState(state, now) resumed = append(resumed, model) } if len(resumed) > 0 { updateAggregatedAvailability(auth, now) } return resumed } // tryRefreshExecutionAuthAfterUnauthorized refreshes OAuth credentials once for // either a local auth or an ephemeral Home dispatch auth. func (m *Manager) tryRefreshExecutionAuthAfterUnauthorized(ctx context.Context, executor ProviderExecutor, auth *Auth, execErr error, alreadyTried bool, homeDispatch bool) (*Auth, bool, error) { if !homeDispatch { refreshed, ok := m.tryRefreshAfterUnauthorized(ctx, auth, execErr, alreadyTried) return refreshed, ok, nil } if m == nil || executor == nil || auth == nil || alreadyTried || execErr == nil { return auth, false, nil } if !isUnauthorizedError(execErr) || auth.AuthKind() != AuthKindOAuth { return auth, false, nil } log.Debugf("unauthorized Home response for %s (%s), refreshing credentials before redispatch", auth.Provider, auth.ID) target := auth.Clone() updated, errRefresh := executor.Refresh(ctx, target) if errRefresh != nil { log.Debugf("Home credential refresh before redispatch failed for %s (%s)", auth.Provider, auth.ID) return auth, false, errRefresh } if updated == nil { updated = target } if updated.ID == "" { updated.ID = auth.ID } if updated.Index == "" { updated.Index = auth.Index } if updated.Provider == "" { updated.Provider = auth.Provider } if updated.Runtime == nil { updated.Runtime = auth.Runtime } preserveHomeRoutingAttributes(updated, auth) prepared, errPrepare := m.prepareHomeAuthSnapshot(ctx, executor, updated) if errPrepare != nil { return auth, false, errPrepare } preserveHomeRoutingAttributes(prepared, auth) return prepared, true, nil } // RefreshHomeSelectionAfterUnauthorized refreshes the credential snapshot that // received a 401, or reuses a newer token already installed on the selection. func (m *Manager) RefreshHomeSelectionAfterUnauthorized(ctx context.Context, selection *HomeDispatchSelection, failedAuth *Auth) (*Auth, bool, error) { if m == nil || selection == nil { return nil, false, nil } current := selection.CloneAuth() if failedAuth == nil { failedAuth = current } if current != nil && failedAuth != nil && current.ID == failedAuth.ID { currentToken := authAccessToken(current) failedToken := authAccessToken(failedAuth) if currentToken != "" && failedToken != "" && currentToken != failedToken { prepared, errPrepare := m.prepareHomeAuthSnapshot(ctx, selection.Executor, current) if errPrepare != nil { return current, false, errPrepare } preserveHomeRoutingAttributes(prepared, current) m.replaceHomeSelectionAuth(selection, prepared) return selection.CloneAuth(), true, nil } } refreshed, okRefresh, errRefresh := m.tryRefreshExecutionAuthAfterUnauthorized(ctx, selection.Executor, failedAuth, &Error{HTTPStatus: http.StatusUnauthorized, Message: "upstream unauthorized"}, false, true) if errRefresh != nil || !okRefresh { return current, false, errRefresh } m.replaceHomeSelectionAuth(selection, refreshed) updated := selection.CloneAuth() if updated == nil { return nil, false, &Error{Code: "auth_not_found", Message: "refreshed Home auth is unavailable", HTTPStatus: http.StatusServiceUnavailable} } return updated, true, nil } // tryRefreshAfterUnauthorized refreshes local OAuth credentials once after a // 401 so the current auth can be retried before fallback/suspend. func (m *Manager) tryRefreshAfterUnauthorized(ctx context.Context, auth *Auth, execErr error, alreadyTried bool) (*Auth, bool) { if m == nil || auth == nil || alreadyTried || execErr == nil { return auth, false } // Request-scoped failures describe this request, not stale credentials. // Refreshing would turn a direct error response into an implicit retry. if isRequestScopedError(execErr) { return auth, false } if !isUnauthorizedError(execErr) || !authHasRefreshCredential(auth) { return auth, false } log.Debugf("unauthorized response for %s (%s), refreshing credentials before fallback", auth.Provider, auth.ID) refreshed, errRefresh := m.refreshAuthForRequest(ctx, auth.ID, authAccessToken(auth)) if errRefresh != nil || refreshed == nil { log.Debugf("credential refresh before fallback failed for %s (%s): %v", auth.Provider, auth.ID, errRefresh) return auth, false } return refreshed, true } func (m *Manager) refreshAuth(ctx context.Context, id string) { _, _ = m.refreshAuthForRequest(ctx, id, "") } // refreshAuthForRequest performs a synchronous credential refresh for the given auth. // failedAccessToken lets concurrent callers reuse a refresh that already replaced the // access token that produced the unauthorized response. func (m *Manager) refreshAuthForRequest(ctx context.Context, id, failedAccessToken string) (*Auth, error) { if m == nil { return nil, errors.New("auth manager is nil") } if ctx == nil { ctx = context.Background() } id = strings.TrimSpace(id) if id == "" { return nil, errors.New("auth id is empty") } lockValue, _ := m.refreshLocks.LoadOrStore(id, &authRefreshLock{}) lock, _ := lockValue.(*authRefreshLock) if lock == nil { lock = &authRefreshLock{} m.refreshLocks.Store(id, lock) } lock.mu.Lock() defer lock.mu.Unlock() m.mu.RLock() auth := m.auths[id] var exec ProviderExecutor if auth != nil { // Use the same effective provider key as request execution so OpenAI-compat // auths registered under namespaced keys still resolve for refresh. exec = m.executors[executorKeyFromAuth(auth)] } m.mu.RUnlock() if auth == nil || exec == nil { return nil, errors.New("auth or executor not found") } // Another request may already have refreshed this credential. if failedAccessToken != "" { if currentToken := authAccessToken(auth); currentToken != "" && currentToken != failedAccessToken { return auth.Clone(), nil } } cloned := auth.Clone() updated, err := exec.Refresh(ctx, cloned) if err != nil && errors.Is(err, context.Canceled) { log.Debugf("refresh canceled for %s, %s", auth.Provider, auth.ID) return nil, err } log.Debugf("refreshed %s, %s, %v", auth.Provider, auth.ID, err) now := time.Now() if err != nil { unauthorized := isUnauthorizedError(err) shouldReschedule := false m.mu.Lock() if current := m.auths[id]; current != nil { current.LastError = refreshErrorFromError(err) if unauthorized { current.NextRefreshAfter = time.Time{} current.Unavailable = true current.Status = StatusError current.StatusMessage = "unauthorized" } else { current.NextRefreshAfter = now.Add(refreshFailureBackoff) } m.auths[id] = current shouldReschedule = true if m.scheduler != nil { m.scheduler.upsertAuth(current.Clone()) } } m.mu.Unlock() if shouldReschedule { m.queueRefreshReschedule(id) } return nil, err } if updated == nil { updated = cloned } // Preserve runtime created by the executor during Refresh. // If executor didn't set one, fall back to the previous runtime. if updated.Runtime == nil { updated.Runtime = auth.Runtime } updated.LastRefreshedAt = now updated.NextRefreshAfter = time.Time{} updated.LastError = nil updated.StatusMessage = "" updated.Unavailable = false if updated.Status == StatusError { updated.Status = StatusActive } updated.UpdatedAt = now modelsToResume := clearUnauthorizedModelStates(updated, now) if m.shouldRefresh(updated, now) { updated.NextRefreshAfter = now.Add(refreshIneffectiveBackoff) } saved, errUpdate := m.Update(ctx, updated) for _, model := range modelsToResume { registry.GetGlobalRegistry().ResumeClientModel(id, model) } if errUpdate != nil { log.Debugf("persist refreshed auth %s (%s) failed: %v", auth.Provider, auth.ID, errUpdate) } if saved != nil { return saved, nil } return updated.Clone(), nil }