vibe-proxy/backend/sdk/cliproxy/service_home.go
2026-08-24 00:10:41 +02:00

802 lines
22 KiB
Go

package cliproxy
import (
"context"
"errors"
"fmt"
"strings"
"sync"
"sync/atomic"
"time"
internalconfig "github.com/router-for-me/CLIProxyAPI/v7/internal/config"
"github.com/router-for-me/CLIProxyAPI/v7/internal/home"
"github.com/router-for-me/CLIProxyAPI/v7/internal/homeplugins"
"github.com/router-for-me/CLIProxyAPI/v7/internal/logging"
"github.com/router-for-me/CLIProxyAPI/v7/internal/redisqueue"
"github.com/router-for-me/CLIProxyAPI/v7/internal/util"
"github.com/router-for-me/CLIProxyAPI/v7/internal/watcher/diff"
coreauth "github.com/router-for-me/CLIProxyAPI/v7/sdk/cliproxy/auth"
"github.com/router-for-me/CLIProxyAPI/v7/sdk/cliproxy/executionregistry"
"github.com/router-for-me/CLIProxyAPI/v7/sdk/config"
log "github.com/sirupsen/logrus"
)
type homeSubscriberSupervisor struct {
cancel context.CancelFunc
done chan struct{}
publisherMu sync.Mutex
publisherDone <-chan struct{}
}
func (s *homeSubscriberSupervisor) setPublisherCompletion(done <-chan struct{}) {
if s == nil {
return
}
s.publisherMu.Lock()
s.publisherDone = done
s.publisherMu.Unlock()
}
func (s *homeSubscriberSupervisor) publisherCompletion() <-chan struct{} {
if s == nil {
return nil
}
s.publisherMu.Lock()
defer s.publisherMu.Unlock()
return s.publisherDone
}
type homeConfigWorkQueue struct {
mu sync.Mutex
items [][]byte
wake chan struct{}
}
func newHomeConfigWorkQueue() *homeConfigWorkQueue {
return &homeConfigWorkQueue{wake: make(chan struct{}, 1)}
}
func (q *homeConfigWorkQueue) enqueue(raw []byte) {
if q == nil {
return
}
item := append([]byte(nil), raw...)
q.mu.Lock()
q.items = append(q.items, item)
q.mu.Unlock()
select {
case q.wake <- struct{}{}:
default:
}
}
func (q *homeConfigWorkQueue) dequeue(ctx context.Context) ([]byte, bool) {
if q == nil || ctx == nil {
return nil, false
}
for {
if ctx.Err() != nil {
return nil, false
}
q.mu.Lock()
if ctx.Err() != nil {
q.mu.Unlock()
return nil, false
}
if len(q.items) > 0 {
item := q.items[0]
q.items[0] = nil
q.items = q.items[1:]
q.mu.Unlock()
return item, true
}
q.mu.Unlock()
select {
case <-ctx.Done():
return nil, false
case <-q.wake:
}
}
}
type homeLogForwarder interface {
Bind(*home.Client)
Deactivate(*home.Client)
Stop()
}
var startHomeLogForwarder = func(queueSize int) homeLogForwarder {
return logging.StartHomeAppLogForwarder(queueSize)
}
func (s *Service) applyHomeOverlay(remoteCfg *config.Config) {
if errApply := s.applyHomeOverlayContext(context.Background(), remoteCfg); errApply != nil {
log.Warnf("failed to apply home config payload: %v", errApply)
}
}
func (s *Service) applyHomeOverlayContext(ctx context.Context, remoteCfg *config.Config) error {
return s.applyHomeOverlayWithClient(ctx, remoteCfg, nil)
}
func (s *Service) applyHomeOverlayWithClient(ctx context.Context, remoteCfg *config.Config, client *home.Client) error {
work, errStage := s.stageHomeOverlayWithClient(ctx, remoteCfg, client)
if errStage != nil {
return errStage
}
if ctx != nil {
if errContext := ctx.Err(); errContext != nil {
return errContext
}
}
if work.config != nil {
if !s.applyConfigUpdateWithAuthSynthesis(ctx, work.config, true) {
return context.Canceled
}
work.committed = true
}
if errFinalize := s.finalizeHomePluginWork(ctx, client, work); errFinalize != nil {
return errFinalize
}
return nil
}
func (s *Service) stageHomeOverlayWithClient(ctx context.Context, remoteCfg *config.Config, client *home.Client) (*homePluginFinalization, error) {
work := &homePluginFinalization{}
if s == nil || remoteCfg == nil {
return work, nil
}
if ctx == nil {
ctx = context.Background()
}
if errContext := ctx.Err(); errContext != nil {
return nil, errContext
}
s.cfgMu.RLock()
baseCfg := s.cfg
s.cfgMu.RUnlock()
if baseCfg == nil {
return work, nil
}
merged := *remoteCfg
merged.Host = baseCfg.Host
merged.Port = baseCfg.Port
merged.TLS = baseCfg.TLS
merged.Home = baseCfg.Home
storeAuth := merged.Plugins.StoreAuth
forceHomeRuntimeConfig(&merged)
syncCfg := merged
syncCfg.Plugins.StoreAuth = storeAuth
logHomeConfigChanges(baseCfg, &merged)
report, syncKey, didSync, errSync := s.syncHomePluginsWithClient(ctx, &syncCfg, client)
if errSync != nil {
return nil, fmt.Errorf("sync home plugins: %w", errSync)
}
if errContext := ctx.Err(); errContext != nil {
return nil, errContext
}
if didSync {
if errLoad := homeplugins.MarkLoadResults(&report, s.pluginHost); errLoad != nil {
return nil, fmt.Errorf("load home plugins: %w", errLoad)
}
}
if strings.TrimSpace(report.Task) != "" {
work.syncKey = syncKey
work.markSynced = true
if strings.TrimSpace(merged.Home.NodeID) != "" {
work.statusWork = append(work.statusWork, homePluginStatusWork{cfg: &merged, report: report})
}
}
taskWork, errTasks := s.stageHomePluginTasksWithClient(ctx, &merged, client)
if errTasks != nil {
return nil, fmt.Errorf("stage home plugin tasks: %w", errTasks)
}
work.taskWork = append(work.taskWork, taskWork...)
if errContext := ctx.Err(); errContext != nil {
return nil, errContext
}
work.config = &merged
return work, nil
}
func (s *Service) commitHomeConfig(lifetimeCtx, homeCtx context.Context, generation uint64, work *homePluginFinalization) bool {
if s == nil || work == nil || work.config == nil {
return false
}
s.homeConfigCommitMu.Lock()
defer s.homeConfigCommitMu.Unlock()
if !s.homeLifetimeActive(homeCtx, lifetimeCtx, generation) {
return false
}
if s.homeConfigCommitHook != nil {
s.homeConfigCommitHook()
}
if !s.homeLifetimeActive(homeCtx, lifetimeCtx, generation) {
return false
}
commit := s.commitConfigUpdate(work.config)
if commit.cfg == nil {
return false
}
work.config = commit.cfg
work.configCommit = commit
work.committed = true
return true
}
func (s *Service) homeLifetimeActive(homeCtx, lifetimeCtx context.Context, generation uint64) bool {
if s == nil || homeCtx.Err() != nil || lifetimeCtx.Err() != nil {
return false
}
s.homeMu.Lock()
active := s.homeGeneration == generation
s.homeMu.Unlock()
return active
}
func (s *Service) finalizeHomePluginWorkUntilDone(ctx, homeCtx context.Context, generation uint64, client *home.Client, work *homePluginFinalization, publish func() bool) error {
stopClose := closeHomeClientOnCancellation(ctx, client)
defer stopClose()
for {
if errContext := ctx.Err(); errContext != nil {
return errContext
}
s.homeOwnershipMu.Lock()
if !s.homeLifetimeActive(homeCtx, ctx, generation) {
s.homeOwnershipMu.Unlock()
return context.Canceled
}
errFinalize := s.finalizeHomePluginWork(ctx, client, work)
if errFinalize == nil && (publish == nil || publish()) {
s.homeOwnershipMu.Unlock()
return nil
}
s.homeOwnershipMu.Unlock()
if errFinalize == nil {
return context.Canceled
}
log.WithError(errFinalize).Warn("failed to finalize home plugins; retrying")
timer := time.NewTimer(homeSubscriberPreAckRetryBackoff)
select {
case <-ctx.Done():
timer.Stop()
return ctx.Err()
case <-timer.C:
}
}
}
func closeHomeClientOnCancellation(ctx context.Context, client *home.Client) func() {
if ctx == nil || client == nil {
return func() {}
}
stop := make(chan struct{})
go func() {
select {
case <-ctx.Done():
client.Close()
case <-stop:
}
}()
return func() { close(stop) }
}
func logHomeConfigChanges(oldCfg, newCfg *config.Config) {
if oldCfg == nil || newCfg == nil || !newCfg.Home.Enabled || (!oldCfg.Debug && !newCfg.Debug) {
return
}
details := diff.BuildConfigChangeDetails(oldCfg, newCfg)
if len(details) == 0 {
return
}
if newCfg.Debug && !log.IsLevelEnabled(log.DebugLevel) {
util.SetLogLevel(newCfg)
}
log.Debugf("home config changes detected:")
for _, detail := range details {
log.Debugf(" %s", detail)
}
}
func (s *Service) startHomeUsageForwarder(ctx context.Context, client *home.Client) {
if s == nil || client == nil {
return
}
if ctx == nil {
ctx = context.Background()
}
sleep := func(d time.Duration) bool {
if d <= 0 {
return true
}
timer := time.NewTimer(d)
defer timer.Stop()
select {
case <-ctx.Done():
return false
case <-timer.C:
return true
}
}
go func() {
for {
select {
case <-ctx.Done():
return
default:
}
if !client.HeartbeatOK() {
if !sleep(time.Second) {
return
}
continue
}
items := redisqueue.PopOldest(64)
if len(items) == 0 {
if !sleep(500 * time.Millisecond) {
return
}
continue
}
for i := range items {
if errPush := client.LPushUsage(ctx, items[i]); errPush != nil {
for j := i; j < len(items); j++ {
redisqueue.Enqueue(items[j])
}
if !sleep(time.Second) {
return
}
break
}
}
}
}()
}
func applyHomeObservationBarrier(registry *executionregistry.Registry, revision int64) {
if registry != nil {
registry.ObserveBarrier(revision)
}
}
func applyHomeInFlightPublisherConfig(manager *coreauth.Manager, cfg internalconfig.CredentialInFlightConfig) error {
publisherCfg, errConfig := coreauth.HomeInFlightPublisherConfigFromConfig(cfg)
if errConfig != nil {
return errConfig
}
if manager != nil {
manager.ApplyHomeInFlightPublisherConfig(publisherCfg)
}
return nil
}
func (s *Service) startHomeSubscriber(ctx context.Context) {
if s == nil {
return
}
s.cfgMu.RLock()
cfg := s.cfg
s.cfgMu.RUnlock()
if cfg == nil || !cfg.Home.Enabled {
return
}
parentCtx := ctx
if parentCtx == nil {
parentCtx = context.Background()
}
s.homeLifecycleMu.Lock()
defer s.homeLifecycleMu.Unlock()
if previousSupervisor := s.homeSupervisor; previousSupervisor != nil {
s.homeConfigCommitMu.Lock()
previousSupervisor.cancel()
s.homeConfigCommitMu.Unlock()
<-previousSupervisor.done
}
if !s.drainDetachedHomeLifetime(parentCtx) {
return
}
if parentCtx.Err() != nil {
return
}
homeCtx, cancel := context.WithCancel(parentCtx)
done := make(chan struct{})
s.homeMu.Lock()
s.homeGeneration++
generation := s.homeGeneration
s.homeCancel = cancel
s.homeMu.Unlock()
supervisor := &homeSubscriberSupervisor{cancel: cancel, done: done}
s.homeSupervisor = supervisor
go s.runHomeSubscriber(homeCtx, parentCtx, cfg.Home, generation, supervisor)
}
func (s *Service) drainDetachedHomeLifetime(parentCtx context.Context) bool {
s.homeMu.Lock()
previousCancel := s.homeCancel
previousClient := s.homeClient
previousRegistry := s.homeRegistry
previousBundle := s.homeDispatchBundle
previousDrainBound := s.homeDrainBound
previousForwarder := s.homeLogForwarder
previousForwarderClient := s.homeLogForwarderClient
s.homeCancel = nil
s.homeClient = nil
s.homeRegistry = nil
s.homeDispatchBundle = nil
s.homeDrainBound = 0
s.homeLogForwarderClient = nil
s.homeMu.Unlock()
if s.coreManager != nil {
s.coreManager.ClearHomeDispatchBundle(previousBundle)
}
home.ClearCurrentIf(previousClient)
if previousCancel != nil {
previousCancel()
}
if previousForwarder != nil && previousForwarderClient == previousClient {
previousForwarder.Deactivate(previousClient)
}
if previousRegistry != nil {
if previousDrainBound <= 0 {
previousDrainBound = internalconfig.CredentialConcurrencyConfig{}.WithDefaults().CPACancelBound
}
drainCtx, cancelDrain := context.WithTimeout(context.WithoutCancel(parentCtx), previousDrainBound)
errDrain := previousRegistry.Drain(drainCtx)
cancelDrain()
if errDrain != nil {
if previousClient != nil {
previousClient.Close()
}
if parentCtx.Err() == nil {
log.WithError(errDrain).Error("failed to drain replaced Home execution registry")
s.cancelServiceRun()
}
return false
}
}
if previousClient != nil {
previousClient.Close()
}
return true
}
func (s *Service) runHomeSubscriber(homeCtx context.Context, parentCtx context.Context, homeCfg internalconfig.HomeConfig, generation uint64, supervisor *homeSubscriberSupervisor) {
defer func() {
s.homeMu.Lock()
if s.homeGeneration == generation {
s.homeCancel = nil
}
s.homeMu.Unlock()
close(supervisor.done)
}()
var previousClient *home.Client
registry := executionregistry.New()
cancelBound := atomic.Int64{}
cancelBound.Store(int64(internalconfig.CredentialConcurrencyConfig{}.WithDefaults().CPACancelBound))
releaseFlusher := home.NewReleaseFlusher(nil, nil)
registry.SetReleaseSink(releaseFlusher.MarkDirty)
defer func() {
registry.SetReleaseSink(nil)
drainBound := time.Duration(cancelBound.Load())
if drainBound <= 0 {
drainBound = internalconfig.CredentialConcurrencyConfig{}.WithDefaults().CPACancelBound
}
drainCtx, cancelDrain := context.WithTimeout(context.WithoutCancel(parentCtx), drainBound)
errDrain := registry.Drain(drainCtx)
cancelDrain()
if errDrain != nil && !errors.Is(errDrain, executionregistry.ErrRegistryClosed) && parentCtx.Err() == nil {
log.WithError(errDrain).Error("failed to drain detached Home execution registry")
s.cancelServiceRun()
}
}()
for homeCtx.Err() == nil {
supervisor.setPublisherCompletion(nil)
client := previousClient
if client == nil {
client = home.New(homeCfg)
} else {
client = client.NewLifetime()
}
client.SetManagedLifetime(true)
releaseCtx, releaseCancel := context.WithCancel(context.WithoutCancel(homeCtx))
releaseFlusher.SetConfigProvider(client.LimiterConfig)
releaseFlusher.SetSender(client.PushConcurrencyRelease)
releaseDone := make(chan struct{})
go func() {
defer close(releaseDone)
releaseFlusher.Run(releaseCtx)
}()
lifetimeCtx, lifetimeCancel := context.WithCancel(homeCtx)
queue := newHomeConfigWorkQueue()
ready := make(chan struct{})
var readyOnce sync.Once
var published atomic.Bool
workerDone := make(chan struct{})
go func() {
defer close(workerDone)
s.runHomeConfigWorkerWithSupervisor(lifetimeCtx, homeCtx, generation, client, registry, queue, ready, &published, &cancelBound, supervisor)
}()
errRun := client.RunConfigSubscriberLifetime(lifetimeCtx, func(raw []byte) error {
parsed, errParse := config.ParseConfigBytes(raw)
if errParse != nil {
log.Warnf("failed to parse home config payload: %v", errParse)
return errParse
}
if errSetLifecycle := client.SetLifecycleConfig(parsed.CredentialConcurrency); errSetLifecycle != nil {
log.Warnf("failed to apply Home lifecycle config: %v", errSetLifecycle)
return errSetLifecycle
}
if errPublisherConfig := applyHomeInFlightPublisherConfig(s.coreManager, parsed.CredentialInFlight); errPublisherConfig != nil {
log.Warnf("failed to apply Home in-flight publisher config: %v", errPublisherConfig)
return errPublisherConfig
}
applyHomeObservationBarrier(registry, parsed.CredentialConcurrency.ObservationBarrierRevision)
cancelBound.Store(int64(parsed.CredentialConcurrency.WithDefaults().CPACancelBound))
queue.enqueue(raw)
return nil
}, func() {
readyOnce.Do(func() { close(ready) })
})
lifetimeCancel()
<-workerDone
if publisherDone := supervisor.publisherCompletion(); publisherDone != nil {
<-publisherDone
}
s.detachHomeSubscriberLifetime(client, registry)
retry := errRun != nil && homeCtx.Err() == nil
if retry {
releaseCancel()
<-releaseDone
client.Close()
settleBound := time.Duration(cancelBound.Load())
settleCtx, cancelSettle := context.WithTimeout(context.WithoutCancel(parentCtx), settleBound)
errPending := registry.WaitPending(settleCtx)
cancelSettle()
if errPending != nil {
log.WithError(errPending).Error("failed to settle pending Home dispatches before subscriber replacement")
s.cancelServiceRun()
return
}
legacyProtocol := home.IsLegacyMembershipProtocolError(errRun)
if legacyProtocol {
client.EnableLegacyMembership()
}
if client.AmbiguousDispatch() || home.IsMembershipTakeoverUnavailableError(errRun) || legacyProtocol || client.LegacyMembership() {
registry.SetReleaseSink(nil)
drainCtx, cancelDrain := context.WithTimeout(context.WithoutCancel(parentCtx), settleBound)
errDrain := registry.Drain(drainCtx)
cancelDrain()
if errDrain != nil {
log.WithError(errDrain).Error("failed to drain Home executions after unsafe subscriber replacement")
s.cancelServiceRun()
return
}
client.SuppressTakeover()
registry = executionregistry.New()
releaseFlusher = home.NewReleaseFlusher(nil, nil)
registry.SetReleaseSink(releaseFlusher.MarkDirty)
}
log.WithError(errRun).Warn("home config subscription lifetime ended")
if !published.Load() && !waitForHomeSubscriberRetry(homeCtx, homeSubscriberPreAckRetryBackoff) {
return
}
previousClient = client
continue
}
drainBound := time.Duration(cancelBound.Load())
drainCtx, cancelDrain := context.WithTimeout(context.WithoutCancel(parentCtx), drainBound)
errDrain := registry.Drain(drainCtx)
var errFlush error
if errDrain == nil {
errFlush = releaseFlusher.Flush(drainCtx)
}
cancelDrain()
releaseCancel()
<-releaseDone
client.Close()
if errDrain != nil {
if parentCtx.Err() == nil {
log.WithError(errDrain).Error("failed to drain Home execution registry")
s.cancelServiceRun()
}
return
}
if errFlush != nil {
if parentCtx.Err() == nil {
log.WithError(errFlush).Error("failed to flush Home concurrency releases")
s.cancelServiceRun()
}
return
}
return
}
}
func (s *Service) runHomeConfigWorker(lifetimeCtx, homeCtx context.Context, generation uint64, client *home.Client, registry *executionregistry.Registry, queue *homeConfigWorkQueue, ready <-chan struct{}, published *atomic.Bool, cancelBound *atomic.Int64) {
s.runHomeConfigWorkerWithSupervisor(lifetimeCtx, homeCtx, generation, client, registry, queue, ready, published, cancelBound, nil)
}
func (s *Service) runHomeConfigWorkerWithSupervisor(lifetimeCtx, homeCtx context.Context, generation uint64, client *home.Client, registry *executionregistry.Registry, queue *homeConfigWorkQueue, ready <-chan struct{}, published *atomic.Bool, cancelBound *atomic.Int64, supervisor *homeSubscriberSupervisor) {
select {
case <-lifetimeCtx.Done():
return
case <-ready:
}
for {
if lifetimeCtx.Err() != nil {
return
}
raw, ok := queue.dequeue(lifetimeCtx)
if !ok {
return
}
if lifetimeCtx.Err() != nil {
return
}
var work *homePluginFinalization
for {
if lifetimeCtx.Err() != nil {
return
}
parsed, errParse := config.ParseConfigBytes(raw)
if errParse == nil {
work, errParse = s.stageHomeOverlayWithClient(lifetimeCtx, parsed, client)
}
if errParse == nil {
break
}
if lifetimeCtx.Err() != nil {
return
}
log.WithError(errParse).Warn("failed to stage home config; retrying")
if !waitForHomeSubscriberRetry(lifetimeCtx, homeSubscriberPreAckRetryBackoff) {
return
}
}
var publish func() bool
if !published.Load() {
publish = func() bool {
s.homeMu.Lock()
defer s.homeMu.Unlock()
if homeCtx.Err() != nil || lifetimeCtx.Err() != nil || s.homeGeneration != generation {
return false
}
s.homeClient = client
s.homeRegistry = registry
s.homeDrainBound = time.Duration(cancelBound.Load())
if s.coreManager != nil {
s.homeDispatchBundle = s.coreManager.PublishHomeDispatch(client, registry, generation)
}
home.SetCurrent(client)
if s.homeLogForwarder == nil {
s.homeLogForwarder = startHomeLogForwarder(0)
}
s.homeLogForwarder.Bind(client)
s.homeLogForwarderClient = client
published.Store(true)
return true
}
}
if s.homeConfigStageHook != nil {
s.homeConfigStageHook()
}
if !s.commitHomeConfig(lifetimeCtx, homeCtx, generation, work) {
return
}
if s.homeConfigRuntimeHook != nil {
s.homeConfigRuntimeHook()
}
if !s.homeLifetimeActive(homeCtx, lifetimeCtx, generation) || !s.applyConfigRuntime(lifetimeCtx, work.configCommit, true) {
return
}
if errFinalize := s.finalizeHomePluginWorkUntilDone(lifetimeCtx, homeCtx, generation, client, work, publish); errFinalize != nil {
if !errors.Is(errFinalize, context.Canceled) {
log.WithError(errFinalize).Warn("home plugin finalization ended")
}
return
}
if publish != nil {
s.startHomeInFlightPublisher(lifetimeCtx, client, registry, supervisor)
s.startHomeUsageForwarder(lifetimeCtx, client)
}
}
}
func (s *Service) startHomeInFlightPublisher(ctx context.Context, client *home.Client, registry *executionregistry.Registry, supervisor *homeSubscriberSupervisor) {
if s == nil || s.coreManager == nil {
return
}
done := make(chan struct{})
if supervisor != nil {
supervisor.setPublisherCompletion(done)
}
go func() {
defer close(done)
s.coreManager.StartHomeInFlightPublisher(ctx, client, registry)
}()
}
func waitForHomeSubscriberRetry(ctx context.Context, delay time.Duration) bool {
timer := time.NewTimer(delay)
defer timer.Stop()
select {
case <-ctx.Done():
return false
case <-timer.C:
return true
}
}
func (s *Service) detachHomeSubscriberLifetime(client *home.Client, registry *executionregistry.Registry) {
if s == nil {
return
}
s.homeMu.Lock()
var bundle *coreauth.HomeDispatchBundle
if s.homeClient == client && s.homeRegistry == registry {
bundle = s.homeDispatchBundle
s.homeClient = nil
s.homeRegistry = nil
s.homeDispatchBundle = nil
s.homeDrainBound = 0
}
forwarder := s.homeLogForwarder
if s.homeLogForwarderClient == client {
s.homeLogForwarderClient = nil
} else {
forwarder = nil
}
s.homeMu.Unlock()
if s.coreManager != nil {
s.coreManager.ClearHomeDispatchBundle(bundle)
}
home.ClearCurrentIf(client)
if forwarder != nil {
forwarder.Deactivate(client)
}
}
func (s *Service) cancelServiceRun() {
if s == nil {
return
}
s.homeMu.Lock()
cancel := s.runCancel
if cancel == nil {
cancel = s.homeCancel
}
s.homeMu.Unlock()
if cancel != nil {
cancel()
}
}