shoal/vibe-proxy
Watch
4
0
Fork
You've already forked vibe-proxy
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add projects

Browse source
This commit is contained in:
Alois 2026-08-24 00:10:41 +02:00
commit 8b607dd700
Signed by: alois
SSH key fingerprint: SHA256:GBzT2DXvAuGV9XIV5W3WrzVpjU54FThmxHXdbz95J24
1802 changed files with 503346 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

151
backend/internal/logging/cpa_trace.go Normal file
View file

@ -0,0 +1,151 @@
package logging
import (
"strings"
"sync"
"time"
"github.com/gin-gonic/gin"
)
// CPATraceIDHeader is the downstream response header used to correlate requests with selected credentials.
const CPATraceIDHeader = "X-CPA-TRACE-ID"
const ginCPATraceStateKey = "__cpa_trace_state__"
// FormatCPATraceID builds a CPA trace ID from the selection time, auth index, and request ID.
func FormatCPATraceID(selectedAt time.Time, authIndex, requestID string) string {
authIndex = strings.TrimSpace(authIndex)
requestID = strings.TrimSpace(requestID)
if selectedAt.IsZero() || authIndex == "" || requestID == "" {
return ""
}
return selectedAt.Format("20060102150405") + "-" + authIndex + "-" + requestID
}
type cpaTraceState struct {
mu sync.RWMutex
traceID string
}
func (s *cpaTraceState) set(traceID string) {
if s == nil {
return
}
s.mu.Lock()
s.traceID = strings.TrimSpace(traceID)
s.mu.Unlock()
}
func (s *cpaTraceState) get() string {
if s == nil {
return ""
}
s.mu.RLock()
traceID := s.traceID
s.mu.RUnlock()
return traceID
}
func ginCPATraceState(c *gin.Context) *cpaTraceState {
if c == nil {
return nil
}
if value, exists := c.Get(ginCPATraceStateKey); exists {
if state, ok := value.(*cpaTraceState); ok && state != nil {
return state
}
}
state := &cpaTraceState{}
c.Set(ginCPATraceStateKey, state)
return state
}
// GinCPATraceIDCallback returns a callback that is safe to invoke after the Gin context is released.
func GinCPATraceIDCallback(c *gin.Context) func(string) {
state := ginCPATraceState(c)
if state == nil {
return nil
}
requestID := GetGinRequestID(c)
if requestID == "" && c.Request != nil {
requestID = GetRequestID(c.Request.Context())
}
requestID = strings.TrimSpace(requestID)
if requestID == "" {
return nil
}
return func(authIndex string) {
if traceID := FormatCPATraceID(time.Now(), authIndex, requestID); traceID != "" {
state.set(traceID)
}
}
}
// SetGinCPATraceID stores the trace ID until the downstream response headers are committed.
func SetGinCPATraceID(c *gin.Context, authIndex string) {
if callback := GinCPATraceIDCallback(c); callback != nil {
callback(authIndex)
}
}
// GetGinCPATraceID returns the trace ID stored for the current request.
func GetGinCPATraceID(c *gin.Context) string {
if c == nil {
return ""
}
value, exists := c.Get(ginCPATraceStateKey)
if !exists {
return ""
}
state, _ := value.(*cpaTraceState)
return state.get()
}
// CPATraceIDMiddleware injects a stored trace ID immediately before response headers are committed.
func CPATraceIDMiddleware() gin.HandlerFunc {
return func(c *gin.Context) {
state := ginCPATraceState(c)
c.Writer = &cpaTraceResponseWriter{ResponseWriter: c.Writer, state: state}
c.Next()
}
}
type cpaTraceResponseWriter struct {
gin.ResponseWriter
state *cpaTraceState
}
func (w *cpaTraceResponseWriter) WriteHeader(statusCode int) {
w.applyTraceHeader()
w.ResponseWriter.WriteHeader(statusCode)
}
func (w *cpaTraceResponseWriter) WriteHeaderNow() {
w.applyTraceHeader()
w.ResponseWriter.WriteHeaderNow()
}
func (w *cpaTraceResponseWriter) Write(data []byte) (int, error) {
w.applyTraceHeader()
return w.ResponseWriter.Write(data)
}
func (w *cpaTraceResponseWriter) WriteString(data string) (int, error) {
w.applyTraceHeader()
return w.ResponseWriter.WriteString(data)
}
func (w *cpaTraceResponseWriter) Flush() {
w.applyTraceHeader()
w.ResponseWriter.Flush()
}
func (w *cpaTraceResponseWriter) applyTraceHeader() {
if w == nil || w.ResponseWriter == nil || w.ResponseWriter.Written() {
return
}
if traceID := w.state.get(); traceID != "" {
w.ResponseWriter.Header().Set(CPATraceIDHeader, traceID)
}
}

115
backend/internal/logging/cpa_trace_test.go Normal file
View file

@ -0,0 +1,115 @@
package logging
import (
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/gin-gonic/gin"
)
func TestFormatCPATraceID(t *testing.T) {
selectedAt := time.Date(2026, time.July, 17, 21, 58, 49, 0, time.UTC)
got := FormatCPATraceID(selectedAt, "auth-index", "request1")
if want := "20260717215849-auth-index-request1"; got != want {
t.Fatalf("FormatCPATraceID() = %q, want %q", got, want)
}
for _, test := range []struct {
name string
selectedAt time.Time
authIndex string
requestID string
}{
{name: "zero time", authIndex: "auth-index", requestID: "request1"},
{name: "empty auth index", selectedAt: selectedAt, requestID: "request1"},
{name: "empty request ID", selectedAt: selectedAt, authIndex: "auth-index"},
} {
t.Run(test.name, func(t *testing.T) {
if gotEmpty := FormatCPATraceID(test.selectedAt, test.authIndex, test.requestID); gotEmpty != "" {
t.Fatalf("FormatCPATraceID() = %q, want empty", gotEmpty)
}
})
}
}
func TestCPATraceIDMiddlewareRequiresAuthIndexBeforeResponseCommit(t *testing.T) {
gin.SetMode(gin.TestMode)
engine := gin.New()
engine.Use(CPATraceIDMiddleware())
engine.GET("/selected", func(c *gin.Context) {
SetGinRequestID(c, "1234abcd")
SetGinCPATraceID(c, "auth-index")
c.Status(http.StatusOK)
})
engine.GET("/unselected", func(c *gin.Context) {
SetGinRequestID(c, "1234abcd")
SetGinCPATraceID(c, "")
c.Status(http.StatusOK)
})
engine.GET("/committed", func(c *gin.Context) {
SetGinRequestID(c, "1234abcd")
c.Writer.WriteHeaderNow()
SetGinCPATraceID(c, "auth-index")
})
t.Run("writes selected auth trace", func(t *testing.T) {
recorder := httptest.NewRecorder()
engine.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/selected", nil))
traceID := recorder.Header().Get(CPATraceIDHeader)
if len(traceID) != len("20060102150405-auth-index-1234abcd") {
t.Fatalf("trace ID = %q, unexpected length", traceID)
}
if got := traceID[15:]; got != "auth-index-1234abcd" {
t.Fatalf("trace suffix = %q, want %q", got, "auth-index-1234abcd")
}
})
t.Run("skips empty auth index", func(t *testing.T) {
recorder := httptest.NewRecorder()
engine.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/unselected", nil))
if got := recorder.Header().Get(CPATraceIDHeader); got != "" {
t.Fatalf("trace ID = %q, want empty", got)
}
})
t.Run("skips committed response", func(t *testing.T) {
recorder := httptest.NewRecorder()
engine.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/committed", nil))
if got := recorder.Header().Get(CPATraceIDHeader); got != "" {
t.Fatalf("trace ID = %q, want empty", got)
}
})
}
func TestCPATraceIDConcurrentSelectionAndResponseCommit(t *testing.T) {
gin.SetMode(gin.TestMode)
engine := gin.New()
engine.Use(CPATraceIDMiddleware())
engine.GET("/race", func(c *gin.Context) {
SetGinRequestID(c, "1234abcd")
traceCallback := GinCPATraceIDCallback(c)
start := make(chan struct{})
done := make(chan struct{})
go func() {
defer close(done)
<-start
traceCallback("auth-index")
}()
close(start)
_, _ = c.Writer.Write([]byte("\n"))
<-done
})
for range 100 {
recorder := httptest.NewRecorder()
engine.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/race", nil))
if recorder.Code != http.StatusOK {
t.Fatalf("status = %d, want %d", recorder.Code, http.StatusOK)
}
}
}

166
backend/internal/logging/gin_logger.go Normal file
View file

@ -0,0 +1,166 @@
// Package logging provides Gin middleware for HTTP request logging and panic recovery.
// It integrates Gin web framework with logrus for structured logging of HTTP requests,
// responses, and error handling with panic recovery capabilities.
package logging
import (
"errors"
"fmt"
"net/http"
"runtime/debug"
"strings"
"time"
"github.com/gin-gonic/gin"
"github.com/router-for-me/CLIProxyAPI/v7/internal/util"
log "github.com/sirupsen/logrus"
)
// aiAPIPrefixes defines path prefixes for AI API requests that should have request ID tracking.
var aiAPIPrefixes = []string{
"/v1",
"/v1beta",
"/openai/v1",
"/backend-api/codex",
}
const (
skipGinLogKey = "__gin_skip_request_logging__"
creditsUsedKey = "__antigravity_credits_used__"
)
// GinLogrusLogger returns a Gin middleware handler that logs HTTP requests and responses
// using logrus. It captures request details including method, path, status code, latency,
// client IP, and any error messages. Request ID is only added for AI API requests.
//
// Output format (AI API): [2025-12-23 20:14:10] [info ] | a1b2c3d4 | 200 | 23.559s | ...
// Output format (others): [2025-12-23 20:14:10] [info ] | -------- | 200 | 23.559s | ...
//
// Returns:
// - gin.HandlerFunc: A middleware handler for request logging
func GinLogrusLogger() gin.HandlerFunc {
return func(c *gin.Context) {
start := time.Now()
path := c.Request.URL.Path
raw := util.MaskSensitiveQuery(c.Request.URL.RawQuery)
// Only generate request ID for AI API paths
var requestID string
if isAIAPIPath(path) {
requestID = GenerateRequestID()
SetGinRequestID(c, requestID)
ctx := WithRequestID(c.Request.Context(), requestID)
c.Request = c.Request.WithContext(ctx)
}
c.Next()
if shouldSkipGinRequestLogging(c) {
return
}
if raw != "" {
path = path + "?" + raw
}
latency := time.Since(start)
if latency > time.Minute {
latency = latency.Truncate(time.Second)
} else {
latency = latency.Truncate(time.Millisecond)
}
statusCode := c.Writer.Status()
clientIP := c.ClientIP()
method := c.Request.Method
errorMessage := c.Errors.ByType(gin.ErrorTypePrivate).String()
if requestID == "" {
requestID = "--------"
}
logLine := fmt.Sprintf("%3d | %13v | %15s | %-7s \"%s\"", statusCode, latency, clientIP, method, path)
if creditsUsed(c) {
logLine += " [credits]"
}
if errorMessage != "" {
logLine = logLine + " | " + errorMessage
}
entry := log.WithField("request_id", requestID)
switch {
case statusCode >= http.StatusInternalServerError:
entry.Error(logLine)
case statusCode >= http.StatusBadRequest:
entry.Warn(logLine)
default:
entry.Info(logLine)
}
}
}
// isAIAPIPath checks if the given path is an AI API endpoint that should have request ID tracking.
func isAIAPIPath(path string) bool {
for _, prefix := range aiAPIPrefixes {
if path == prefix || strings.HasPrefix(path, prefix+"/") {
return true
}
}
return false
}
// GinLogrusRecovery returns a Gin middleware handler that recovers from panics and logs
// them using logrus. When a panic occurs, it captures the panic value, stack trace,
// and request path, then returns a 500 Internal Server Error response to the client.
//
// Returns:
// - gin.HandlerFunc: A middleware handler for panic recovery
func GinLogrusRecovery() gin.HandlerFunc {
return gin.CustomRecovery(func(c *gin.Context, recovered interface{}) {
if err, ok := recovered.(error); ok && errors.Is(err, http.ErrAbortHandler) {
// Let net/http handle ErrAbortHandler so the connection is aborted without noisy stack logs.
panic(http.ErrAbortHandler)
}
log.WithFields(log.Fields{
"panic": recovered,
"stack": string(debug.Stack()),
"path": c.Request.URL.Path,
}).Error("recovered from panic")
c.AbortWithStatus(http.StatusInternalServerError)
})
}
// SkipGinRequestLogging marks the provided Gin context so that GinLogrusLogger
// will skip emitting a log line for the associated request.
func SkipGinRequestLogging(c *gin.Context) {
if c == nil {
return
}
c.Set(skipGinLogKey, true)
}
func shouldSkipGinRequestLogging(c *gin.Context) bool {
if c == nil {
return false
}
val, exists := c.Get(skipGinLogKey)
if !exists {
return false
}
flag, ok := val.(bool)
return ok && flag
}
func creditsUsed(c *gin.Context) bool {
if c == nil {
return false
}
val, exists := c.Get(creditsUsedKey)
if !exists {
return false
}
flag, ok := val.(bool)
return ok && flag
}

150
backend/internal/logging/gin_logger_test.go Normal file
View file

@ -0,0 +1,150 @@
package logging
import (
"errors"
"net/http"
"net/http/httptest"
"testing"
"github.com/gin-gonic/gin"
)
func TestGinLogrusRecoveryRepanicsErrAbortHandler(t *testing.T) {
gin.SetMode(gin.TestMode)
engine := gin.New()
engine.Use(GinLogrusRecovery())
engine.GET("/abort", func(c *gin.Context) {
panic(http.ErrAbortHandler)
})
req := httptest.NewRequest(http.MethodGet, "/abort", nil)
recorder := httptest.NewRecorder()
defer func() {
recovered := recover()
if recovered == nil {
t.Fatalf("expected panic, got nil")
}
err, ok := recovered.(error)
if !ok {
t.Fatalf("expected error panic, got %T", recovered)
}
if !errors.Is(err, http.ErrAbortHandler) {
t.Fatalf("expected ErrAbortHandler, got %v", err)
}
if err != http.ErrAbortHandler {
t.Fatalf("expected exact ErrAbortHandler sentinel, got %v", err)
}
}()
engine.ServeHTTP(recorder, req)
}
func TestGinLogrusRecoveryHandlesRegularPanic(t *testing.T) {
gin.SetMode(gin.TestMode)
engine := gin.New()
engine.Use(GinLogrusRecovery())
engine.GET("/panic", func(c *gin.Context) {
panic("boom")
})
req := httptest.NewRequest(http.MethodGet, "/panic", nil)
recorder := httptest.NewRecorder()
engine.ServeHTTP(recorder, req)
if recorder.Code != http.StatusInternalServerError {
t.Fatalf("expected 500, got %d", recorder.Code)
}
}
func TestIsAIAPIPathIncludesPublicAPIGroups(t *testing.T) {
for _, path := range []string{
"/v1",
"/v1/models",
"/v1/alpha/search",
"/v1beta/interactions",
"/openai/v1/videos",
"/backend-api/codex/responses",
} {
if !isAIAPIPath(path) {
t.Fatalf("expected %s to be treated as AI API path", path)
}
}
for _, path := range []string{
"/v0/management/config",
"/v10/models",
"/openai/v10/videos",
"/backend-api/codex-status",
} {
if isAIAPIPath(path) {
t.Fatalf("expected %s not to be treated as AI API path", path)
}
}
}
func TestIsAIAPIPathIncludesImages(t *testing.T) {
if !isAIAPIPath("/v1/images/generations") {
t.Fatalf("expected /v1/images/generations to be treated as AI API path")
}
if !isAIAPIPath("/v1/images/edits") {
t.Fatalf("expected /v1/images/edits to be treated as AI API path")
}
if !isAIAPIPath("/v1/videos") {
t.Fatalf("expected /v1/videos to be treated as AI API path")
}
if !isAIAPIPath("/v1/videos/video_123") {
t.Fatalf("expected /v1/videos/video_123 to be treated as AI API path")
}
if !isAIAPIPath("/openai/v1/videos") {
t.Fatalf("expected /openai/v1/videos to be treated as AI API path")
}
if !isAIAPIPath("/openai/v1/videos/video_123/content") {
t.Fatalf("expected /openai/v1/videos/video_123/content to be treated as AI API path")
}
}
func TestIsAIAPIPathIncludesCodexBackend(t *testing.T) {
paths := []string{
"/backend-api/codex/responses",
"/backend-api/codex/responses/compact",
}
for _, path := range paths {
if !isAIAPIPath(path) {
t.Fatalf("expected %s to be treated as AI API path", path)
}
}
if isAIAPIPath("/backend-api/codex-status") {
t.Fatalf("expected /backend-api/codex-status not to be treated as AI API path")
}
}
func TestGinLogrusLoggerAddsRequestIDForCodexBackend(t *testing.T) {
gin.SetMode(gin.TestMode)
engine := gin.New()
engine.Use(GinLogrusLogger())
var requestIDFromContext string
var requestIDFromGin string
engine.POST("/backend-api/codex/responses", func(c *gin.Context) {
requestIDFromContext = GetRequestID(c.Request.Context())
requestIDFromGin = GetGinRequestID(c)
c.Status(http.StatusOK)
})
req := httptest.NewRequest(http.MethodPost, "/backend-api/codex/responses", nil)
recorder := httptest.NewRecorder()
engine.ServeHTTP(recorder, req)
if recorder.Code != http.StatusOK {
t.Fatalf("expected 200, got %d", recorder.Code)
}
if requestIDFromContext == "" {
t.Fatalf("expected request ID in request context")
}
if requestIDFromGin != requestIDFromContext {
t.Fatalf("expected Gin request ID %q to match context request ID %q", requestIDFromGin, requestIDFromContext)
}
}

241
backend/internal/logging/global_logger.go Normal file
View file

@ -0,0 +1,241 @@
package logging
import (
"bytes"
"fmt"
"io"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"github.com/gin-gonic/gin"
"github.com/router-for-me/CLIProxyAPI/v7/internal/config"
"github.com/router-for-me/CLIProxyAPI/v7/internal/util"
log "github.com/sirupsen/logrus"
"gopkg.in/natefinch/lumberjack.v2"
)
var (
setupOnce sync.Once
writerMu sync.Mutex
logWriter *lumberjack.Logger
ginInfoWriter *io.PipeWriter
ginErrorWriter *io.PipeWriter
)
// LogFormatter defines a custom log format for logrus.
// This formatter adds timestamp, level, request ID, and source location to each log entry.
// Format: [2025-12-23 20:14:04] [debug] [manager.go:524] | a1b2c3d4 | Use API key sk-9...0RHO for model gpt-5.2
type LogFormatter struct{}
// logFieldOrder defines the display order for common log fields.
var logFieldOrder = []string{
"provider", "model",
"plugin_id", "plugin_name", "source_id",
"version", "active_version", "retired_version", "overwritten",
"mode", "budget", "level", "original_mode", "original_value", "min", "max", "clamped_to", "error",
"credential", "connection", "proxy_scheme", "remote_transport",
"media_session_id", "call_id", "peer", "state", "reason",
}
var quotedLogFields = map[string]struct{}{
"credential": {},
"connection": {},
"proxy_scheme": {},
"remote_transport": {},
"media_session_id": {},
"call_id": {},
"peer": {},
"state": {},
"reason": {},
}
var pluginPathFieldOrder = []string{"path", "active_path", "retired_path"}
func formatLogFieldValue(key string, value any) string {
if _, quoted := quotedLogFields[key]; quoted {
if stringValue, ok := value.(string); ok {
return strconv.Quote(stringValue)
}
}
return fmt.Sprint(value)
}
// Format renders a single log entry with custom formatting.
func (m *LogFormatter) Format(entry *log.Entry) ([]byte, error) {
var buffer *bytes.Buffer
if entry.Buffer != nil {
buffer = entry.Buffer
} else {
buffer = &bytes.Buffer{}
}
timestamp := entry.Time.Format("2006-01-02 15:04:05")
message := strings.TrimRight(entry.Message, "\r\n")
reqID := "--------"
if id, ok := entry.Data["request_id"].(string); ok && id != "" {
reqID = id
}
level := entry.Level.String()
if level == "warning" {
level = "warn"
}
levelStr := fmt.Sprintf("%-5s", level)
// Build fields string (only print fields in logFieldOrder)
var fieldsStr string
if len(entry.Data) > 0 {
var fields []string
for _, k := range logFieldOrder {
if v, ok := entry.Data[k]; ok {
fields = append(fields, fmt.Sprintf("%s=%s", k, formatLogFieldValue(k, v)))
}
}
if pluginID, ok := entry.Data["plugin_id"]; ok && strings.TrimSpace(fmt.Sprint(pluginID)) != "" {
for _, k := range pluginPathFieldOrder {
if v, ok := entry.Data[k]; ok {
fields = append(fields, fmt.Sprintf("%s=%v", k, v))
}
}
}
if len(fields) > 0 {
fieldsStr = " " + strings.Join(fields, " ")
}
}
var formatted string
if entry.Caller != nil {
formatted = fmt.Sprintf("[%s] [%s] [%s] [%s:%d] %s%s\n", timestamp, reqID, levelStr, filepath.Base(entry.Caller.File), entry.Caller.Line, message, fieldsStr)
} else {
formatted = fmt.Sprintf("[%s] [%s] [%s] %s%s\n", timestamp, reqID, levelStr, message, fieldsStr)
}
buffer.WriteString(formatted)
return buffer.Bytes(), nil
}
// SetupBaseLogger configures the shared logrus instance and Gin writers.
// It is safe to call multiple times; initialization happens only once.
func SetupBaseLogger() {
setupOnce.Do(func() {
log.SetOutput(os.Stdout)
log.SetReportCaller(true)
log.SetFormatter(&LogFormatter{})
ginInfoWriter = log.StandardLogger().Writer()
gin.DefaultWriter = ginInfoWriter
ginErrorWriter = log.StandardLogger().WriterLevel(log.ErrorLevel)
gin.DefaultErrorWriter = ginErrorWriter
gin.DebugPrintFunc = func(format string, values ...interface{}) {
format = strings.TrimRight(format, "\r\n")
log.StandardLogger().Infof(format, values...)
}
log.RegisterExitHandler(closeLogOutputs)
})
}
// isDirWritable checks if the specified directory exists and is writable by attempting to create and remove a test file.
func isDirWritable(dir string) bool {
info, err := os.Stat(dir)
if err != nil || !info.IsDir() {
return false
}
testFile := filepath.Join(dir, ".perm_test")
f, err := os.Create(testFile)
if err != nil {
return false
}
defer func() {
_ = f.Close()
_ = os.Remove(testFile)
}()
return true
}
// ResolveLogDirectory determines the directory used for application logs.
func ResolveLogDirectory(cfg *config.Config) string {
logDir := "logs"
if base := util.WritablePath(); base != "" {
return filepath.Join(base, "logs")
}
if cfg == nil {
return logDir
}
if !isDirWritable(logDir) {
authDir, err := util.ResolveAuthDir(cfg.AuthDir)
if err != nil {
log.Warnf("Failed to resolve auth-dir %q for log directory: %v", cfg.AuthDir, err)
}
if authDir != "" {
logDir = filepath.Join(authDir, "logs")
}
}
return logDir
}
// ConfigureLogOutput switches the global log destination between rotating files and stdout.
// When logsMaxTotalSizeMB > 0, a background cleaner removes the oldest log files in the logs directory
// until the total size is within the limit.
func ConfigureLogOutput(cfg *config.Config) error {
SetupBaseLogger()
writerMu.Lock()
defer writerMu.Unlock()
logDir := ResolveLogDirectory(cfg)
protectedPath := ""
if cfg.LoggingToFile {
if err := os.MkdirAll(logDir, 0o755); err != nil {
return fmt.Errorf("logging: failed to create log directory: %w", err)
}
if logWriter != nil {
_ = logWriter.Close()
}
protectedPath = filepath.Join(logDir, "main.log")
logWriter = &lumberjack.Logger{
Filename: protectedPath,
MaxSize: 10,
MaxBackups: 0,
MaxAge: 0,
Compress: false,
}
log.SetOutput(logWriter)
} else {
if logWriter != nil {
_ = logWriter.Close()
logWriter = nil
}
log.SetOutput(os.Stdout)
}
configureLogDirCleanerLocked(logDir, cfg.LogsMaxTotalSizeMB, protectedPath)
return nil
}
func closeLogOutputs() {
writerMu.Lock()
defer writerMu.Unlock()
stopLogDirCleanerLocked()
if logWriter != nil {
_ = logWriter.Close()
logWriter = nil
}
if ginInfoWriter != nil {
_ = ginInfoWriter.Close()
ginInfoWriter = nil
}
if ginErrorWriter != nil {
_ = ginErrorWriter.Close()
ginErrorWriter = nil
}
}

124
backend/internal/logging/global_logger_test.go Normal file
View file

@ -0,0 +1,124 @@
package logging
import (
"strings"
"testing"
"time"
log "github.com/sirupsen/logrus"
)
func TestLogFormatterPrintsVersionField(t *testing.T) {
entry := log.NewEntry(log.New())
entry.Time = time.Date(2026, 6, 9, 11, 10, 2, 0, time.Local)
entry.Level = log.InfoLevel
entry.Message = "fetched latest antigravity version"
entry.Data["version"] = "2.1.0"
formatted, errFormat := (&LogFormatter{}).Format(entry)
if errFormat != nil {
t.Fatalf("Format() error = %v", errFormat)
}
line := string(formatted)
if !strings.Contains(line, "version=2.1.0") {
t.Fatalf("formatted line %q missing version field", line)
}
}
func TestLogFormatterPrintsMediaForwardingFields(t *testing.T) {
entry := log.NewEntry(log.New())
entry.Time = time.Date(2026, 7, 25, 7, 36, 4, 0, time.Local)
entry.Level = log.InfoLevel
entry.Message = "codex live remote media forwarding started"
entry.Data["credential"] = "Voice credential\nsecondary"
entry.Data["connection"] = "via socks5 proxy"
entry.Data["proxy_scheme"] = "socks5"
entry.Data["remote_transport"] = "tcp"
entry.Data["media_session_id"] = "media-session-id"
entry.Data["call_id"] = "call-id"
entry.Data["peer"] = "remote"
entry.Data["state"] = "connected"
formatted, errFormat := (&LogFormatter{}).Format(entry)
if errFormat != nil {
t.Fatalf("Format() error = %v", errFormat)
}
line := string(formatted)
for _, want := range []string{
`credential="Voice credential\nsecondary"`,
`connection="via socks5 proxy"`,
`proxy_scheme="socks5"`,
`remote_transport="tcp"`,
`media_session_id="media-session-id"`,
`call_id="call-id"`,
`peer="remote"`,
`state="connected"`,
} {
if !strings.Contains(line, want) {
t.Fatalf("formatted line %q missing %s", line, want)
}
}
if strings.Count(line, "\n") != 1 {
t.Fatalf("formatted line contains an unescaped newline: %q", line)
}
}
func TestLogFormatterPrintsPluginFields(t *testing.T) {
entry := log.NewEntry(log.New())
entry.Time = time.Date(2026, 6, 25, 20, 10, 0, 0, time.Local)
entry.Level = log.InfoLevel
entry.Message = "pluginhost: plugin loaded"
entry.Data["plugin_id"] = "sample-provider"
entry.Data["plugin_name"] = "Sample Provider"
entry.Data["version"] = "0.2.0"
entry.Data["active_version"] = "0.1.0"
entry.Data["retired_version"] = "0.2.0"
entry.Data["path"] = "plugins/windows/amd64/sample-provider-v0.2.0.dll"
entry.Data["active_path"] = "plugins/windows/amd64/sample-provider-v0.1.0.dll"
entry.Data["retired_path"] = "plugins/windows/amd64/sample-provider-v0.2.0.dll"
formatted, errFormat := (&LogFormatter{}).Format(entry)
if errFormat != nil {
t.Fatalf("Format() error = %v", errFormat)
}
line := string(formatted)
for _, want := range []string{
"plugin_id=sample-provider",
"plugin_name=Sample Provider",
"version=0.2.0",
"active_version=0.1.0",
"retired_version=0.2.0",
"path=plugins/windows/amd64/sample-provider-v0.2.0.dll",
"active_path=plugins/windows/amd64/sample-provider-v0.1.0.dll",
"retired_path=plugins/windows/amd64/sample-provider-v0.2.0.dll",
} {
if !strings.Contains(line, want) {
t.Fatalf("formatted line %q missing %s", line, want)
}
}
}
func TestLogFormatterOmitsGenericPathField(t *testing.T) {
entry := log.NewEntry(log.New())
entry.Time = time.Date(2026, 6, 25, 20, 20, 0, 0, time.Local)
entry.Level = log.WarnLevel
entry.Message = "failed to roll back token"
entry.Data["path"] = "auths/private-token.json"
entry.Data["active_path"] = "plugins/windows/amd64/sample-provider-v0.1.0.dll"
entry.Data["retired_path"] = "plugins/windows/amd64/sample-provider-v0.2.0.dll"
formatted, errFormat := (&LogFormatter{}).Format(entry)
if errFormat != nil {
t.Fatalf("Format() error = %v", errFormat)
}
line := string(formatted)
for _, forbidden := range []string{"path=", "active_path=", "retired_path="} {
if strings.Contains(line, forbidden) {
t.Fatalf("formatted line %q contains generic %s field", line, forbidden)
}
}
}

296
backend/internal/logging/home_app_log_forwarder.go Normal file
View file

@ -0,0 +1,296 @@
package logging
import (
"context"
"encoding/json"
"errors"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/router-for-me/CLIProxyAPI/v7/internal/home"
log "github.com/sirupsen/logrus"
)
const defaultHomeAppLogQueueSize = 1024
type homeAppLogClient interface {
HeartbeatOK() bool
RPushAppLog(ctx context.Context, payload []byte) error
}
type homeAppLogPayload struct {
Line string `json:"line"`
Level string `json:"level,omitempty"`
Timestamp string `json:"timestamp,omitempty"`
RequestID string `json:"request_id,omitempty"`
client homeAppLogClient
}
// HomeAppLogForwarder forwards application logs to Home after the control connection is healthy.
type HomeAppLogForwarder struct {
formatter log.Formatter
queue chan homeAppLogPayload
stop chan struct{}
stopOnce sync.Once
wg sync.WaitGroup
enabled atomic.Bool
stopped atomic.Bool
ownerMu sync.Mutex
owner homeAppLogClient
}
type homeAppLogMux struct {
mu sync.Mutex
targets map[*HomeAppLogForwarder]struct{}
}
func (h *homeAppLogMux) Levels() []log.Level {
return log.AllLevels
}
func (h *homeAppLogMux) Fire(entry *log.Entry) error {
h.mu.Lock()
targets := make([]*HomeAppLogForwarder, 0, len(h.targets))
for target := range h.targets {
targets = append(targets, target)
}
h.mu.Unlock()
for _, target := range targets {
if errFire := target.Fire(entry); errFire != nil {
return errFire
}
}
return nil
}
func (h *homeAppLogMux) register(target *HomeAppLogForwarder) {
if target == nil {
return
}
h.mu.Lock()
defer h.mu.Unlock()
if h.targets == nil {
h.targets = make(map[*HomeAppLogForwarder]struct{})
}
h.targets[target] = struct{}{}
}
func (h *homeAppLogMux) unregister(target *HomeAppLogForwarder) {
if target == nil {
return
}
h.mu.Lock()
delete(h.targets, target)
h.mu.Unlock()
}
var (
homeAppLogMuxHook = &homeAppLogMux{}
homeAppLogMuxInstallOnce sync.Once
)
func registerHomeAppLogForwarder(forwarder *HomeAppLogForwarder) {
homeAppLogMuxInstallOnce.Do(func() {
log.AddHook(homeAppLogMuxHook)
})
homeAppLogMuxHook.register(forwarder)
}
// StartHomeAppLogForwarder registers a Home log forwarding target with the process-wide logrus hook.
func StartHomeAppLogForwarder(queueSize int) *HomeAppLogForwarder {
if queueSize <= 0 {
queueSize = defaultHomeAppLogQueueSize
}
forwarder := &HomeAppLogForwarder{
formatter: &LogFormatter{},
queue: make(chan homeAppLogPayload, queueSize),
stop: make(chan struct{}),
}
forwarder.enabled.Store(true)
forwarder.wg.Add(1)
go forwarder.run()
registerHomeAppLogForwarder(forwarder)
return forwarder
}
// Stop disables forwarding and waits for the background sender to exit.
func (f *HomeAppLogForwarder) Stop() {
if f == nil {
return
}
f.stopOnce.Do(func() {
f.stopped.Store(true)
f.ownerMu.Lock()
f.owner = nil
f.ownerMu.Unlock()
f.enabled.Store(false)
homeAppLogMuxHook.unregister(f)
close(f.stop)
f.wg.Wait()
})
}
// Bind activates forwarding to client.
func (f *HomeAppLogForwarder) Bind(client *home.Client) {
f.bind(client)
}
func (f *HomeAppLogForwarder) bind(client homeAppLogClient) {
if f == nil || client == nil || f.stopped.Load() {
return
}
f.ownerMu.Lock()
defer f.ownerMu.Unlock()
if f.stopped.Load() {
return
}
f.owner = client
f.enabled.Store(true)
}
// Deactivate stops forwarding only when client owns the forwarder.
func (f *HomeAppLogForwarder) Deactivate(client *home.Client) {
f.deactivate(client)
}
func (f *HomeAppLogForwarder) deactivate(client homeAppLogClient) {
if f == nil || client == nil {
return
}
f.ownerMu.Lock()
if f.owner == client {
f.owner = nil
}
f.ownerMu.Unlock()
}
func (f *HomeAppLogForwarder) client() homeAppLogClient {
f.ownerMu.Lock()
defer f.ownerMu.Unlock()
return f.owner
}
// Levels implements logrus.Hook.
func (f *HomeAppLogForwarder) Levels() []log.Level {
return log.AllLevels
}
// Fire implements logrus.Hook.
func (f *HomeAppLogForwarder) Fire(entry *log.Entry) error {
if f == nil || entry == nil || !f.enabled.Load() {
return nil
}
client := f.client()
if client == nil || !client.HeartbeatOK() {
return nil
}
line, errFormat := f.formatEntry(entry)
if errFormat != nil || strings.TrimSpace(line) == "" {
return nil
}
payload := homeAppLogPayload{
Line: line,
Level: entry.Level.String(),
Timestamp: entry.Time.Format(time.RFC3339Nano),
RequestID: appLogRequestID(entry),
client: client,
}
select {
case f.queue <- payload:
default:
}
return nil
}
func appLogRequestID(entry *log.Entry) string {
if entry == nil {
return ""
}
requestID, _ := entry.Data["request_id"].(string)
requestID = strings.TrimSpace(requestID)
if requestID == "--------" {
return ""
}
return requestID
}
func (f *HomeAppLogForwarder) formatEntry(entry *log.Entry) (string, error) {
formatter := f.formatter
if formatter == nil {
formatter = &LogFormatter{}
}
raw, errFormat := formatter.Format(entry)
if errFormat != nil {
return "", errFormat
}
return string(raw), nil
}
func (f *HomeAppLogForwarder) run() {
defer f.wg.Done()
for {
select {
case <-f.stop:
return
case payload := <-f.queue:
f.forward(payload)
}
}
}
func (f *HomeAppLogForwarder) forward(payload homeAppLogPayload) {
client := payload.client
if client == nil {
client = f.client()
}
if !f.enabled.Load() || client == nil || f.client() != client {
return
}
if !client.HeartbeatOK() {
return
}
raw, errMarshal := json.Marshal(&payload)
if errMarshal != nil {
return
}
if errPush := client.RPushAppLog(context.Background(), raw); errPush != nil && isHomeAppLogUnsupported(errPush) {
f.disableIfCurrentOwner(client)
}
}
func (f *HomeAppLogForwarder) disableIfCurrentOwner(client homeAppLogClient) {
f.ownerMu.Lock()
defer f.ownerMu.Unlock()
if f.owner != client {
return
}
f.enabled.Store(false)
}
func isHomeAppLogUnsupported(err error) bool {
if err == nil {
return false
}
msg := strings.ToLower(strings.TrimSpace(err.Error()))
if msg == "" {
return false
}
for {
switch {
case strings.Contains(msg, "unsupported key"):
return true
case strings.Contains(msg, "unknown command"):
return true
case strings.Contains(msg, "unsupported command"):
return true
}
err = errors.Unwrap(err)
if err == nil {
return false
}
msg = strings.ToLower(strings.TrimSpace(err.Error()))
}
}

384
backend/internal/logging/home_app_log_forwarder_test.go Normal file
View file

@ -0,0 +1,384 @@
package logging
import (
"bytes"
"context"
"encoding/json"
"errors"
"strings"
"sync"
"testing"
"time"
internalconfig "github.com/router-for-me/CLIProxyAPI/v7/internal/config"
"github.com/router-for-me/CLIProxyAPI/v7/internal/home"
log "github.com/sirupsen/logrus"
)
type stubHomeAppLogClient struct {
mu sync.Mutex
heartbeatOK bool
err error
pushed [][]byte
}
func (c *stubHomeAppLogClient) HeartbeatOK() bool { return c.heartbeatOK }
func (c *stubHomeAppLogClient) RPushAppLog(_ context.Context, payload []byte) error {
c.mu.Lock()
defer c.mu.Unlock()
if c.err != nil {
return c.err
}
c.pushed = append(c.pushed, bytes.Clone(payload))
return nil
}
func (c *stubHomeAppLogClient) pushedCount() int {
c.mu.Lock()
defer c.mu.Unlock()
return len(c.pushed)
}
func (c *stubHomeAppLogClient) pushedAt(index int) []byte {
c.mu.Lock()
defer c.mu.Unlock()
if index < 0 || index >= len(c.pushed) {
return nil
}
return bytes.Clone(c.pushed[index])
}
func TestHomeAppLogForwarder_ForwardsFormattedLogWhenBoundOwnerIsHealthy(t *testing.T) {
stub := &stubHomeAppLogClient{heartbeatOK: true}
forwarder := &HomeAppLogForwarder{
formatter: &LogFormatter{},
queue: make(chan homeAppLogPayload, 4),
stop: make(chan struct{}),
}
forwarder.enabled.Store(true)
forwarder.bind(stub)
forwarder.wg.Add(1)
go forwarder.run()
defer forwarder.Stop()
entry := log.NewEntry(log.StandardLogger())
entry.Time = time.Date(2026, 5, 29, 8, 0, 0, 0, time.Local)
entry.Level = log.DebugLevel
entry.Message = "debug details"
entry.Data["request_id"] = "req-app-1"
if errFire := forwarder.Fire(entry); errFire != nil {
t.Fatalf("Fire error: %v", errFire)
}
deadline := time.Now().Add(time.Second)
for stub.pushedCount() == 0 && time.Now().Before(deadline) {
time.Sleep(10 * time.Millisecond)
}
if stub.pushedCount() != 1 {
t.Fatalf("pushed records = %d, want 1", stub.pushedCount())
}
var got homeAppLogPayload
if errUnmarshal := json.Unmarshal(stub.pushedAt(0), &got); errUnmarshal != nil {
t.Fatalf("unmarshal payload: %v", errUnmarshal)
}
if got.Level != "debug" {
t.Fatalf("level = %q, want debug", got.Level)
}
if got.RequestID != "req-app-1" {
t.Fatalf("request_id = %q, want req-app-1", got.RequestID)
}
if !strings.Contains(got.Line, "debug details") {
t.Fatalf("line %q missing log message", got.Line)
}
if !strings.Contains(got.Line, "[req-app-1]") {
t.Fatalf("line %q missing matching request id", got.Line)
}
if strings.TrimSpace(got.Timestamp) == "" {
t.Fatal("timestamp empty, want non-empty")
}
}
func TestHomeAppLogForwarder_StopUnregistersMuxTarget(t *testing.T) {
beforeHooks := homeAppLogForwarderHookCount()
beforeTargets := homeAppLogForwarderTargetCount()
forwarder := StartHomeAppLogForwarder(1)
if got := homeAppLogForwarderHookCount(); got != beforeHooks {
forwarder.Stop()
t.Fatalf("direct Home log forwarder hooks = %d, want %d", got, beforeHooks)
}
if got := homeAppLogForwarderTargetCount(); got != beforeTargets+1 {
forwarder.Stop()
t.Fatalf("Home log forwarder targets = %d, want %d", got, beforeTargets+1)
}
forwarder.Stop()
if got := homeAppLogForwarderTargetCount(); got != beforeTargets {
t.Fatalf("Home log forwarder targets after Stop = %d, want %d", got, beforeTargets)
}
}
func TestHomeAppLogForwardersUseOneProcessWideMuxHook(t *testing.T) {
first := StartHomeAppLogForwarder(1)
second := StartHomeAppLogForwarder(1)
t.Cleanup(first.Stop)
t.Cleanup(second.Stop)
if got := homeAppLogForwarderHookCount(); got != 0 {
t.Fatalf("direct Home log forwarder hooks = %d, want 0", got)
}
if got := homeAppLogMuxHookCount(); got != 1 {
t.Fatalf("Home log mux hooks = %d, want 1", got)
}
}
func homeAppLogForwarderHookCount() int {
count := 0
for _, hooks := range log.StandardLogger().Hooks {
for _, hook := range hooks {
if _, ok := hook.(*HomeAppLogForwarder); ok {
count++
}
}
}
return count / len(log.AllLevels)
}
func homeAppLogMuxHookCount() int {
count := 0
for _, hooks := range log.StandardLogger().Hooks {
for _, hook := range hooks {
if _, ok := hook.(*homeAppLogMux); ok {
count++
}
}
}
return count / len(log.AllLevels)
}
func homeAppLogForwarderTargetCount() int {
homeAppLogMuxHook.mu.Lock()
defer homeAppLogMuxHook.mu.Unlock()
return len(homeAppLogMuxHook.targets)
}
func TestHomeAppLogForwarder_RebindsOnlyToCurrentOwner(t *testing.T) {
first := &stubHomeAppLogClient{heartbeatOK: true}
second := &stubHomeAppLogClient{heartbeatOK: true}
forwarder := &HomeAppLogForwarder{
formatter: &LogFormatter{},
queue: make(chan homeAppLogPayload, 4),
stop: make(chan struct{}),
}
forwarder.enabled.Store(true)
forwarder.wg.Add(1)
go forwarder.run()
t.Cleanup(forwarder.Stop)
forwarder.bind(first)
if errFire := forwarder.Fire(log.NewEntry(log.StandardLogger())); errFire != nil {
t.Fatalf("Fire() error = %v", errFire)
}
waitForHomeAppLogPush(t, first, 1)
forwarder.bind(second)
forwarder.deactivate(first)
if errFire := forwarder.Fire(log.NewEntry(log.StandardLogger())); errFire != nil {
t.Fatalf("Fire() error = %v", errFire)
}
waitForHomeAppLogPush(t, second, 1)
if first.pushedCount() != 1 {
t.Fatalf("stale owner received %d records, want 1", first.pushedCount())
}
forwarder.deactivate(first)
if errFire := forwarder.Fire(log.NewEntry(log.StandardLogger())); errFire != nil {
t.Fatalf("Fire() error = %v", errFire)
}
waitForHomeAppLogPush(t, second, 2)
forwarder.deactivate(second)
if errFire := forwarder.Fire(log.NewEntry(log.StandardLogger())); errFire != nil {
t.Fatalf("Fire() error = %v", errFire)
}
time.Sleep(20 * time.Millisecond)
if second.pushedCount() != 2 {
t.Fatalf("detached owner received %d records, want 2", second.pushedCount())
}
}
func waitForHomeAppLogPush(t *testing.T, client *stubHomeAppLogClient, want int) {
t.Helper()
deadline := time.Now().Add(time.Second)
for client.pushedCount() < want && time.Now().Before(deadline) {
time.Sleep(time.Millisecond)
}
if got := client.pushedCount(); got != want {
t.Fatalf("pushed records = %d, want %d", got, want)
}
}
type delayedUnsupportedHomeAppLogClient struct {
started chan struct{}
startedOnce sync.Once
release <-chan struct{}
}
func (c *delayedUnsupportedHomeAppLogClient) HeartbeatOK() bool { return true }
func (c *delayedUnsupportedHomeAppLogClient) RPushAppLog(_ context.Context, _ []byte) error {
c.startedOnce.Do(func() { close(c.started) })
<-c.release
return errors.New("ERR unsupported key")
}
func TestHomeAppLogForwarder_DelayedOldOwnerUnsupportedDoesNotDisableNewOwner(t *testing.T) {
release := make(chan struct{})
oldOwner := &delayedUnsupportedHomeAppLogClient{started: make(chan struct{}), release: release}
newOwner := &stubHomeAppLogClient{heartbeatOK: true}
forwarder := &HomeAppLogForwarder{
formatter: &LogFormatter{},
queue: make(chan homeAppLogPayload, 1),
stop: make(chan struct{}),
}
forwarder.enabled.Store(true)
forwarder.wg.Add(1)
go forwarder.run()
t.Cleanup(forwarder.Stop)
forwarder.bind(oldOwner)
forwardDone := make(chan struct{})
go func() {
forwarder.forward(homeAppLogPayload{Line: "old owner", client: oldOwner})
close(forwardDone)
}()
select {
case <-oldOwner.started:
case <-time.After(time.Second):
t.Fatal("old owner did not start forwarding")
}
forwarder.bind(newOwner)
close(release)
select {
case <-forwardDone:
case <-time.After(time.Second):
t.Fatal("old owner forwarding did not finish")
}
if !forwarder.enabled.Load() {
t.Fatal("old owner unsupported response disabled the new owner")
}
forwarder.forward(homeAppLogPayload{Line: "new owner", client: newOwner})
waitForHomeAppLogPush(t, newOwner, 1)
}
func TestHomeAppLogForwarder_UnboundNeverUsesGlobalFallbackClient(t *testing.T) {
fallback := home.New(internalconfig.HomeConfig{Enabled: true})
home.SetCurrent(fallback)
t.Cleanup(home.ClearCurrent)
forwarder := &HomeAppLogForwarder{
formatter: &LogFormatter{},
queue: make(chan homeAppLogPayload, 1),
stop: make(chan struct{}),
}
forwarder.enabled.Store(true)
if client := forwarder.client(); client != nil {
t.Fatalf("unbound client = %v, want nil", client)
}
if errFire := forwarder.Fire(log.NewEntry(log.StandardLogger())); errFire != nil {
t.Fatalf("Fire() error = %v", errFire)
}
if queued := len(forwarder.queue); queued != 0 {
t.Fatalf("unbound queued records = %d, want 0", queued)
}
}
func TestHomeAppLogForwarder_DropsPreACKAndReconnectGapLogs(t *testing.T) {
oldClient := home.New(internalconfig.HomeConfig{Enabled: true})
newClient := home.New(internalconfig.HomeConfig{Enabled: true})
home.SetCurrent(oldClient)
t.Cleanup(home.ClearCurrent)
preACKForwarder := &HomeAppLogForwarder{
formatter: &LogFormatter{},
queue: make(chan homeAppLogPayload, 1),
stop: make(chan struct{}),
}
preACKForwarder.enabled.Store(true)
if client := preACKForwarder.client(); client != nil {
t.Fatalf("pre-ACK client = %v, want nil", client)
}
if errFire := preACKForwarder.Fire(log.NewEntry(log.StandardLogger())); errFire != nil {
t.Fatalf("pre-ACK Fire() error = %v", errFire)
}
preACKForwarder.bind(oldClient)
preACKForwarder.deactivate(oldClient)
home.SetCurrent(newClient)
if errFire := preACKForwarder.Fire(log.NewEntry(log.StandardLogger())); errFire != nil {
t.Fatalf("reconnect-gap Fire() error = %v", errFire)
}
if got := len(preACKForwarder.queue); got != 0 {
t.Fatalf("pre-ACK/reconnect-gap queued records = %d, want 0", got)
}
if client := preACKForwarder.client(); client != nil {
t.Fatalf("reconnect-gap client = %v, want nil", client)
}
}
func TestHomeAppLogForwarder_OmitsPlaceholderRequestID(t *testing.T) {
entry := log.NewEntry(log.StandardLogger())
entry.Data["request_id"] = "--------"
if got := appLogRequestID(entry); got != "" {
t.Fatalf("request id = %q, want empty for placeholder", got)
}
}
func TestHomeAppLogForwarder_SkipsWhenBoundOwnerHeartbeatIsDown(t *testing.T) {
stub := &stubHomeAppLogClient{heartbeatOK: false}
forwarder := &HomeAppLogForwarder{
formatter: &LogFormatter{},
queue: make(chan homeAppLogPayload, 4),
stop: make(chan struct{}),
}
forwarder.enabled.Store(true)
forwarder.bind(stub)
entry := log.NewEntry(log.StandardLogger())
entry.Time = time.Now()
entry.Level = log.InfoLevel
entry.Message = "should stay local"
if errFire := forwarder.Fire(entry); errFire != nil {
t.Fatalf("Fire error: %v", errFire)
}
if stub.pushedCount() != 0 {
t.Fatalf("pushed records = %d, want 0", stub.pushedCount())
}
}
func TestHomeAppLogForwarder_DisablesForwardingWhenBoundOwnerDoesNotSupportAppLog(t *testing.T) {
stub := &stubHomeAppLogClient{
heartbeatOK: true,
err: errors.New("ERR unsupported key"),
}
forwarder := &HomeAppLogForwarder{
formatter: &LogFormatter{},
queue: make(chan homeAppLogPayload, 4),
stop: make(chan struct{}),
}
forwarder.enabled.Store(true)
forwarder.bind(stub)
forwarder.forward(homeAppLogPayload{Line: "legacy home cannot receive app logs"})
if forwarder.enabled.Load() {
t.Fatal("forwarder still enabled, want disabled after unsupported app-log response")
}
}

166
backend/internal/logging/log_dir_cleaner.go Normal file
View file

@ -0,0 +1,166 @@
package logging
import (
"context"
"os"
"path/filepath"
"sort"
"strings"
"time"
log "github.com/sirupsen/logrus"
)
const logDirCleanerInterval = time.Minute
var logDirCleanerCancel context.CancelFunc
func configureLogDirCleanerLocked(logDir string, maxTotalSizeMB int, protectedPath string) {
stopLogDirCleanerLocked()
if maxTotalSizeMB <= 0 {
return
}
maxBytes := int64(maxTotalSizeMB) * 1024 * 1024
if maxBytes <= 0 {
return
}
dir := strings.TrimSpace(logDir)
if dir == "" {
return
}
ctx, cancel := context.WithCancel(context.Background())
logDirCleanerCancel = cancel
go runLogDirCleaner(ctx, filepath.Clean(dir), maxBytes, strings.TrimSpace(protectedPath))
}
func stopLogDirCleanerLocked() {
if logDirCleanerCancel == nil {
return
}
logDirCleanerCancel()
logDirCleanerCancel = nil
}
func runLogDirCleaner(ctx context.Context, logDir string, maxBytes int64, protectedPath string) {
ticker := time.NewTicker(logDirCleanerInterval)
defer ticker.Stop()
cleanOnce := func() {
deleted, errClean := enforceLogDirSizeLimit(logDir, maxBytes, protectedPath)
if errClean != nil {
log.WithError(errClean).Warn("logging: failed to enforce log directory size limit")
return
}
if deleted > 0 {
log.Debugf("logging: removed %d old log file(s) to enforce log directory size limit", deleted)
}
}
cleanOnce()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
cleanOnce()
}
}
}
func enforceLogDirSizeLimit(logDir string, maxBytes int64, protectedPath string) (int, error) {
if maxBytes <= 0 {
return 0, nil
}
dir := strings.TrimSpace(logDir)
if dir == "" {
return 0, nil
}
dir = filepath.Clean(dir)
entries, errRead := os.ReadDir(dir)
if errRead != nil {
if os.IsNotExist(errRead) {
return 0, nil
}
return 0, errRead
}
protected := strings.TrimSpace(protectedPath)
if protected != "" {
protected = filepath.Clean(protected)
}
type logFile struct {
path string
size int64
modTime time.Time
}
var (
files []logFile
total int64
)
for _, entry := range entries {
if entry.IsDir() {
continue
}
name := entry.Name()
if !isLogFileName(name) {
continue
}
info, errInfo := entry.Info()
if errInfo != nil {
continue
}
if !info.Mode().IsRegular() {
continue
}
path := filepath.Join(dir, name)
files = append(files, logFile{
path: path,
size: info.Size(),
modTime: info.ModTime(),
})
total += info.Size()
}
if total <= maxBytes {
return 0, nil
}
sort.Slice(files, func(i, j int) bool {
return files[i].modTime.Before(files[j].modTime)
})
deleted := 0
for _, file := range files {
if total <= maxBytes {
break
}
if protected != "" && filepath.Clean(file.path) == protected {
continue
}
if errRemove := os.Remove(file.path); errRemove != nil {
log.WithError(errRemove).Warnf("logging: failed to remove old log file: %s", filepath.Base(file.path))
continue
}
total -= file.size
deleted++
}
return deleted, nil
}
func isLogFileName(name string) bool {
trimmed := strings.TrimSpace(name)
if trimmed == "" {
return false
}
lower := strings.ToLower(trimmed)
return strings.HasSuffix(lower, ".log") || strings.HasSuffix(lower, ".log.gz")
}

70
backend/internal/logging/log_dir_cleaner_test.go Normal file
View file

@ -0,0 +1,70 @@
package logging
import (
"os"
"path/filepath"
"testing"
"time"
)
func TestEnforceLogDirSizeLimitDeletesOldest(t *testing.T) {
dir := t.TempDir()
writeLogFile(t, filepath.Join(dir, "old.log"), 60, time.Unix(1, 0))
writeLogFile(t, filepath.Join(dir, "mid.log"), 60, time.Unix(2, 0))
protected := filepath.Join(dir, "main.log")
writeLogFile(t, protected, 60, time.Unix(3, 0))
deleted, err := enforceLogDirSizeLimit(dir, 120, protected)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if deleted != 1 {
t.Fatalf("expected 1 deleted file, got %d", deleted)
}
if _, err := os.Stat(filepath.Join(dir, "old.log")); !os.IsNotExist(err) {
t.Fatalf("expected old.log to be removed, stat error: %v", err)
}
if _, err := os.Stat(filepath.Join(dir, "mid.log")); err != nil {
t.Fatalf("expected mid.log to remain, stat error: %v", err)
}
if _, err := os.Stat(protected); err != nil {
t.Fatalf("expected protected main.log to remain, stat error: %v", err)
}
}
func TestEnforceLogDirSizeLimitSkipsProtected(t *testing.T) {
dir := t.TempDir()
protected := filepath.Join(dir, "main.log")
writeLogFile(t, protected, 200, time.Unix(1, 0))
writeLogFile(t, filepath.Join(dir, "other.log"), 50, time.Unix(2, 0))
deleted, err := enforceLogDirSizeLimit(dir, 100, protected)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if deleted != 1 {
t.Fatalf("expected 1 deleted file, got %d", deleted)
}
if _, err := os.Stat(protected); err != nil {
t.Fatalf("expected protected main.log to remain, stat error: %v", err)
}
if _, err := os.Stat(filepath.Join(dir, "other.log")); !os.IsNotExist(err) {
t.Fatalf("expected other.log to be removed, stat error: %v", err)
}
}
func writeLogFile(t *testing.T, path string, size int, modTime time.Time) {
t.Helper()
data := make([]byte, size)
if err := os.WriteFile(path, data, 0o644); err != nil {
t.Fatalf("write file: %v", err)
}
if err := os.Chtimes(path, modTime, modTime); err != nil {
t.Fatalf("set times: %v", err)
}
}

207
backend/internal/logging/request_logger.go Normal file
View file

@ -0,0 +1,207 @@
// Package logging provides request logging functionality for the CLI Proxy API server.
// It handles capturing and storing detailed HTTP request and response data when enabled
// through configuration, supporting both regular and streaming responses.
package logging
import (
"fmt"
"path/filepath"
"time"
"github.com/router-for-me/CLIProxyAPI/v7/internal/interfaces"
)
const (
WebsocketTimelineSourceContextKey = "WEBSOCKET_TIMELINE_SOURCE"
APIRequestSourceContextKey = "API_REQUEST_SOURCE"
DeferredAPIRequestContextKey = "DEFERRED_API_REQUEST"
APIResponseSourceContextKey = "API_RESPONSE_SOURCE"
APIResponseCapturedContextKey = "API_RESPONSE_CAPTURED"
APIWebsocketTimelineSourceContextKey = "API_WEBSOCKET_TIMELINE_SOURCE"
)
// DeferredAPIRequest builds an upstream request log only when an error log needs it.
type DeferredAPIRequest func() []byte
// RequestLogger defines the interface for logging HTTP requests and responses.
// It provides methods for logging both regular and streaming HTTP request/response cycles.
type RequestLogger interface {
// LogRequest logs a complete non-streaming request/response cycle.
//
// Parameters:
// - url: The request URL
// - method: The HTTP method
// - requestHeaders: The request headers
// - body: The request body
// - statusCode: The response status code
// - responseHeaders: The response headers
// - response: The raw response data
// - websocketTimeline: Optional downstream websocket event timeline
// - apiRequest: The API request data
// - apiResponse: The API response data
// - apiWebsocketTimeline: Optional upstream websocket event timeline
// - requestID: Optional request ID for log file naming
// - requestTimestamp: When the request was received
// - apiResponseTimestamp: When the API response was received
//
// Returns:
// - error: An error if logging fails, nil otherwise
LogRequest(url, method string, requestHeaders map[string][]string, body []byte, statusCode int, responseHeaders map[string][]string, response, websocketTimeline, apiRequest, apiResponse, apiWebsocketTimeline []byte, apiResponseErrors []*interfaces.ErrorMessage, requestID string, requestTimestamp, apiResponseTimestamp time.Time) error
// LogStreamingRequest initiates logging for a streaming request and returns a writer for chunks.
//
// Parameters:
// - url: The request URL
// - method: The HTTP method
// - headers: The request headers
// - body: The request body
// - requestID: Optional request ID for log file naming
//
// Returns:
// - StreamingLogWriter: A writer for streaming response chunks
// - error: An error if logging initialization fails, nil otherwise
LogStreamingRequest(url, method string, headers map[string][]string, body []byte, requestID string) (StreamingLogWriter, error)
// IsEnabled returns whether request logging is currently enabled.
//
// Returns:
// - bool: True if logging is enabled, false otherwise
IsEnabled() bool
}
// StreamingLogWriter handles real-time logging of streaming response chunks.
// It provides methods for writing streaming response data asynchronously.
type StreamingLogWriter interface {
// WriteChunkAsync writes a response chunk asynchronously (non-blocking).
//
// Parameters:
// - chunk: The response chunk to write
WriteChunkAsync(chunk []byte)
// WriteStatus writes the response status and headers to the log.
//
// Parameters:
// - status: The response status code
// - headers: The response headers
//
// Returns:
// - error: An error if writing fails, nil otherwise
WriteStatus(status int, headers map[string][]string) error
// WriteAPIRequest writes the upstream API request details to the log.
// This should be called before WriteStatus to maintain proper log ordering.
//
// Parameters:
// - apiRequest: The API request data (typically includes URL, headers, body sent upstream)
//
// Returns:
// - error: An error if writing fails, nil otherwise
WriteAPIRequest(apiRequest []byte) error
// WriteAPIResponse writes the upstream API response details to the log.
// This should be called after the streaming response is complete.
//
// Parameters:
// - apiResponse: The API response data
//
// Returns:
// - error: An error if writing fails, nil otherwise
WriteAPIResponse(apiResponse []byte) error
// WriteAPIWebsocketTimeline writes the upstream websocket timeline to the log.
// This should be called when upstream communication happened over websocket.
//
// Parameters:
// - apiWebsocketTimeline: The upstream websocket event timeline
//
// Returns:
// - error: An error if writing fails, nil otherwise
WriteAPIWebsocketTimeline(apiWebsocketTimeline []byte) error
// SetFirstChunkTimestamp sets the TTFB timestamp captured when first chunk was received.
//
// Parameters:
// - timestamp: The time when first response chunk was received
SetFirstChunkTimestamp(timestamp time.Time)
// Close finalizes the log file and cleans up resources.
//
// Returns:
// - error: An error if closing fails, nil otherwise
Close() error
}
// FileRequestLogger implements RequestLogger using file-based storage.
// It provides file-based logging functionality for HTTP requests and responses.
type FileRequestLogger struct {
// enabled indicates whether request logging is currently enabled.
enabled bool
// logsDir is the directory where log files are stored.
logsDir string
// errorLogsMaxFiles limits the number of error log files retained.
errorLogsMaxFiles int
homeEnabled bool
}
// NewFileRequestLogger creates a new file-based request logger.
//
// Parameters:
// - enabled: Whether request logging should be enabled
// - logsDir: The directory where log files should be stored (can be relative)
// - configDir: The directory of the configuration file; when logsDir is
// relative, it will be resolved relative to this directory
// - errorLogsMaxFiles: Maximum number of error log files to retain (0 = no cleanup)
//
// Returns:
// - *FileRequestLogger: A new file-based request logger instance
func NewFileRequestLogger(enabled bool, logsDir string, configDir string, errorLogsMaxFiles int) *FileRequestLogger {
// Resolve logsDir relative to the configuration file directory when it's not absolute.
if !filepath.IsAbs(logsDir) {
// If configDir is provided, resolve logsDir relative to it.
if configDir != "" {
logsDir = filepath.Join(configDir, logsDir)
}
}
return &FileRequestLogger{
enabled: enabled,
logsDir: logsDir,
errorLogsMaxFiles: errorLogsMaxFiles,
homeEnabled: false,
}
}
// IsEnabled returns whether request logging is currently enabled.
//
// Returns:
// - bool: True if logging is enabled, false otherwise
func (l *FileRequestLogger) IsEnabled() bool {
return l.enabled
}
// SetEnabled updates the request logging enabled state.
// This method allows dynamic enabling/disabling of request logging.
//
// Parameters:
// - enabled: Whether request logging should be enabled
func (l *FileRequestLogger) SetEnabled(enabled bool) {
l.enabled = enabled
}
// SetErrorLogsMaxFiles updates the maximum number of error log files to retain.
func (l *FileRequestLogger) SetErrorLogsMaxFiles(maxFiles int) {
l.errorLogsMaxFiles = maxFiles
}
// NewFileBodySource creates a temp-backed source under the request log directory.
func (l *FileRequestLogger) NewFileBodySource(prefix string) (*FileBodySource, error) {
if l == nil {
return nil, fmt.Errorf("file request logger is nil")
}
if errEnsure := l.ensureLogsDir(); errEnsure != nil {
return nil, errEnsure
}
return NewFileBodySourceInDir(l.logsDir, prefix)
}

256
backend/internal/logging/request_logger_body_source.go Normal file
View file

@ -0,0 +1,256 @@
package logging
import (
"bytes"
"fmt"
"io"
"os"
"strings"
"sync"
log "github.com/sirupsen/logrus"
)
// FileBodySource stores large log sections as ordered temp-file parts.
type FileBodySource struct {
mu sync.Mutex
dir string
paths []string
cleaned bool
}
// NewFileBodySourceInDir creates a temp-backed source under baseDir.
func NewFileBodySourceInDir(baseDir string, prefix string) (*FileBodySource, error) {
prefix = sanitizeTempPrefix(prefix)
baseDir = strings.TrimSpace(baseDir)
if baseDir == "" {
return nil, fmt.Errorf("base directory is required")
}
if errMkdir := os.MkdirAll(baseDir, 0755); errMkdir != nil {
return nil, errMkdir
}
dir, errCreate := os.MkdirTemp(baseDir, "request-log-parts-"+prefix+"-*")
if errCreate != nil {
return nil, errCreate
}
return &FileBodySource{dir: dir}, nil
}
func sanitizeTempPrefix(prefix string) string {
prefix = strings.TrimSpace(prefix)
if prefix == "" {
return "log"
}
var builder strings.Builder
for _, r := range prefix {
switch {
case r >= 'a' && r <= 'z':
builder.WriteRune(r)
case r >= 'A' && r <= 'Z':
builder.WriteRune(r)
case r >= '0' && r <= '9':
builder.WriteRune(r)
case r == '-' || r == '_':
builder.WriteRune(r)
default:
builder.WriteByte('-')
}
}
out := strings.Trim(builder.String(), "-_")
if out == "" {
return "log"
}
return out
}
// CreatePart creates one ordered detail log part.
func (s *FileBodySource) CreatePart(prefix string) (*os.File, error) {
if s == nil {
return nil, fmt.Errorf("file body source is nil")
}
s.mu.Lock()
defer s.mu.Unlock()
if s.cleaned {
return nil, fmt.Errorf("file body source has been cleaned")
}
prefix = sanitizeTempPrefix(prefix)
if errMkdir := os.MkdirAll(s.dir, 0755); errMkdir != nil {
return nil, errMkdir
}
file, errCreate := os.CreateTemp(s.dir, prefix+"-*.tmp")
if errCreate != nil {
return nil, errCreate
}
s.paths = append(s.paths, file.Name())
return file, nil
}
// AppendPart appends one complete ordered part to the source.
func (s *FileBodySource) AppendPart(data []byte) error {
data = bytes.TrimSpace(data)
if len(data) == 0 {
return nil
}
file, errCreate := s.CreatePart("part")
if errCreate != nil {
return errCreate
}
writeErr := writeLogPart(file, data, false)
if errClose := file.Close(); errClose != nil {
if writeErr == nil {
writeErr = errClose
}
}
return writeErr
}
// AppendBytes appends raw bytes to a single ordered part.
func (s *FileBodySource) AppendBytes(data []byte) error {
if s == nil {
return fmt.Errorf("file body source is nil")
}
if len(data) == 0 {
return nil
}
s.mu.Lock()
defer s.mu.Unlock()
if s.cleaned {
return fmt.Errorf("file body source has been cleaned")
}
if errMkdir := os.MkdirAll(s.dir, 0755); errMkdir != nil {
return errMkdir
}
var file *os.File
var errOpen error
if len(s.paths) == 0 {
file, errOpen = os.CreateTemp(s.dir, "part-*.tmp")
if errOpen == nil {
s.paths = append(s.paths, file.Name())
}
} else {
file, errOpen = os.OpenFile(s.paths[len(s.paths)-1], os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
}
if errOpen != nil {
return errOpen
}
_, writeErr := file.Write(data)
if errClose := file.Close(); errClose != nil {
if writeErr == nil {
writeErr = errClose
}
}
return writeErr
}
// HasPayload reports whether any detail parts were recorded.
func (s *FileBodySource) HasPayload() bool {
if s == nil {
return false
}
s.mu.Lock()
defer s.mu.Unlock()
return len(s.paths) > 0 && !s.cleaned
}
// Paths returns a copy of the ordered part paths.
func (s *FileBodySource) Paths() []string {
if s == nil {
return nil
}
s.mu.Lock()
defer s.mu.Unlock()
out := make([]string, len(s.paths))
copy(out, s.paths)
return out
}
// WriteTo merges all ordered parts into w.
func (s *FileBodySource) WriteTo(w io.Writer) error {
if s == nil || w == nil {
return nil
}
paths := s.Paths()
wrote := false
for _, path := range paths {
file, errOpen := os.Open(path)
if errOpen != nil {
if os.IsNotExist(errOpen) {
continue
}
return errOpen
}
if wrote {
if _, errWrite := io.WriteString(w, "\n"); errWrite != nil {
if errClose := file.Close(); errClose != nil {
log.WithError(errClose).Warn("failed to close log part file")
}
return errWrite
}
}
_, errCopy := io.Copy(w, file)
if errClose := file.Close(); errClose != nil {
log.WithError(errClose).Warn("failed to close log part file")
if errCopy == nil {
errCopy = errClose
}
}
if errCopy != nil {
return errCopy
}
wrote = true
}
return nil
}
// Bytes merges all ordered parts into memory.
func (s *FileBodySource) Bytes() ([]byte, error) {
var buf bytes.Buffer
if errWrite := s.WriteTo(&buf); errWrite != nil {
return nil, errWrite
}
return buf.Bytes(), nil
}
// Cleanup removes all temp detail parts and their directory.
func (s *FileBodySource) Cleanup() error {
if s == nil {
return nil
}
s.mu.Lock()
if s.cleaned {
s.mu.Unlock()
return nil
}
paths := make([]string, len(s.paths))
copy(paths, s.paths)
dir := s.dir
s.paths = nil
s.cleaned = true
s.mu.Unlock()
var firstErr error
for _, path := range paths {
if errRemove := os.Remove(path); errRemove != nil && !os.IsNotExist(errRemove) && firstErr == nil {
firstErr = errRemove
}
}
if dir != "" {
if errRemove := os.RemoveAll(dir); errRemove != nil && firstErr == nil {
firstErr = errRemove
}
}
return firstErr
}
func cleanupFileBodySources(sources ...*FileBodySource) {
for _, source := range sources {
if source == nil {
continue
}
if errCleanup := source.Cleanup(); errCleanup != nil {
log.WithError(errCleanup).Warn("failed to clean up log part files")
}
}
}

720
backend/internal/logging/request_logger_format.go Normal file
View file

@ -0,0 +1,720 @@
package logging
import (
"bufio"
"bytes"
"compress/flate"
"compress/gzip"
"fmt"
"io"
"os"
"strings"
"time"
"github.com/andybalholm/brotli"
"github.com/klauspost/compress/zstd"
"github.com/router-for-me/CLIProxyAPI/v7/internal/buildinfo"
"github.com/router-for-me/CLIProxyAPI/v7/internal/interfaces"
"github.com/router-for-me/CLIProxyAPI/v7/internal/util"
log "github.com/sirupsen/logrus"
)
func (l *FileRequestLogger) writeNonStreamingLog(
w io.Writer,
url, method string,
requestHeaders map[string][]string,
requestBody []byte,
requestBodyPath string,
websocketTimeline []byte,
websocketTimelineSource *FileBodySource,
apiRequest []byte,
apiRequestSource *FileBodySource,
apiResponse []byte,
apiResponseSource *FileBodySource,
apiWebsocketTimeline []byte,
apiWebsocketTimelineSource *FileBodySource,
apiResponseErrors []*interfaces.ErrorMessage,
statusCode int,
responseHeaders map[string][]string,
response []byte,
decompressErr error,
requestTimestamp time.Time,
apiResponseTimestamp time.Time,
) error {
if requestTimestamp.IsZero() {
requestTimestamp = time.Now()
}
isWebsocketTranscript := hasSectionPayload(websocketTimeline) || hasFileBodySourcePayload(websocketTimelineSource)
downstreamTransport := inferDownstreamTransport(requestHeaders, websocketTimeline, websocketTimelineSource)
upstreamTransport := inferUpstreamTransport(apiRequest, apiRequestSource, apiResponse, apiResponseSource, apiWebsocketTimeline, apiWebsocketTimelineSource, apiResponseErrors)
if errWrite := writeRequestInfoWithBody(w, url, method, requestHeaders, requestBody, requestBodyPath, requestTimestamp, downstreamTransport, upstreamTransport, !isWebsocketTranscript); errWrite != nil {
return errWrite
}
if errWrite := writeAPISectionWithSource(w, "=== WEBSOCKET TIMELINE ===\n", "=== WEBSOCKET TIMELINE", websocketTimeline, websocketTimelineSource, time.Time{}); errWrite != nil {
return errWrite
}
if errWrite := writeAPISectionWithSource(w, "=== API WEBSOCKET TIMELINE ===\n", "=== API WEBSOCKET TIMELINE", apiWebsocketTimeline, apiWebsocketTimelineSource, time.Time{}); errWrite != nil {
return errWrite
}
if errWrite := writePreformattedAPISectionWithSource(w, "=== API REQUEST ===\n", "=== API REQUEST", apiRequest, apiRequestSource, time.Time{}); errWrite != nil {
return errWrite
}
if errWrite := writeAPIErrorResponses(w, apiResponseErrors); errWrite != nil {
return errWrite
}
if errWrite := writePreformattedAPISectionWithSource(w, "=== API RESPONSE ===\n", "=== API RESPONSE", apiResponse, apiResponseSource, apiResponseTimestamp); errWrite != nil {
return errWrite
}
if isWebsocketTranscript {
// Intentionally omit the generic downstream HTTP response section for websocket
// transcripts. The durable session exchange is captured in WEBSOCKET TIMELINE,
// and appending a one-off upgrade response snapshot would dilute that transcript.
return nil
}
return writeResponseSection(w, statusCode, true, responseHeaders, bytes.NewReader(response), decompressErr, true)
}
func writeRequestInfoWithBody(
w io.Writer,
url, method string,
headers map[string][]string,
body []byte,
bodyPath string,
timestamp time.Time,
downstreamTransport string,
upstreamTransport string,
includeBody bool,
) error {
if _, errWrite := io.WriteString(w, "=== REQUEST INFO ===\n"); errWrite != nil {
return errWrite
}
if _, errWrite := io.WriteString(w, fmt.Sprintf("Version: %s\n", buildinfo.Version)); errWrite != nil {
return errWrite
}
if _, errWrite := io.WriteString(w, fmt.Sprintf("URL: %s\n", url)); errWrite != nil {
return errWrite
}
if _, errWrite := io.WriteString(w, fmt.Sprintf("Method: %s\n", method)); errWrite != nil {
return errWrite
}
if strings.TrimSpace(downstreamTransport) != "" {
if _, errWrite := io.WriteString(w, fmt.Sprintf("Downstream Transport: %s\n", downstreamTransport)); errWrite != nil {
return errWrite
}
}
if strings.TrimSpace(upstreamTransport) != "" {
if _, errWrite := io.WriteString(w, fmt.Sprintf("Upstream Transport: %s\n", upstreamTransport)); errWrite != nil {
return errWrite
}
}
if _, errWrite := io.WriteString(w, fmt.Sprintf("Timestamp: %s\n", timestamp.Format(time.RFC3339Nano))); errWrite != nil {
return errWrite
}
if errWrite := writeSectionSpacing(w, 1); errWrite != nil {
return errWrite
}
if _, errWrite := io.WriteString(w, "=== HEADERS ===\n"); errWrite != nil {
return errWrite
}
for key, values := range headers {
for _, value := range values {
masked := util.MaskSensitiveHeaderValue(key, value)
if _, errWrite := io.WriteString(w, fmt.Sprintf("%s: %s\n", key, masked)); errWrite != nil {
return errWrite
}
}
}
if errWrite := writeSectionSpacing(w, 1); errWrite != nil {
return errWrite
}
if !includeBody {
return nil
}
if _, errWrite := io.WriteString(w, "=== REQUEST BODY ===\n"); errWrite != nil {
return errWrite
}
bodyTrailingNewlines := 1
if bodyPath != "" {
bodyFile, errOpen := os.Open(bodyPath)
if errOpen != nil {
return errOpen
}
tracker := &trailingNewlineTrackingWriter{writer: w}
written, errCopy := io.Copy(tracker, bodyFile)
if errCopy != nil {
_ = bodyFile.Close()
return errCopy
}
if written > 0 {
bodyTrailingNewlines = tracker.trailingNewlines
}
if errClose := bodyFile.Close(); errClose != nil {
log.WithError(errClose).Warn("failed to close request body temp file")
}
} else if _, errWrite := w.Write(body); errWrite != nil {
return errWrite
} else if len(body) > 0 {
bodyTrailingNewlines = countTrailingNewlinesBytes(body)
}
if errWrite := writeSectionSpacing(w, bodyTrailingNewlines); errWrite != nil {
return errWrite
}
return nil
}
func countTrailingNewlinesBytes(payload []byte) int {
count := 0
for i := len(payload) - 1; i >= 0; i-- {
if payload[i] != '\n' {
break
}
count++
}
return count
}
func writeSectionSpacing(w io.Writer, trailingNewlines int) error {
missingNewlines := 3 - trailingNewlines
if missingNewlines <= 0 {
return nil
}
_, errWrite := io.WriteString(w, strings.Repeat("\n", missingNewlines))
return errWrite
}
type trailingNewlineTrackingWriter struct {
writer io.Writer
trailingNewlines int
}
func (t *trailingNewlineTrackingWriter) Write(payload []byte) (int, error) {
written, errWrite := t.writer.Write(payload)
if written > 0 {
writtenPayload := payload[:written]
trailingNewlines := countTrailingNewlinesBytes(writtenPayload)
if trailingNewlines == len(writtenPayload) {
t.trailingNewlines += trailingNewlines
} else {
t.trailingNewlines = trailingNewlines
}
}
return written, errWrite
}
func hasSectionPayload(payload []byte) bool {
return len(bytes.TrimSpace(payload)) > 0
}
func hasFileBodySourcePayload(source *FileBodySource) bool {
return source != nil && source.HasPayload()
}
func inferDownstreamTransport(headers map[string][]string, websocketTimeline []byte, websocketTimelineSource *FileBodySource) string {
if hasSectionPayload(websocketTimeline) || hasFileBodySourcePayload(websocketTimelineSource) {
return "websocket"
}
for key, values := range headers {
if strings.EqualFold(strings.TrimSpace(key), "Upgrade") {
for _, value := range values {
if strings.EqualFold(strings.TrimSpace(value), "websocket") {
return "websocket"
}
}
}
}
return "http"
}
func inferUpstreamTransport(apiRequest []byte, apiRequestSource *FileBodySource, apiResponse []byte, apiResponseSource *FileBodySource, apiWebsocketTimeline []byte, apiWebsocketTimelineSource *FileBodySource, _ []*interfaces.ErrorMessage) string {
hasHTTP := hasSectionPayload(apiRequest) || hasFileBodySourcePayload(apiRequestSource) || hasSectionPayload(apiResponse) || hasFileBodySourcePayload(apiResponseSource)
hasWS := hasSectionPayload(apiWebsocketTimeline) || hasFileBodySourcePayload(apiWebsocketTimelineSource)
switch {
case hasHTTP && hasWS:
return "websocket+http"
case hasWS:
return "websocket"
case hasHTTP:
return "http"
default:
return ""
}
}
func writeLogPart(w io.Writer, payload []byte, prependNewline bool) error {
if w == nil {
return nil
}
if prependNewline {
if _, errWrite := io.WriteString(w, "\n"); errWrite != nil {
return errWrite
}
}
if _, errWrite := w.Write(payload); errWrite != nil {
return errWrite
}
if !bytes.HasSuffix(payload, []byte("\n")) {
if _, errWrite := io.WriteString(w, "\n"); errWrite != nil {
return errWrite
}
}
return nil
}
func writeAPISection(w io.Writer, sectionHeader string, sectionPrefix string, payload []byte, timestamp time.Time) error {
if len(payload) == 0 {
return nil
}
if bytes.HasPrefix(payload, []byte(sectionPrefix)) {
if _, errWrite := w.Write(payload); errWrite != nil {
return errWrite
}
} else {
if _, errWrite := io.WriteString(w, sectionHeader); errWrite != nil {
return errWrite
}
if !timestamp.IsZero() {
if _, errWrite := io.WriteString(w, fmt.Sprintf("Timestamp: %s\n", timestamp.Format(time.RFC3339Nano))); errWrite != nil {
return errWrite
}
}
if _, errWrite := w.Write(payload); errWrite != nil {
return errWrite
}
}
if errWrite := writeSectionSpacing(w, countTrailingNewlinesBytes(payload)); errWrite != nil {
return errWrite
}
return nil
}
func writeAPISectionWithSource(w io.Writer, sectionHeader string, sectionPrefix string, payload []byte, source *FileBodySource, timestamp time.Time) error {
if !hasFileBodySourcePayload(source) {
return writeAPISection(w, sectionHeader, sectionPrefix, payload, timestamp)
}
if len(payload) > 0 {
if errWrite := writeAPISection(w, sectionHeader, sectionPrefix, payload, timestamp); errWrite != nil {
return errWrite
}
}
if _, errWrite := io.WriteString(w, sectionHeader); errWrite != nil {
return errWrite
}
if !timestamp.IsZero() {
if _, errWrite := io.WriteString(w, fmt.Sprintf("Timestamp: %s\n", timestamp.Format(time.RFC3339Nano))); errWrite != nil {
return errWrite
}
}
tracker := &trailingNewlineTrackingWriter{writer: w}
if errWrite := source.WriteTo(tracker); errWrite != nil {
return errWrite
}
if errWrite := writeSectionSpacing(w, tracker.trailingNewlines); errWrite != nil {
return errWrite
}
return nil
}
func writePreformattedAPISectionWithSource(w io.Writer, sectionHeader string, sectionPrefix string, payload []byte, source *FileBodySource, timestamp time.Time) error {
if !hasFileBodySourcePayload(source) {
return writeAPISection(w, sectionHeader, sectionPrefix, payload, timestamp)
}
if len(payload) > 0 {
if errWrite := writeAPISection(w, sectionHeader, sectionPrefix, payload, timestamp); errWrite != nil {
return errWrite
}
}
tracker := &trailingNewlineTrackingWriter{writer: w}
if errWrite := source.WriteTo(tracker); errWrite != nil {
return errWrite
}
if errWrite := writeSectionSpacing(w, tracker.trailingNewlines); errWrite != nil {
return errWrite
}
return nil
}
func writeAPIErrorResponses(w io.Writer, apiResponseErrors []*interfaces.ErrorMessage) error {
for i := 0; i < len(apiResponseErrors); i++ {
if apiResponseErrors[i] == nil {
continue
}
if _, errWrite := io.WriteString(w, "=== API ERROR RESPONSE ===\n"); errWrite != nil {
return errWrite
}
if _, errWrite := io.WriteString(w, fmt.Sprintf("HTTP Status: %d\n", apiResponseErrors[i].StatusCode)); errWrite != nil {
return errWrite
}
trailingNewlines := 1
if apiResponseErrors[i].Error != nil {
errText := apiResponseErrors[i].Error.Error()
if _, errWrite := io.WriteString(w, errText); errWrite != nil {
return errWrite
}
if errText != "" {
trailingNewlines = countTrailingNewlinesBytes([]byte(errText))
}
}
if errWrite := writeSectionSpacing(w, trailingNewlines); errWrite != nil {
return errWrite
}
}
return nil
}
func writeResponseSection(w io.Writer, statusCode int, statusWritten bool, responseHeaders map[string][]string, responseReader io.Reader, decompressErr error, trailingNewline bool) error {
if _, errWrite := io.WriteString(w, "=== RESPONSE ===\n"); errWrite != nil {
return errWrite
}
if statusWritten {
if _, errWrite := io.WriteString(w, fmt.Sprintf("Status: %d\n", statusCode)); errWrite != nil {
return errWrite
}
}
if responseHeaders != nil {
for key, values := range responseHeaders {
for _, value := range values {
if _, errWrite := io.WriteString(w, fmt.Sprintf("%s: %s\n", key, value)); errWrite != nil {
return errWrite
}
}
}
}
var bufferedReader *bufio.Reader
if responseReader != nil {
bufferedReader = bufio.NewReader(responseReader)
}
if !responseBodyStartsWithLeadingNewline(bufferedReader) {
if _, errWrite := io.WriteString(w, "\n"); errWrite != nil {
return errWrite
}
}
if bufferedReader != nil {
if _, errCopy := io.Copy(w, bufferedReader); errCopy != nil {
return errCopy
}
}
if decompressErr != nil {
if _, errWrite := io.WriteString(w, fmt.Sprintf("\n[DECOMPRESSION ERROR: %v]", decompressErr)); errWrite != nil {
return errWrite
}
}
if trailingNewline {
if _, errWrite := io.WriteString(w, "\n"); errWrite != nil {
return errWrite
}
}
return nil
}
func responseBodyStartsWithLeadingNewline(reader *bufio.Reader) bool {
if reader == nil {
return false
}
if peeked, _ := reader.Peek(2); len(peeked) >= 2 && peeked[0] == '\r' && peeked[1] == '\n' {
return true
}
if peeked, _ := reader.Peek(1); len(peeked) >= 1 && peeked[0] == '\n' {
return true
}
return false
}
// formatLogContent creates the complete log content for non-streaming requests.
//
// Parameters:
// - url: The request URL
// - method: The HTTP method
// - headers: The request headers
// - body: The request body
// - websocketTimeline: The downstream websocket event timeline
// - apiRequest: The API request data
// - apiResponse: The API response data
// - response: The raw response data
// - status: The response status code
// - responseHeaders: The response headers
//
// Returns:
// - string: The formatted log content
func (l *FileRequestLogger) formatLogContent(url, method string, headers map[string][]string, body, websocketTimeline, apiRequest, apiResponse, apiWebsocketTimeline, response []byte, status int, responseHeaders map[string][]string, apiResponseErrors []*interfaces.ErrorMessage) string {
var content strings.Builder
isWebsocketTranscript := hasSectionPayload(websocketTimeline)
downstreamTransport := inferDownstreamTransport(headers, websocketTimeline, nil)
upstreamTransport := inferUpstreamTransport(apiRequest, nil, apiResponse, nil, apiWebsocketTimeline, nil, apiResponseErrors)
// Request info
content.WriteString(l.formatRequestInfo(url, method, headers, body, downstreamTransport, upstreamTransport, !isWebsocketTranscript))
if len(websocketTimeline) > 0 {
if bytes.HasPrefix(websocketTimeline, []byte("=== WEBSOCKET TIMELINE")) {
content.Write(websocketTimeline)
if !bytes.HasSuffix(websocketTimeline, []byte("\n")) {
content.WriteString("\n")
}
} else {
content.WriteString("=== WEBSOCKET TIMELINE ===\n")
content.Write(websocketTimeline)
content.WriteString("\n")
}
content.WriteString("\n")
}
if len(apiWebsocketTimeline) > 0 {
if bytes.HasPrefix(apiWebsocketTimeline, []byte("=== API WEBSOCKET TIMELINE")) {
content.Write(apiWebsocketTimeline)
if !bytes.HasSuffix(apiWebsocketTimeline, []byte("\n")) {
content.WriteString("\n")
}
} else {
content.WriteString("=== API WEBSOCKET TIMELINE ===\n")
content.Write(apiWebsocketTimeline)
content.WriteString("\n")
}
content.WriteString("\n")
}
if len(apiRequest) > 0 {
if bytes.HasPrefix(apiRequest, []byte("=== API REQUEST")) {
content.Write(apiRequest)
if !bytes.HasSuffix(apiRequest, []byte("\n")) {
content.WriteString("\n")
}
} else {
content.WriteString("=== API REQUEST ===\n")
content.Write(apiRequest)
content.WriteString("\n")
}
content.WriteString("\n")
}
for i := 0; i < len(apiResponseErrors); i++ {
content.WriteString("=== API ERROR RESPONSE ===\n")
content.WriteString(fmt.Sprintf("HTTP Status: %d\n", apiResponseErrors[i].StatusCode))
content.WriteString(apiResponseErrors[i].Error.Error())
content.WriteString("\n\n")
}
if len(apiResponse) > 0 {
if bytes.HasPrefix(apiResponse, []byte("=== API RESPONSE")) {
content.Write(apiResponse)
if !bytes.HasSuffix(apiResponse, []byte("\n")) {
content.WriteString("\n")
}
} else {
content.WriteString("=== API RESPONSE ===\n")
content.Write(apiResponse)
content.WriteString("\n")
}
content.WriteString("\n")
}
if isWebsocketTranscript {
// Mirror writeNonStreamingLog: websocket transcripts end with the dedicated
// timeline sections instead of a generic downstream HTTP response block.
return content.String()
}
// Response section
content.WriteString("=== RESPONSE ===\n")
content.WriteString(fmt.Sprintf("Status: %d\n", status))
if responseHeaders != nil {
for key, values := range responseHeaders {
for _, value := range values {
content.WriteString(fmt.Sprintf("%s: %s\n", key, value))
}
}
}
content.WriteString("\n")
content.Write(response)
content.WriteString("\n")
return content.String()
}
// decompressResponse decompresses response data based on Content-Encoding header.
//
// Parameters:
// - responseHeaders: The response headers
// - response: The response data to decompress
//
// Returns:
// - []byte: The decompressed response data
// - error: An error if decompression fails, nil otherwise
func (l *FileRequestLogger) decompressResponse(responseHeaders map[string][]string, response []byte) ([]byte, error) {
if responseHeaders == nil || len(response) == 0 {
return response, nil
}
// Check Content-Encoding header
var contentEncoding string
for key, values := range responseHeaders {
if strings.ToLower(key) == "content-encoding" && len(values) > 0 {
contentEncoding = strings.ToLower(values[0])
break
}
}
switch contentEncoding {
case "gzip":
return l.decompressGzip(response)
case "deflate":
return l.decompressDeflate(response)
case "br":
return l.decompressBrotli(response)
case "zstd":
return l.decompressZstd(response)
default:
// No compression or unsupported compression
return response, nil
}
}
// decompressGzip decompresses gzip-encoded data.
//
// Parameters:
// - data: The gzip-encoded data to decompress
//
// Returns:
// - []byte: The decompressed data
// - error: An error if decompression fails, nil otherwise
func (l *FileRequestLogger) decompressGzip(data []byte) ([]byte, error) {
reader, err := gzip.NewReader(bytes.NewReader(data))
if err != nil {
return nil, fmt.Errorf("failed to create gzip reader: %w", err)
}
defer func() {
if errClose := reader.Close(); errClose != nil {
log.WithError(errClose).Warn("failed to close gzip reader in request logger")
}
}()
decompressed, err := io.ReadAll(reader)
if err != nil {
return nil, fmt.Errorf("failed to decompress gzip data: %w", err)
}
return decompressed, nil
}
// decompressDeflate decompresses deflate-encoded data.
//
// Parameters:
// - data: The deflate-encoded data to decompress
//
// Returns:
// - []byte: The decompressed data
// - error: An error if decompression fails, nil otherwise
func (l *FileRequestLogger) decompressDeflate(data []byte) ([]byte, error) {
reader := flate.NewReader(bytes.NewReader(data))
defer func() {
if errClose := reader.Close(); errClose != nil {
log.WithError(errClose).Warn("failed to close deflate reader in request logger")
}
}()
decompressed, err := io.ReadAll(reader)
if err != nil {
return nil, fmt.Errorf("failed to decompress deflate data: %w", err)
}
return decompressed, nil
}
// decompressBrotli decompresses brotli-encoded data.
//
// Parameters:
// - data: The brotli-encoded data to decompress
//
// Returns:
// - []byte: The decompressed data
// - error: An error if decompression fails, nil otherwise
func (l *FileRequestLogger) decompressBrotli(data []byte) ([]byte, error) {
reader := brotli.NewReader(bytes.NewReader(data))
decompressed, err := io.ReadAll(reader)
if err != nil {
return nil, fmt.Errorf("failed to decompress brotli data: %w", err)
}
return decompressed, nil
}
// decompressZstd decompresses zstd-encoded data.
//
// Parameters:
// - data: The zstd-encoded data to decompress
//
// Returns:
// - []byte: The decompressed data
// - error: An error if decompression fails, nil otherwise
func (l *FileRequestLogger) decompressZstd(data []byte) ([]byte, error) {
decoder, err := zstd.NewReader(bytes.NewReader(data))
if err != nil {
return nil, fmt.Errorf("failed to create zstd reader: %w", err)
}
defer decoder.Close()
decompressed, err := io.ReadAll(decoder)
if err != nil {
return nil, fmt.Errorf("failed to decompress zstd data: %w", err)
}
return decompressed, nil
}
// formatRequestInfo creates the request information section of the log.
//
// Parameters:
// - url: The request URL
// - method: The HTTP method
// - headers: The request headers
// - body: The request body
//
// Returns:
// - string: The formatted request information
func (l *FileRequestLogger) formatRequestInfo(url, method string, headers map[string][]string, body []byte, downstreamTransport string, upstreamTransport string, includeBody bool) string {
var content strings.Builder
content.WriteString("=== REQUEST INFO ===\n")
content.WriteString(fmt.Sprintf("Version: %s\n", buildinfo.Version))
content.WriteString(fmt.Sprintf("URL: %s\n", url))
content.WriteString(fmt.Sprintf("Method: %s\n", method))
if strings.TrimSpace(downstreamTransport) != "" {
content.WriteString(fmt.Sprintf("Downstream Transport: %s\n", downstreamTransport))
}
if strings.TrimSpace(upstreamTransport) != "" {
content.WriteString(fmt.Sprintf("Upstream Transport: %s\n", upstreamTransport))
}
content.WriteString(fmt.Sprintf("Timestamp: %s\n", time.Now().Format(time.RFC3339Nano)))
content.WriteString("\n")
content.WriteString("=== HEADERS ===\n")
for key, values := range headers {
for _, value := range values {
masked := util.MaskSensitiveHeaderValue(key, value)
content.WriteString(fmt.Sprintf("%s: %s\n", key, masked))
}
}
content.WriteString("\n")
if !includeBody {
return content.String()
}
content.WriteString("=== REQUEST BODY ===\n")
content.Write(body)
content.WriteString("\n\n")
return content.String()
}

246
backend/internal/logging/request_logger_home.go Normal file
View file

@ -0,0 +1,246 @@
package logging
import (
"bytes"
"context"
"encoding/json"
"strings"
"time"
"github.com/router-for-me/CLIProxyAPI/v7/internal/home"
)
type homeRequestLogClient interface {
HeartbeatOK() bool
RPushRequestLog(ctx context.Context, payload []byte) error
}
var currentHomeRequestLogClient = func() homeRequestLogClient {
return home.Current()
}
type homeRequestLogPayload struct {
Headers map[string][]string `json:"headers,omitempty"`
RequestID string `json:"request_id,omitempty"`
RequestLog string `json:"request_log,omitempty"`
}
func cloneHeaders(headers map[string][]string) map[string][]string {
if len(headers) == 0 {
return nil
}
out := make(map[string][]string, len(headers))
for key, values := range headers {
if strings.TrimSpace(key) == "" {
continue
}
if values == nil {
out[key] = nil
continue
}
copied := make([]string, len(values))
copy(copied, values)
out[key] = copied
}
if len(out) == 0 {
return nil
}
return out
}
func (l *FileRequestLogger) forwardRequestLogToHome(ctx context.Context, headers map[string][]string, requestID string, logText string) error {
if l == nil || !l.homeEnabled {
return nil
}
client := currentHomeRequestLogClient()
if client == nil || !client.HeartbeatOK() {
return nil
}
payload := homeRequestLogPayload{
Headers: cloneHeaders(headers),
RequestID: strings.TrimSpace(requestID),
RequestLog: logText,
}
raw, errMarshal := json.Marshal(&payload)
if errMarshal != nil {
return errMarshal
}
if ctx == nil {
ctx = context.Background()
}
return client.RPushRequestLog(ctx, raw)
}
// SetHomeEnabled toggles home request-log forwarding.
// When enabled, request logs are not written to disk and are instead forwarded to home via Redis RESP.
func (l *FileRequestLogger) SetHomeEnabled(enabled bool) {
if l == nil {
return
}
l.homeEnabled = enabled
}
type homeStreamingLogWriter struct {
url string
method string
timestamp time.Time
requestHeaders map[string][]string
requestBody []byte
chunkChan chan []byte
doneChan chan struct{}
responseStatus int
statusWritten bool
responseHeaders map[string][]string
responseBody bytes.Buffer
apiRequest []byte
apiResponse []byte
apiWebsocketTime []byte
requestID string
apiResponseTS time.Time
firstChunkTS time.Time
}
func newHomeStreamingLogWriter(url, method string, headers map[string][]string, body []byte, requestID string) *homeStreamingLogWriter {
requestHeaders := make(map[string][]string, len(headers))
for key, values := range headers {
headerValues := make([]string, len(values))
copy(headerValues, values)
requestHeaders[key] = headerValues
}
writer := &homeStreamingLogWriter{
url: url,
method: method,
timestamp: time.Now(),
requestHeaders: requestHeaders,
requestBody: append([]byte(nil), body...),
requestID: strings.TrimSpace(requestID),
chunkChan: make(chan []byte, 100),
doneChan: make(chan struct{}),
}
go writer.asyncWriter()
return writer
}
func (w *homeStreamingLogWriter) asyncWriter() {
defer close(w.doneChan)
for chunk := range w.chunkChan {
if len(chunk) == 0 {
continue
}
_, _ = w.responseBody.Write(chunk)
}
}
func (w *homeStreamingLogWriter) WriteChunkAsync(chunk []byte) {
if w == nil || w.chunkChan == nil || len(chunk) == 0 {
return
}
select {
case w.chunkChan <- append([]byte(nil), chunk...):
default:
}
}
func (w *homeStreamingLogWriter) WriteStatus(status int, headers map[string][]string) error {
if w == nil || status == 0 {
return nil
}
w.responseStatus = status
w.statusWritten = true
if headers != nil {
w.responseHeaders = make(map[string][]string, len(headers))
for key, values := range headers {
copied := make([]string, len(values))
copy(copied, values)
w.responseHeaders[key] = copied
}
}
return nil
}
func (w *homeStreamingLogWriter) WriteAPIRequest(apiRequest []byte) error {
if w == nil || len(apiRequest) == 0 {
return nil
}
w.apiRequest = bytes.Clone(apiRequest)
return nil
}
func (w *homeStreamingLogWriter) WriteAPIResponse(apiResponse []byte) error {
if w == nil || len(apiResponse) == 0 {
return nil
}
w.apiResponse = bytes.Clone(apiResponse)
return nil
}
func (w *homeStreamingLogWriter) WriteAPIWebsocketTimeline(apiWebsocketTimeline []byte) error {
if w == nil || len(apiWebsocketTimeline) == 0 {
return nil
}
w.apiWebsocketTime = bytes.Clone(apiWebsocketTimeline)
return nil
}
func (w *homeStreamingLogWriter) SetFirstChunkTimestamp(timestamp time.Time) {
if w == nil {
return
}
if !timestamp.IsZero() {
w.firstChunkTS = timestamp
w.apiResponseTS = timestamp
}
}
func (w *homeStreamingLogWriter) Close() error {
if w == nil {
return nil
}
client := currentHomeRequestLogClient()
if client == nil || !client.HeartbeatOK() {
return nil
}
if w.chunkChan != nil {
close(w.chunkChan)
<-w.doneChan
w.chunkChan = nil
}
responsePayload := w.responseBody.Bytes()
var buf bytes.Buffer
upstreamTransport := inferUpstreamTransport(w.apiRequest, nil, w.apiResponse, nil, w.apiWebsocketTime, nil, nil)
if errWrite := writeRequestInfoWithBody(&buf, w.url, w.method, w.requestHeaders, w.requestBody, "", w.timestamp, "http", upstreamTransport, true); errWrite != nil {
return errWrite
}
if errWrite := writeAPISection(&buf, "=== API WEBSOCKET TIMELINE ===\n", "=== API WEBSOCKET TIMELINE", w.apiWebsocketTime, time.Time{}); errWrite != nil {
return errWrite
}
if errWrite := writeAPISection(&buf, "=== API REQUEST ===\n", "=== API REQUEST", w.apiRequest, time.Time{}); errWrite != nil {
return errWrite
}
if errWrite := writeAPISection(&buf, "=== API RESPONSE ===\n", "=== API RESPONSE", w.apiResponse, w.apiResponseTS); errWrite != nil {
return errWrite
}
if errWrite := writeResponseSection(&buf, w.responseStatus, w.statusWritten, w.responseHeaders, bytes.NewReader(responsePayload), nil, false); errWrite != nil {
return errWrite
}
payload := homeRequestLogPayload{
Headers: cloneHeaders(w.requestHeaders),
RequestID: w.requestID,
RequestLog: buf.String(),
}
raw, errMarshal := json.Marshal(&payload)
if errMarshal != nil {
return errMarshal
}
return client.RPushRequestLog(context.Background(), raw)
}

410
backend/internal/logging/request_logger_home_test.go Normal file
View file

@ -0,0 +1,410 @@
package logging
import (
"bytes"
"context"
"encoding/json"
"net/http"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
type stubHomeRequestLogClient struct {
heartbeatOK bool
pushed [][]byte
}
func (c *stubHomeRequestLogClient) HeartbeatOK() bool { return c.heartbeatOK }
func (c *stubHomeRequestLogClient) RPushRequestLog(_ context.Context, payload []byte) error {
c.pushed = append(c.pushed, bytes.Clone(payload))
return nil
}
func assertFileBodySourceCleaned(t *testing.T, partPaths []string) {
t.Helper()
dirs := make(map[string]struct{}, len(partPaths))
for _, path := range partPaths {
if _, errStat := os.Stat(path); !os.IsNotExist(errStat) {
t.Fatalf("expected part %s to be removed, stat err=%v", path, errStat)
}
dirs[filepath.Dir(path)] = struct{}{}
}
for dir := range dirs {
if _, errStat := os.Stat(dir); !os.IsNotExist(errStat) {
t.Fatalf("expected part dir %s to be removed, stat err=%v", dir, errStat)
}
}
}
func TestFileBodySource_RecreatesPartDirAfterManualCleanup(t *testing.T) {
logsDir := t.TempDir()
source, errSource := NewFileBodySourceInDir(logsDir, "websocket-timeline-test")
if errSource != nil {
t.Fatalf("NewFileBodySourceInDir: %v", errSource)
}
if errAppend := source.AppendPart([]byte("before manual cleanup")); errAppend != nil {
t.Fatalf("AppendPart before cleanup: %v", errAppend)
}
if errRemove := os.RemoveAll(logsDir); errRemove != nil {
t.Fatalf("RemoveAll logs dir: %v", errRemove)
}
if errAppend := source.AppendPart([]byte("after manual cleanup")); errAppend != nil {
t.Fatalf("AppendPart after cleanup: %v", errAppend)
}
raw, errBytes := source.Bytes()
if errBytes != nil {
t.Fatalf("Bytes after cleanup: %v", errBytes)
}
if bytes.Contains(raw, []byte("before manual cleanup")) {
t.Fatalf("expected manually removed part to be skipped, got %q", string(raw))
}
if !bytes.Contains(raw, []byte("after manual cleanup")) {
t.Fatalf("expected recreated part content, got %q", string(raw))
}
partPaths := source.Paths()
if errCleanup := source.Cleanup(); errCleanup != nil {
t.Fatalf("Cleanup: %v", errCleanup)
}
assertFileBodySourceCleaned(t, partPaths)
}
func TestFileRequestLogger_HomeEnabled_ForwardsWhenRequestLogEnabled(t *testing.T) {
original := currentHomeRequestLogClient
defer func() {
currentHomeRequestLogClient = original
}()
stub := &stubHomeRequestLogClient{heartbeatOK: true}
currentHomeRequestLogClient = func() homeRequestLogClient {
return stub
}
logsDir := t.TempDir()
logger := NewFileRequestLogger(true, logsDir, "", 0)
logger.SetHomeEnabled(true)
requestHeaders := map[string][]string{
"Content-Type": {"application/json"},
"Authorization": {"Bearer secret"},
}
errLog := logger.LogRequest(
"/v1/chat/completions",
http.MethodPost,
requestHeaders,
[]byte(`{"input":"hello"}`),
http.StatusOK,
map[string][]string{"Content-Type": {"application/json"}},
[]byte(`{"ok":true}`),
nil,
nil,
nil,
nil,
nil,
"req-1",
time.Now(),
time.Now(),
)
if errLog != nil {
t.Fatalf("LogRequest error: %v", errLog)
}
entries, errRead := os.ReadDir(logsDir)
if errRead != nil {
t.Fatalf("failed to read logs dir: %v", errRead)
}
if len(entries) != 0 {
t.Fatalf("expected no local request log files, got entries: %+v", entries)
}
if len(stub.pushed) != 1 {
t.Fatalf("home pushed records = %d, want 1", len(stub.pushed))
}
var got struct {
Headers map[string][]string `json:"headers"`
RequestID string `json:"request_id"`
RequestLog string `json:"request_log"`
}
if errUnmarshal := json.Unmarshal(stub.pushed[0], &got); errUnmarshal != nil {
t.Fatalf("unmarshal payload: %v payload=%s", errUnmarshal, string(stub.pushed[0]))
}
if got.Headers == nil || got.Headers["Content-Type"][0] != "application/json" {
t.Fatalf("headers.content-type = %+v, want application/json", got.Headers["Content-Type"])
}
if got.Headers == nil || got.Headers["Authorization"][0] != "Bearer secret" {
t.Fatalf("headers.authorization = %+v, want Bearer secret", got.Headers["Authorization"])
}
if got.RequestID != "req-1" {
t.Fatalf("request_id = %q, want req-1", got.RequestID)
}
if got.RequestLog == "" {
t.Fatalf("request_log empty, want non-empty")
}
}
func TestFileRequestLogger_LogRequestWithSourcesWritesLocalLogAndCleansParts(t *testing.T) {
logsDir := t.TempDir()
logger := NewFileRequestLogger(true, logsDir, "", 0)
timelineSource, errSource := logger.NewFileBodySource("websocket-timeline-test")
if errSource != nil {
t.Fatalf("logger.NewFileBodySource: %v", errSource)
}
if errAppend := timelineSource.AppendPart([]byte("Timestamp: 2026-05-25T12:00:00Z\nEvent: websocket.request\n{}")); errAppend != nil {
t.Fatalf("AppendPart request: %v", errAppend)
}
if errAppend := timelineSource.AppendPart([]byte("Timestamp: 2026-05-25T12:00:01Z\nEvent: websocket.response\n{}")); errAppend != nil {
t.Fatalf("AppendPart response: %v", errAppend)
}
partPaths := timelineSource.Paths()
for _, path := range partPaths {
if !strings.HasPrefix(path, logsDir+string(os.PathSeparator)) {
t.Fatalf("part path %s is not under logs dir %s", path, logsDir)
}
}
errLog := logger.LogRequestWithOptionsAndSources(
"/v1/responses/ws",
http.MethodGet,
map[string][]string{"Upgrade": {"websocket"}},
nil,
http.StatusSwitchingProtocols,
map[string][]string{"Upgrade": {"websocket"}},
nil,
nil,
timelineSource,
nil,
nil,
nil,
nil,
nil,
false,
"ws-req-1",
time.Now(),
time.Now(),
)
if errLog != nil {
t.Fatalf("LogRequestWithOptionsAndSources error: %v", errLog)
}
assertFileBodySourceCleaned(t, partPaths)
entries, errRead := os.ReadDir(logsDir)
if errRead != nil {
t.Fatalf("failed to read logs dir: %v", errRead)
}
var logPath string
for _, entry := range entries {
if entry.IsDir() {
continue
}
logPath = logsDir + string(os.PathSeparator) + entry.Name()
break
}
if logPath == "" {
t.Fatal("expected local request log file")
}
raw, errReadLog := os.ReadFile(logPath)
if errReadLog != nil {
t.Fatalf("read log file: %v", errReadLog)
}
if !bytes.Contains(raw, []byte("=== WEBSOCKET TIMELINE ===")) {
t.Fatalf("websocket timeline section missing: %s", string(raw))
}
if !bytes.Contains(raw, []byte("Event: websocket.request")) || !bytes.Contains(raw, []byte("Event: websocket.response")) {
t.Fatalf("merged websocket events missing: %s", string(raw))
}
}
func TestFileRequestLogger_HomeEnabled_ForwardsSourceLogAndCleansParts(t *testing.T) {
original := currentHomeRequestLogClient
defer func() {
currentHomeRequestLogClient = original
}()
stub := &stubHomeRequestLogClient{heartbeatOK: true}
currentHomeRequestLogClient = func() homeRequestLogClient {
return stub
}
logsDir := t.TempDir()
logger := NewFileRequestLogger(true, logsDir, "", 0)
logger.SetHomeEnabled(true)
timelineSource, errSource := logger.NewFileBodySource("home-websocket-timeline-test")
if errSource != nil {
t.Fatalf("logger.NewFileBodySource: %v", errSource)
}
if errAppend := timelineSource.AppendPart([]byte("Timestamp: 2026-05-25T12:00:00Z\nEvent: websocket.request\n{}")); errAppend != nil {
t.Fatalf("AppendPart request: %v", errAppend)
}
partPaths := timelineSource.Paths()
for _, path := range partPaths {
if !strings.HasPrefix(path, logsDir+string(os.PathSeparator)) {
t.Fatalf("part path %s is not under logs dir %s", path, logsDir)
}
}
errLog := logger.LogRequestWithOptionsAndSources(
"/v1/responses/ws",
http.MethodGet,
map[string][]string{"Upgrade": {"websocket"}},
nil,
http.StatusSwitchingProtocols,
map[string][]string{"Upgrade": {"websocket"}},
nil,
nil,
timelineSource,
nil,
nil,
nil,
nil,
nil,
false,
"home-ws-req-1",
time.Now(),
time.Now(),
)
if errLog != nil {
t.Fatalf("LogRequestWithOptionsAndSources error: %v", errLog)
}
if len(stub.pushed) != 1 {
t.Fatalf("home pushed records = %d, want 1", len(stub.pushed))
}
var got struct {
RequestID string `json:"request_id"`
RequestLog string `json:"request_log"`
}
if errUnmarshal := json.Unmarshal(stub.pushed[0], &got); errUnmarshal != nil {
t.Fatalf("unmarshal payload: %v payload=%s", errUnmarshal, string(stub.pushed[0]))
}
if got.RequestID != "home-ws-req-1" {
t.Fatalf("request_id = %q, want home-ws-req-1", got.RequestID)
}
if !strings.Contains(got.RequestLog, "Event: websocket.request") {
t.Fatalf("forwarded request_log missing websocket request: %s", got.RequestLog)
}
assertFileBodySourceCleaned(t, partPaths)
}
func TestFileRequestLogger_HomeEnabled_ForwardsStreamingRequestID(t *testing.T) {
original := currentHomeRequestLogClient
defer func() {
currentHomeRequestLogClient = original
}()
stub := &stubHomeRequestLogClient{heartbeatOK: true}
currentHomeRequestLogClient = func() homeRequestLogClient {
return stub
}
logsDir := t.TempDir()
logger := NewFileRequestLogger(true, logsDir, "", 0)
logger.SetHomeEnabled(true)
writer, errLog := logger.LogStreamingRequest(
"/v1/responses",
http.MethodPost,
map[string][]string{"Content-Type": {"application/json"}},
[]byte(`{"input":"hello"}`),
"stream-req-1",
)
if errLog != nil {
t.Fatalf("LogStreamingRequest error: %v", errLog)
}
if errStatus := writer.WriteStatus(http.StatusOK, map[string][]string{"Content-Type": {"text/event-stream"}}); errStatus != nil {
t.Fatalf("WriteStatus error: %v", errStatus)
}
writer.WriteChunkAsync([]byte("data: ok\n\n"))
if errClose := writer.Close(); errClose != nil {
t.Fatalf("Close error: %v", errClose)
}
if len(stub.pushed) != 1 {
t.Fatalf("home pushed records = %d, want 1", len(stub.pushed))
}
var got struct {
RequestID string `json:"request_id"`
RequestLog string `json:"request_log"`
}
if errUnmarshal := json.Unmarshal(stub.pushed[0], &got); errUnmarshal != nil {
t.Fatalf("unmarshal payload: %v payload=%s", errUnmarshal, string(stub.pushed[0]))
}
if got.RequestID != "stream-req-1" {
t.Fatalf("request_id = %q, want stream-req-1", got.RequestID)
}
if got.RequestLog == "" {
t.Fatalf("request_log empty, want non-empty")
}
}
func TestFileRequestLogger_HomeEnabled_DoesNotForwardForcedErrorLogsWhenRequestLogDisabled(t *testing.T) {
original := currentHomeRequestLogClient
defer func() {
currentHomeRequestLogClient = original
}()
stub := &stubHomeRequestLogClient{heartbeatOK: true}
currentHomeRequestLogClient = func() homeRequestLogClient {
return stub
}
logsDir := t.TempDir()
logger := NewFileRequestLogger(false, logsDir, "", 0)
logger.SetHomeEnabled(true)
errLog := logger.LogRequestWithOptions(
"/v1/chat/completions",
http.MethodPost,
map[string][]string{"Content-Type": {"application/json"}},
[]byte(`{"input":"hello"}`),
http.StatusBadGateway,
map[string][]string{"Content-Type": {"application/json"}},
[]byte(`{"error":"upstream failure"}`),
nil,
nil,
nil,
nil,
nil,
true,
"req-2",
time.Now(),
time.Now(),
)
if errLog != nil {
t.Fatalf("LogRequestWithOptions error: %v", errLog)
}
if len(stub.pushed) != 0 {
t.Fatalf("home pushed records = %d, want 0", len(stub.pushed))
}
entries, errRead := os.ReadDir(logsDir)
if errRead != nil {
t.Fatalf("failed to read logs dir: %v", errRead)
}
found := false
for _, entry := range entries {
if entry.IsDir() {
continue
}
if entry.Name() != "" {
found = true
break
}
}
if !found {
t.Fatalf("expected local forced error log file when request-log disabled")
}
}

380
backend/internal/logging/request_logger_streaming.go Normal file
View file

@ -0,0 +1,380 @@
package logging
import (
"bytes"
"fmt"
"os"
"time"
log "github.com/sirupsen/logrus"
)
// FileStreamingLogWriter implements StreamingLogWriter for file-based streaming logs.
// It spools streaming response chunks to a temporary file to avoid retaining large responses in memory.
// The final log file is assembled when Close is called.
type FileStreamingLogWriter struct {
// logFilePath is the final log file path.
logFilePath string
// url is the request URL (masked upstream in middleware).
url string
// method is the HTTP method.
method string
// timestamp is captured when the streaming log is initialized.
timestamp time.Time
// requestHeaders stores the request headers.
requestHeaders map[string][]string
// requestBodyPath is a temporary file path holding the request body.
requestBodyPath string
// responseBodyPath is a temporary file path holding the streaming response body.
responseBodyPath string
// responseBodyFile is the temp file where chunks are appended by the async writer.
responseBodyFile *os.File
// chunkChan is a channel for receiving response chunks to spool.
chunkChan chan []byte
// closeChan is a channel for signaling when the writer is closed.
closeChan chan struct{}
// errorChan is a channel for reporting errors during writing.
errorChan chan error
// responseStatus stores the HTTP status code.
responseStatus int
// statusWritten indicates whether a non-zero status was recorded.
statusWritten bool
// responseHeaders stores the response headers.
responseHeaders map[string][]string
// apiRequest stores the upstream API request data.
apiRequest []byte
// apiRequestSource stores file-backed upstream API request data.
apiRequestSource *FileBodySource
// apiResponse stores the upstream API response data.
apiResponse []byte
// apiResponseSource stores file-backed upstream API response data.
apiResponseSource *FileBodySource
// apiWebsocketTimeline stores the upstream websocket event timeline.
apiWebsocketTimeline []byte
// apiResponseTimestamp captures when the API response was received.
apiResponseTimestamp time.Time
}
// WriteChunkAsync writes a response chunk asynchronously (non-blocking).
//
// Parameters:
// - chunk: The response chunk to write
func (w *FileStreamingLogWriter) WriteChunkAsync(chunk []byte) {
if w.chunkChan == nil {
return
}
// Make a copy of the chunk to avoid data races
chunkCopy := make([]byte, len(chunk))
copy(chunkCopy, chunk)
// Non-blocking send
select {
case w.chunkChan <- chunkCopy:
default:
// Channel is full, skip this chunk to avoid blocking
}
}
// WriteStatus buffers the response status and headers for later writing.
//
// Parameters:
// - status: The response status code
// - headers: The response headers
//
// Returns:
// - error: Always returns nil (buffering cannot fail)
func (w *FileStreamingLogWriter) WriteStatus(status int, headers map[string][]string) error {
if status == 0 {
return nil
}
w.responseStatus = status
if headers != nil {
w.responseHeaders = make(map[string][]string, len(headers))
for key, values := range headers {
headerValues := make([]string, len(values))
copy(headerValues, values)
w.responseHeaders[key] = headerValues
}
}
w.statusWritten = true
return nil
}
// WriteAPIRequest buffers the upstream API request details for later writing.
//
// Parameters:
// - apiRequest: The API request data (typically includes URL, headers, body sent upstream)
//
// Returns:
// - error: Always returns nil (buffering cannot fail)
func (w *FileStreamingLogWriter) WriteAPIRequest(apiRequest []byte) error {
if len(apiRequest) == 0 {
return nil
}
w.apiRequest = bytes.Clone(apiRequest)
return nil
}
// WriteAPIRequestSource buffers a file-backed upstream API request for final writing.
func (w *FileStreamingLogWriter) WriteAPIRequestSource(apiRequestSource *FileBodySource) error {
if apiRequestSource == nil || !apiRequestSource.HasPayload() {
return nil
}
w.apiRequestSource = apiRequestSource
return nil
}
// WriteAPIResponse buffers the upstream API response details for later writing.
//
// Parameters:
// - apiResponse: The API response data
//
// Returns:
// - error: Always returns nil (buffering cannot fail)
func (w *FileStreamingLogWriter) WriteAPIResponse(apiResponse []byte) error {
if len(apiResponse) == 0 {
return nil
}
w.apiResponse = bytes.Clone(apiResponse)
return nil
}
// WriteAPIResponseSource buffers a file-backed upstream API response for final writing.
func (w *FileStreamingLogWriter) WriteAPIResponseSource(apiResponseSource *FileBodySource) error {
if apiResponseSource == nil || !apiResponseSource.HasPayload() {
return nil
}
w.apiResponseSource = apiResponseSource
return nil
}
// WriteAPIWebsocketTimeline buffers the upstream websocket timeline for later writing.
//
// Parameters:
// - apiWebsocketTimeline: The upstream websocket event timeline
//
// Returns:
// - error: Always returns nil (buffering cannot fail)
func (w *FileStreamingLogWriter) WriteAPIWebsocketTimeline(apiWebsocketTimeline []byte) error {
if len(apiWebsocketTimeline) == 0 {
return nil
}
w.apiWebsocketTimeline = bytes.Clone(apiWebsocketTimeline)
return nil
}
func (w *FileStreamingLogWriter) SetFirstChunkTimestamp(timestamp time.Time) {
if !timestamp.IsZero() {
w.apiResponseTimestamp = timestamp
}
}
// Close finalizes the log file and cleans up resources.
// It writes all buffered data to the file in the correct order:
// API WEBSOCKET TIMELINE -> API REQUEST -> API RESPONSE -> RESPONSE (status, headers, body chunks)
//
// Returns:
// - error: An error if closing fails, nil otherwise
func (w *FileStreamingLogWriter) Close() error {
if w.chunkChan != nil {
close(w.chunkChan)
}
// Wait for async writer to finish spooling chunks
if w.closeChan != nil {
<-w.closeChan
w.chunkChan = nil
}
select {
case errWrite := <-w.errorChan:
w.cleanupTempFiles()
return errWrite
default:
}
if w.logFilePath == "" {
w.cleanupTempFiles()
return nil
}
logFile, errOpen := os.OpenFile(w.logFilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
if errOpen != nil {
w.cleanupTempFiles()
return fmt.Errorf("failed to create log file: %w", errOpen)
}
writeErr := w.writeFinalLog(logFile)
if errClose := logFile.Close(); errClose != nil {
log.WithError(errClose).Warn("failed to close request log file")
if writeErr == nil {
writeErr = errClose
}
}
w.cleanupTempFiles()
return writeErr
}
// asyncWriter runs in a goroutine to buffer chunks from the channel.
// It continuously reads chunks from the channel and appends them to a temp file for later assembly.
func (w *FileStreamingLogWriter) asyncWriter() {
defer close(w.closeChan)
for chunk := range w.chunkChan {
if w.responseBodyFile == nil {
continue
}
if _, errWrite := w.responseBodyFile.Write(chunk); errWrite != nil {
select {
case w.errorChan <- errWrite:
default:
}
if errClose := w.responseBodyFile.Close(); errClose != nil {
select {
case w.errorChan <- errClose:
default:
}
}
w.responseBodyFile = nil
}
}
if w.responseBodyFile == nil {
return
}
if errClose := w.responseBodyFile.Close(); errClose != nil {
select {
case w.errorChan <- errClose:
default:
}
}
w.responseBodyFile = nil
}
func (w *FileStreamingLogWriter) writeFinalLog(logFile *os.File) error {
if errWrite := writeRequestInfoWithBody(logFile, w.url, w.method, w.requestHeaders, nil, w.requestBodyPath, w.timestamp, "http", inferUpstreamTransport(w.apiRequest, w.apiRequestSource, w.apiResponse, w.apiResponseSource, w.apiWebsocketTimeline, nil, nil), true); errWrite != nil {
return errWrite
}
if errWrite := writeAPISection(logFile, "=== API WEBSOCKET TIMELINE ===\n", "=== API WEBSOCKET TIMELINE", w.apiWebsocketTimeline, time.Time{}); errWrite != nil {
return errWrite
}
if errWrite := writePreformattedAPISectionWithSource(logFile, "=== API REQUEST ===\n", "=== API REQUEST", w.apiRequest, w.apiRequestSource, time.Time{}); errWrite != nil {
return errWrite
}
if errWrite := writePreformattedAPISectionWithSource(logFile, "=== API RESPONSE ===\n", "=== API RESPONSE", w.apiResponse, w.apiResponseSource, w.apiResponseTimestamp); errWrite != nil {
return errWrite
}
responseBodyFile, errOpen := os.Open(w.responseBodyPath)
if errOpen != nil {
return errOpen
}
defer func() {
if errClose := responseBodyFile.Close(); errClose != nil {
log.WithError(errClose).Warn("failed to close response body temp file")
}
}()
return writeResponseSection(logFile, w.responseStatus, w.statusWritten, w.responseHeaders, responseBodyFile, nil, false)
}
func (w *FileStreamingLogWriter) cleanupTempFiles() {
if w.requestBodyPath != "" {
if errRemove := os.Remove(w.requestBodyPath); errRemove != nil {
log.WithError(errRemove).Warn("failed to remove request body temp file")
}
w.requestBodyPath = ""
}
if w.responseBodyPath != "" {
if errRemove := os.Remove(w.responseBodyPath); errRemove != nil {
log.WithError(errRemove).Warn("failed to remove response body temp file")
}
w.responseBodyPath = ""
}
}
// NoOpStreamingLogWriter is a no-operation implementation for when logging is disabled.
// It implements the StreamingLogWriter interface but performs no actual logging operations.
type NoOpStreamingLogWriter struct{}
// WriteChunkAsync is a no-op implementation that does nothing.
//
// Parameters:
// - chunk: The response chunk (ignored)
func (w *NoOpStreamingLogWriter) WriteChunkAsync(_ []byte) {}
// WriteStatus is a no-op implementation that does nothing and always returns nil.
//
// Parameters:
// - status: The response status code (ignored)
// - headers: The response headers (ignored)
//
// Returns:
// - error: Always returns nil
func (w *NoOpStreamingLogWriter) WriteStatus(_ int, _ map[string][]string) error {
return nil
}
// WriteAPIRequest is a no-op implementation that does nothing and always returns nil.
//
// Parameters:
// - apiRequest: The API request data (ignored)
//
// Returns:
// - error: Always returns nil
func (w *NoOpStreamingLogWriter) WriteAPIRequest(_ []byte) error {
return nil
}
// WriteAPIResponse is a no-op implementation that does nothing and always returns nil.
//
// Parameters:
// - apiResponse: The API response data (ignored)
//
// Returns:
// - error: Always returns nil
func (w *NoOpStreamingLogWriter) WriteAPIResponse(_ []byte) error {
return nil
}
// WriteAPIWebsocketTimeline is a no-op implementation that does nothing and always returns nil.
//
// Parameters:
// - apiWebsocketTimeline: The upstream websocket event timeline (ignored)
//
// Returns:
// - error: Always returns nil
func (w *NoOpStreamingLogWriter) WriteAPIWebsocketTimeline(_ []byte) error {
return nil
}
func (w *NoOpStreamingLogWriter) SetFirstChunkTimestamp(_ time.Time) {}
// Close is a no-op implementation that does nothing and always returns nil.
//
// Returns:
// - error: Always returns nil
func (w *NoOpStreamingLogWriter) Close() error { return nil }

413
backend/internal/logging/request_logger_writer.go Normal file
View file

@ -0,0 +1,413 @@
package logging
import (
"bytes"
"context"
"fmt"
"io"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"sync/atomic"
"time"
"github.com/router-for-me/CLIProxyAPI/v7/internal/interfaces"
log "github.com/sirupsen/logrus"
)
var requestLogID atomic.Uint64
// LogRequest logs a complete non-streaming request/response cycle to a file.
//
// Parameters:
// - url: The request URL
// - method: The HTTP method
// - requestHeaders: The request headers
// - body: The request body
// - statusCode: The response status code
// - responseHeaders: The response headers
// - response: The raw response data
// - apiRequest: The API request data
// - apiResponse: The API response data
// - requestID: Optional request ID for log file naming
// - requestTimestamp: When the request was received
// - apiResponseTimestamp: When the API response was received
//
// Returns:
// - error: An error if logging fails, nil otherwise
func (l *FileRequestLogger) LogRequest(url, method string, requestHeaders map[string][]string, body []byte, statusCode int, responseHeaders map[string][]string, response, websocketTimeline, apiRequest, apiResponse, apiWebsocketTimeline []byte, apiResponseErrors []*interfaces.ErrorMessage, requestID string, requestTimestamp, apiResponseTimestamp time.Time) error {
return l.logRequest(url, method, requestHeaders, body, statusCode, responseHeaders, response, websocketTimeline, apiRequest, apiResponse, apiWebsocketTimeline, apiResponseErrors, false, requestID, requestTimestamp, apiResponseTimestamp)
}
// LogRequestWithOptions logs a request with optional forced logging behavior.
// The force flag allows writing error logs even when regular request logging is disabled.
func (l *FileRequestLogger) LogRequestWithOptions(url, method string, requestHeaders map[string][]string, body []byte, statusCode int, responseHeaders map[string][]string, response, websocketTimeline, apiRequest, apiResponse, apiWebsocketTimeline []byte, apiResponseErrors []*interfaces.ErrorMessage, force bool, requestID string, requestTimestamp, apiResponseTimestamp time.Time) error {
return l.logRequestWithSources(url, method, requestHeaders, body, statusCode, responseHeaders, response, websocketTimeline, nil, apiRequest, nil, apiResponse, nil, apiWebsocketTimeline, nil, apiResponseErrors, force, requestID, requestTimestamp, apiResponseTimestamp)
}
func (l *FileRequestLogger) logRequest(url, method string, requestHeaders map[string][]string, body []byte, statusCode int, responseHeaders map[string][]string, response, websocketTimeline, apiRequest, apiResponse, apiWebsocketTimeline []byte, apiResponseErrors []*interfaces.ErrorMessage, force bool, requestID string, requestTimestamp, apiResponseTimestamp time.Time) error {
return l.logRequestWithSources(url, method, requestHeaders, body, statusCode, responseHeaders, response, websocketTimeline, nil, apiRequest, nil, apiResponse, nil, apiWebsocketTimeline, nil, apiResponseErrors, force, requestID, requestTimestamp, apiResponseTimestamp)
}
// LogRequestWithOptionsAndSources logs a request with optional file-backed large sections.
func (l *FileRequestLogger) LogRequestWithOptionsAndSources(url, method string, requestHeaders map[string][]string, body []byte, statusCode int, responseHeaders map[string][]string, response, websocketTimeline []byte, websocketTimelineSource *FileBodySource, apiRequest, apiResponse, apiWebsocketTimeline []byte, apiWebsocketTimelineSource *FileBodySource, apiResponseErrors []*interfaces.ErrorMessage, force bool, requestID string, requestTimestamp, apiResponseTimestamp time.Time) error {
return l.logRequestWithSources(url, method, requestHeaders, body, statusCode, responseHeaders, response, websocketTimeline, websocketTimelineSource, apiRequest, nil, apiResponse, nil, apiWebsocketTimeline, apiWebsocketTimelineSource, apiResponseErrors, force, requestID, requestTimestamp, apiResponseTimestamp)
}
// LogRequestWithOptionsAndAllSources logs a request with optional file-backed request and response sections.
func (l *FileRequestLogger) LogRequestWithOptionsAndAllSources(url, method string, requestHeaders map[string][]string, body []byte, statusCode int, responseHeaders map[string][]string, response, websocketTimeline []byte, websocketTimelineSource *FileBodySource, apiRequest []byte, apiRequestSource *FileBodySource, apiResponse []byte, apiResponseSource *FileBodySource, apiWebsocketTimeline []byte, apiWebsocketTimelineSource *FileBodySource, apiResponseErrors []*interfaces.ErrorMessage, force bool, requestID string, requestTimestamp, apiResponseTimestamp time.Time) error {
return l.logRequestWithSources(url, method, requestHeaders, body, statusCode, responseHeaders, response, websocketTimeline, websocketTimelineSource, apiRequest, apiRequestSource, apiResponse, apiResponseSource, apiWebsocketTimeline, apiWebsocketTimelineSource, apiResponseErrors, force, requestID, requestTimestamp, apiResponseTimestamp)
}
func (l *FileRequestLogger) logRequestWithSources(url, method string, requestHeaders map[string][]string, body []byte, statusCode int, responseHeaders map[string][]string, response, websocketTimeline []byte, websocketTimelineSource *FileBodySource, apiRequest []byte, apiRequestSource *FileBodySource, apiResponse []byte, apiResponseSource *FileBodySource, apiWebsocketTimeline []byte, apiWebsocketTimelineSource *FileBodySource, apiResponseErrors []*interfaces.ErrorMessage, force bool, requestID string, requestTimestamp, apiResponseTimestamp time.Time) error {
defer cleanupFileBodySources(websocketTimelineSource, apiRequestSource, apiResponseSource, apiWebsocketTimelineSource)
if !l.enabled && !force {
return nil
}
if l.homeEnabled && l.enabled {
responseToWrite, decompressErr := l.decompressResponse(responseHeaders, response)
if decompressErr != nil {
responseToWrite = response
}
var buf bytes.Buffer
writeErr := l.writeNonStreamingLog(
&buf,
url,
method,
requestHeaders,
body,
"",
websocketTimeline,
websocketTimelineSource,
apiRequest,
apiRequestSource,
apiResponse,
apiResponseSource,
apiWebsocketTimeline,
apiWebsocketTimelineSource,
apiResponseErrors,
statusCode,
responseHeaders,
responseToWrite,
decompressErr,
requestTimestamp,
apiResponseTimestamp,
)
if writeErr != nil {
return fmt.Errorf("failed to build request log content: %w", writeErr)
}
return l.forwardRequestLogToHome(context.Background(), requestHeaders, requestID, buf.String())
}
// Ensure logs directory exists
if errEnsure := l.ensureLogsDir(); errEnsure != nil {
return fmt.Errorf("failed to create logs directory: %w", errEnsure)
}
// Generate filename with request ID
filename := l.generateFilename(url, requestID)
if force && !l.enabled {
filename = l.generateErrorFilename(url, requestID)
}
filePath := filepath.Join(l.logsDir, filename)
requestBodyPath, errTemp := l.writeRequestBodyTempFile(body)
if errTemp != nil {
log.WithError(errTemp).Warn("failed to create request body temp file, falling back to direct write")
}
if requestBodyPath != "" {
defer func() {
if errRemove := os.Remove(requestBodyPath); errRemove != nil {
log.WithError(errRemove).Warn("failed to remove request body temp file")
}
}()
}
responseToWrite, decompressErr := l.decompressResponse(responseHeaders, response)
if decompressErr != nil {
// If decompression fails, continue with original response and annotate the log output.
responseToWrite = response
}
logFile, errOpen := os.OpenFile(filePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
if errOpen != nil {
return fmt.Errorf("failed to create log file: %w", errOpen)
}
writeErr := l.writeNonStreamingLog(
logFile,
url,
method,
requestHeaders,
body,
requestBodyPath,
websocketTimeline,
websocketTimelineSource,
apiRequest,
apiRequestSource,
apiResponse,
apiResponseSource,
apiWebsocketTimeline,
apiWebsocketTimelineSource,
apiResponseErrors,
statusCode,
responseHeaders,
responseToWrite,
decompressErr,
requestTimestamp,
apiResponseTimestamp,
)
if errClose := logFile.Close(); errClose != nil {
log.WithError(errClose).Warn("failed to close request log file")
if writeErr == nil {
return errClose
}
}
if writeErr != nil {
return fmt.Errorf("failed to write log file: %w", writeErr)
}
if force && !l.enabled {
if errCleanup := l.cleanupOldErrorLogs(); errCleanup != nil {
log.WithError(errCleanup).Warn("failed to clean up old error logs")
}
}
return nil
}
// LogStreamingRequest initiates logging for a streaming request.
//
// Parameters:
// - url: The request URL
// - method: The HTTP method
// - headers: The request headers
// - body: The request body
// - requestID: Optional request ID for log file naming
//
// Returns:
// - StreamingLogWriter: A writer for streaming response chunks
// - error: An error if logging initialization fails, nil otherwise
func (l *FileRequestLogger) LogStreamingRequest(url, method string, headers map[string][]string, body []byte, requestID string) (StreamingLogWriter, error) {
if !l.enabled {
return &NoOpStreamingLogWriter{}, nil
}
if l.homeEnabled {
client := currentHomeRequestLogClient()
if client == nil || !client.HeartbeatOK() {
return &NoOpStreamingLogWriter{}, nil
}
return newHomeStreamingLogWriter(url, method, headers, body, requestID), nil
}
// Ensure logs directory exists
if err := l.ensureLogsDir(); err != nil {
return nil, fmt.Errorf("failed to create logs directory: %w", err)
}
// Generate filename with request ID
filename := l.generateFilename(url, requestID)
filePath := filepath.Join(l.logsDir, filename)
requestHeaders := make(map[string][]string, len(headers))
for key, values := range headers {
headerValues := make([]string, len(values))
copy(headerValues, values)
requestHeaders[key] = headerValues
}
requestBodyPath, errTemp := l.writeRequestBodyTempFile(body)
if errTemp != nil {
return nil, fmt.Errorf("failed to create request body temp file: %w", errTemp)
}
responseBodyFile, errCreate := os.CreateTemp(l.logsDir, "response-body-*.tmp")
if errCreate != nil {
_ = os.Remove(requestBodyPath)
return nil, fmt.Errorf("failed to create response body temp file: %w", errCreate)
}
responseBodyPath := responseBodyFile.Name()
// Create streaming writer
writer := &FileStreamingLogWriter{
logFilePath: filePath,
url: url,
method: method,
timestamp: time.Now(),
requestHeaders: requestHeaders,
requestBodyPath: requestBodyPath,
responseBodyPath: responseBodyPath,
responseBodyFile: responseBodyFile,
chunkChan: make(chan []byte, 100), // Buffered channel for async writes
closeChan: make(chan struct{}),
errorChan: make(chan error, 1),
}
// Start async writer goroutine
go writer.asyncWriter()
return writer, nil
}
// generateErrorFilename creates a filename with an error prefix to differentiate forced error logs.
func (l *FileRequestLogger) generateErrorFilename(url string, requestID ...string) string {
return fmt.Sprintf("error-%s", l.generateFilename(url, requestID...))
}
// ensureLogsDir creates the logs directory if it doesn't exist.
//
// Returns:
// - error: An error if directory creation fails, nil otherwise
func (l *FileRequestLogger) ensureLogsDir() error {
if _, err := os.Stat(l.logsDir); os.IsNotExist(err) {
return os.MkdirAll(l.logsDir, 0755)
}
return nil
}
// generateFilename creates a sanitized filename from the URL path and current timestamp.
// Format: v1-responses-2025-12-23T195811-a1b2c3d4.log
//
// Parameters:
// - url: The request URL
// - requestID: Optional request ID to include in filename
//
// Returns:
// - string: A sanitized filename for the log file
func (l *FileRequestLogger) generateFilename(url string, requestID ...string) string {
// Extract path from URL
path := url
if strings.Contains(url, "?") {
path = strings.Split(url, "?")[0]
}
// Remove leading slash
if strings.HasPrefix(path, "/") {
path = path[1:]
}
// Sanitize path for filename
sanitized := l.sanitizeForFilename(path)
// Add timestamp
timestamp := time.Now().Format("2006-01-02T150405")
// Use request ID if provided, otherwise use sequential ID
var idPart string
if len(requestID) > 0 && requestID[0] != "" {
idPart = requestID[0]
} else {
id := requestLogID.Add(1)
idPart = fmt.Sprintf("%d", id)
}
return fmt.Sprintf("%s-%s-%s.log", sanitized, timestamp, idPart)
}
// sanitizeForFilename replaces characters that are not safe for filenames.
//
// Parameters:
// - path: The path to sanitize
//
// Returns:
// - string: A sanitized filename
func (l *FileRequestLogger) sanitizeForFilename(path string) string {
// Replace slashes with hyphens
sanitized := strings.ReplaceAll(path, "/", "-")
// Replace colons with hyphens
sanitized = strings.ReplaceAll(sanitized, ":", "-")
// Replace other problematic characters with hyphens
reg := regexp.MustCompile(`[<>:"|?*\s]`)
sanitized = reg.ReplaceAllString(sanitized, "-")
// Remove multiple consecutive hyphens
reg = regexp.MustCompile(`-+`)
sanitized = reg.ReplaceAllString(sanitized, "-")
// Remove leading/trailing hyphens
sanitized = strings.Trim(sanitized, "-")
// Handle empty result
if sanitized == "" {
sanitized = "root"
}
return sanitized
}
// cleanupOldErrorLogs keeps only the newest errorLogsMaxFiles forced error log files.
func (l *FileRequestLogger) cleanupOldErrorLogs() error {
if l.errorLogsMaxFiles <= 0 {
return nil
}
entries, errRead := os.ReadDir(l.logsDir)
if errRead != nil {
return errRead
}
type logFile struct {
name string
modTime time.Time
}
var files []logFile
for _, entry := range entries {
if entry.IsDir() {
continue
}
name := entry.Name()
if !strings.HasPrefix(name, "error-") || !strings.HasSuffix(name, ".log") {
continue
}
info, errInfo := entry.Info()
if errInfo != nil {
log.WithError(errInfo).Warn("failed to read error log info")
continue
}
files = append(files, logFile{name: name, modTime: info.ModTime()})
}
if len(files) <= l.errorLogsMaxFiles {
return nil
}
sort.Slice(files, func(i, j int) bool {
return files[i].modTime.After(files[j].modTime)
})
for _, file := range files[l.errorLogsMaxFiles:] {
if errRemove := os.Remove(filepath.Join(l.logsDir, file.name)); errRemove != nil {
log.WithError(errRemove).Warnf("failed to remove old error log: %s", file.name)
}
}
return nil
}
func (l *FileRequestLogger) writeRequestBodyTempFile(body []byte) (string, error) {
tmpFile, errCreate := os.CreateTemp(l.logsDir, "request-body-*.tmp")
if errCreate != nil {
return "", errCreate
}
tmpPath := tmpFile.Name()
if _, errCopy := io.Copy(tmpFile, bytes.NewReader(body)); errCopy != nil {
_ = tmpFile.Close()
_ = os.Remove(tmpPath)
return "", errCopy
}
if errClose := tmpFile.Close(); errClose != nil {
_ = os.Remove(tmpPath)
return "", errClose
}
return tmpPath, nil
}

61
backend/internal/logging/requestid.go Normal file
View file

@ -0,0 +1,61 @@
package logging
import (
"context"
"crypto/rand"
"encoding/hex"
"github.com/gin-gonic/gin"
)
// requestIDKey is the context key for storing/retrieving request IDs.
type requestIDKey struct{}
// ginRequestIDKey is the Gin context key for request IDs.
const ginRequestIDKey = "__request_id__"
// GenerateRequestID creates a new 8-character hex request ID.
func GenerateRequestID() string {
b := make([]byte, 4)
if _, err := rand.Read(b); err != nil {
return "00000000"
}
return hex.EncodeToString(b)
}
// WithRequestID returns a new context with the request ID attached.
func WithRequestID(ctx context.Context, requestID string) context.Context {
return context.WithValue(ctx, requestIDKey{}, requestID)
}
// GetRequestID retrieves the request ID from the context.
// Returns empty string if not found.
func GetRequestID(ctx context.Context) string {
if ctx == nil {
return ""
}
if id, ok := ctx.Value(requestIDKey{}).(string); ok {
return id
}
return ""
}
// SetGinRequestID stores the request ID in the Gin context.
func SetGinRequestID(c *gin.Context, requestID string) {
if c != nil {
c.Set(ginRequestIDKey, requestID)
}
}
// GetGinRequestID retrieves the request ID from the Gin context.
func GetGinRequestID(c *gin.Context) string {
if c == nil {
return ""
}
if id, exists := c.Get(ginRequestIDKey); exists {
if s, ok := id.(string); ok {
return s
}
}
return ""
}

144
backend/internal/logging/requestmeta.go Normal file
View file

@ -0,0 +1,144 @@
package logging
import (
"context"
"net/http"
"sync"
"sync/atomic"
)
type endpointKey struct{}
type responseStatusKey struct{}
type responseHeadersKey struct{}
type clientRequestMetadataKey struct{}
// ClientRequestMetadata stores immutable downstream request metadata for asynchronous consumers.
type ClientRequestMetadata struct {
ClientIP string
XForwardedFor string
UserAgent string
}
type responseStatusHolder struct {
status atomic.Int32
}
type responseHeadersHolder struct {
mu sync.RWMutex
headers http.Header
}
func WithEndpoint(ctx context.Context, endpoint string) context.Context {
if ctx == nil {
ctx = context.Background()
}
return context.WithValue(ctx, endpointKey{}, endpoint)
}
func GetEndpoint(ctx context.Context) string {
if ctx == nil {
return ""
}
if endpoint, ok := ctx.Value(endpointKey{}).(string); ok {
return endpoint
}
return ""
}
// WithClientRequestMetadata stores a snapshot of downstream request metadata in ctx.
func WithClientRequestMetadata(ctx context.Context, metadata ClientRequestMetadata) context.Context {
if ctx == nil {
ctx = context.Background()
}
return context.WithValue(ctx, clientRequestMetadataKey{}, metadata)
}
// GetClientRequestMetadata returns downstream request metadata stored in ctx.
func GetClientRequestMetadata(ctx context.Context) ClientRequestMetadata {
if ctx == nil {
return ClientRequestMetadata{}
}
if metadata, ok := ctx.Value(clientRequestMetadataKey{}).(ClientRequestMetadata); ok {
return metadata
}
return ClientRequestMetadata{}
}
func WithResponseStatusHolder(ctx context.Context) context.Context {
if ctx == nil {
ctx = context.Background()
}
if holder, ok := ctx.Value(responseStatusKey{}).(*responseStatusHolder); ok && holder != nil {
return ctx
}
return context.WithValue(ctx, responseStatusKey{}, &responseStatusHolder{})
}
func WithResponseHeadersHolder(ctx context.Context) context.Context {
if ctx == nil {
ctx = context.Background()
}
if holder, ok := ctx.Value(responseHeadersKey{}).(*responseHeadersHolder); ok && holder != nil {
return ctx
}
return context.WithValue(ctx, responseHeadersKey{}, &responseHeadersHolder{})
}
func SetResponseStatus(ctx context.Context, status int) {
if ctx == nil || status <= 0 {
return
}
holder, ok := ctx.Value(responseStatusKey{}).(*responseStatusHolder)
if !ok || holder == nil {
return
}
holder.status.Store(int32(status))
}
func SetResponseHeaders(ctx context.Context, headers http.Header) {
if ctx == nil {
return
}
holder, ok := ctx.Value(responseHeadersKey{}).(*responseHeadersHolder)
if !ok || holder == nil {
return
}
holder.mu.Lock()
defer holder.mu.Unlock()
holder.headers = cloneHTTPHeader(headers)
}
func GetResponseStatus(ctx context.Context) int {
if ctx == nil {
return 0
}
holder, ok := ctx.Value(responseStatusKey{}).(*responseStatusHolder)
if !ok || holder == nil {
return 0
}
return int(holder.status.Load())
}
func GetResponseHeaders(ctx context.Context) http.Header {
if ctx == nil {
return nil
}
holder, ok := ctx.Value(responseHeadersKey{}).(*responseHeadersHolder)
if !ok || holder == nil {
return nil
}
holder.mu.RLock()
defer holder.mu.RUnlock()
return cloneHTTPHeader(holder.headers)
}
func cloneHTTPHeader(src http.Header) http.Header {
if len(src) == 0 {
return nil
}
dst := make(http.Header, len(src))
for key, values := range src {
dst[key] = append([]string(nil), values...)
}
return dst
}