Add projects

backend/examples/realtime-openai-go/README.md

@@ -0,0 +1,89 @@
+# OpenAI Go SDK Realtime Voice Example
+
+This example sends spoken audio to CLIProxyAPI and saves the model's spoken reply as a WAV file.
+
+It uses the official [`github.com/openai/openai-go/v3`](https://github.com/openai/openai-go) SDK to create a short-lived Realtime client secret. The official Go SDK currently exposes the Realtime REST resources but does not provide a WebSocket connection helper, so `github.com/gorilla/websocket` is used for the standard Realtime audio events.
+
+## Prerequisites
+
+1. Start CLIProxyAPI with at least one working ChatGPT/Codex OAuth credential.
+2. Configure a proxy API key in `config.yaml`.
+3. Use Go 1.26 or newer.
+4. Prepare a PCM WAV file with these exact properties:
+   - 24,000 Hz sample rate
+   - 16-bit signed PCM
+   - mono
+   - little-endian
+
+Convert an existing recording with FFmpeg:
+
+```bash
+ffmpeg -i recording.m4a -ar 24000 -ac 1 -c:a pcm_s16le question.wav
+```
+
+## Run
+
+```bash
+cd examples/realtime-openai-go
+
+OPENAI_BASE_URL="http://127.0.0.1:8317/v1" \
+OPENAI_API_KEY="your-proxy-api-key" \
+OPENAI_REALTIME_MODEL="gpt-realtime-2.1" \
+OPENAI_REALTIME_INPUT_WAV="question.wav" \
+OPENAI_REALTIME_OUTPUT_WAV="response.wav" \
+go run .
+```
+
+Expected output:
+
+```text
+Loaded question.wav (2.4s, 115200 PCM bytes)
+Connected to ws://127.0.0.1:8317/v1/realtime?model=gpt-realtime-2.1 using model gpt-realtime-2.1 and voice marin
+Sent 2.4s of speech audio
+Assistant transcript: The connection is working correctly.
+Saved spoken response to response.wav (1.8s, 86400 PCM bytes)
+```
+
+Play the response:
+
+```bash
+# macOS
+afplay response.wav
+
+# Linux
+aplay response.wav
+
+# Cross-platform with FFmpeg
+ffplay -autoexit response.wav
+```
+
+## Environment variables
+
+| Variable | Required | Default | Description |
+| --- | --- | --- | --- |
+| `OPENAI_API_KEY` | Yes | — | API key configured for CLIProxyAPI. |
+| `OPENAI_REALTIME_INPUT_WAV` | Yes | — | Input speech WAV file. It must be 24kHz, 16-bit, mono PCM. |
+| `OPENAI_REALTIME_OUTPUT_WAV` | No | `response.wav` | Destination for the spoken response. |
+| `OPENAI_BASE_URL` | No | `http://127.0.0.1:8317/v1` | CLIProxyAPI OpenAI-compatible base URL. `/v1` is added when the URL has no path. |
+| `OPENAI_REALTIME_MODEL` | No | `gpt-realtime-2.1` | Standard Realtime model name. CLIProxyAPI uses it for the upstream standard WebSocket while selecting a compatible Codex OAuth credential internally. |
+| `OPENAI_REALTIME_VOICE` | No | `marin` | Realtime output voice. Other common values include `cedar`, `alloy`, `ash`, `coral`, and `echo`. |
+| `OPENAI_REALTIME_INSTRUCTIONS` | No | Short spoken response instruction | Session instructions attached to the client secret. |
+| `OPENAI_REALTIME_DEBUG` | No | `false` | Print every received Realtime server event. |
+
+## Audio flow
+
+1. The official OpenAI Go SDK calls `POST /v1/realtime/client_secrets` with an audio session configured for 24kHz PCM input and output.
+2. The returned local `ek_...` credential authenticates the `/v1/realtime` WebSocket.
+3. Input WAV samples are sent in 200ms `input_audio_buffer.append` chunks.
+4. The client sends `input_audio_buffer.commit` and `response.create`.
+5. Base64 `response.output_audio.delta` events are decoded and written to the output WAV.
+
+The client secret returned by CLIProxyAPI is local to that proxy instance and is not valid against `api.openai.com`.
+
+## Test
+
+```bash
+go test -race ./...
+```
+
+The test starts an in-process HTTP/WebSocket server and verifies client-secret configuration, input audio streaming, output audio decoding, and WAV generation.

backend/examples/realtime-openai-go/go.mod

@@ -0,0 +1,15 @@
+module github.com/router-for-me/CLIProxyAPI/v7/examples/realtime-openai-go
+
+go 1.26.0
+
+require (
+	github.com/gorilla/websocket v1.5.3
+	github.com/openai/openai-go/v3 v3.50.0
+)
+
+require (
+	github.com/tidwall/gjson v1.19.0 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.1 // indirect
+	github.com/tidwall/sjson v1.2.5 // indirect
+)

backend/examples/realtime-openai-go/go.sum

@@ -0,0 +1,14 @@
+github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
+github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/openai/openai-go/v3 v3.50.0 h1:CXn+C8a10oQiI5CMyMbCiykhITVhVxhdHX8j3CfLa2U=
+github.com/openai/openai-go/v3 v3.50.0/go.mod h1:Ogjo0gDct+Jm7yCqaCjLGQGygeV8xNfNHV1/yKvCji0=
+github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.19.0 h1:xwxm7n691Uf3u5OFjzngavjGTh55KX5q/9w9xHW88JU=
+github.com/tidwall/gjson v1.19.0/go.mod h1:V37/opeE/JbLUOfH0QTXiNez2l0RUjYUhpT4szFQAfc=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
+github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
+github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=

backend/examples/realtime-openai-go/main.go

@@ -0,0 +1,341 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"os/signal"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/gorilla/websocket"
+	"github.com/openai/openai-go/v3"
+	"github.com/openai/openai-go/v3/option"
+	"github.com/openai/openai-go/v3/realtime"
+)
+
+const (
+	defaultBaseURL      = "http://127.0.0.1:8317/v1"
+	defaultModel        = "gpt-realtime-2.1"
+	defaultInstructions = "Listen to the user's speech and reply with a short spoken response."
+	defaultOutputWAV    = "response.wav"
+	defaultVoice        = "marin"
+	audioSampleRate     = 24000
+	audioBytesPerSample = 2
+	audioChunkDuration  = 200 * time.Millisecond
+)
+
+type appConfig struct {
+	baseURL      string
+	apiKey       string
+	model        string
+	inputWAV     string
+	outputWAV    string
+	instructions string
+	voice        string
+	debug        bool
+}
+
+type realtimeServerEvent struct {
+	Type  string `json:"type"`
+	Delta string `json:"delta"`
+	Error *struct {
+		Message string `json:"message"`
+		Type    string `json:"type"`
+		Code    string `json:"code"`
+	} `json:"error,omitempty"`
+	Response *struct {
+		Status string `json:"status"`
+	} `json:"response,omitempty"`
+}
+
+func main() {
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+
+	cfg, errConfig := loadConfig()
+	if errConfig != nil {
+		fmt.Fprintf(os.Stderr, "configuration error: %v\n", errConfig)
+		os.Exit(1)
+	}
+	if errRun := run(ctx, cfg, os.Stdout); errRun != nil {
+		fmt.Fprintf(os.Stderr, "realtime example failed: %v\n", errRun)
+		os.Exit(1)
+	}
+}
+
+func loadConfig() (appConfig, error) {
+	baseURL, errBaseURL := normalizeBaseURL(envOrDefault("OPENAI_BASE_URL", defaultBaseURL))
+	if errBaseURL != nil {
+		return appConfig{}, errBaseURL
+	}
+	apiKey := strings.TrimSpace(os.Getenv("OPENAI_API_KEY"))
+	if apiKey == "" {
+		return appConfig{}, errors.New("OPENAI_API_KEY is required")
+	}
+	inputWAV := strings.TrimSpace(os.Getenv("OPENAI_REALTIME_INPUT_WAV"))
+	if inputWAV == "" {
+		return appConfig{}, errors.New("OPENAI_REALTIME_INPUT_WAV is required")
+	}
+	return appConfig{
+		baseURL:      baseURL,
+		apiKey:       apiKey,
+		model:        envOrDefault("OPENAI_REALTIME_MODEL", defaultModel),
+		inputWAV:     inputWAV,
+		outputWAV:    envOrDefault("OPENAI_REALTIME_OUTPUT_WAV", defaultOutputWAV),
+		instructions: envOrDefault("OPENAI_REALTIME_INSTRUCTIONS", defaultInstructions),
+		voice:        envOrDefault("OPENAI_REALTIME_VOICE", defaultVoice),
+		debug:        strings.EqualFold(strings.TrimSpace(os.Getenv("OPENAI_REALTIME_DEBUG")), "true"),
+	}, nil
+}
+
+func run(ctx context.Context, cfg appConfig, output io.Writer) error {
+	inputPCM, errInput := readPCM16WAV(cfg.inputWAV)
+	if errInput != nil {
+		return fmt.Errorf("read input WAV: %w", errInput)
+	}
+	inputDuration := time.Duration(len(inputPCM)) * time.Second / (audioSampleRate * audioBytesPerSample)
+	fmt.Fprintf(output, "Loaded %s (%s, %d PCM bytes)\n", cfg.inputWAV, inputDuration.Round(time.Millisecond), len(inputPCM))
+
+	client := openai.NewClient(
+		option.WithAPIKey(cfg.apiKey),
+		option.WithBaseURL(cfg.baseURL),
+	)
+	pcmFormat := realtime.RealtimeAudioFormatsUnionParam{
+		OfAudioPCM: &realtime.RealtimeAudioFormatsAudioPCMParam{
+			Rate: audioSampleRate,
+			Type: "audio/pcm",
+		},
+	}
+	credentialCtx, cancelCredential := context.WithTimeout(ctx, 30*time.Second)
+	secret, errSecret := client.Realtime.ClientSecrets.New(credentialCtx, realtime.ClientSecretNewParams{
+		ExpiresAfter: realtime.ClientSecretNewParamsExpiresAfter{
+			Anchor:  "created_at",
+			Seconds: openai.Int(600),
+		},
+		Session: realtime.ClientSecretNewParamsSessionUnion{
+			OfRealtime: &realtime.RealtimeSessionCreateRequestParam{
+				Model:            realtime.RealtimeSessionCreateRequestModel(cfg.model),
+				Instructions:     openai.String(cfg.instructions),
+				OutputModalities: []string{"audio"},
+				Audio: realtime.RealtimeAudioConfigParam{
+					Input: realtime.RealtimeAudioConfigInputParam{
+						Format: pcmFormat,
+					},
+					Output: realtime.RealtimeAudioConfigOutputParam{
+						Format: pcmFormat,
+						Voice: realtime.RealtimeAudioConfigOutputVoiceUnionParam{
+							OfString: openai.String(cfg.voice),
+						},
+					},
+				},
+			},
+		},
+	}, option.WithJSONSet("session.audio.input.turn_detection", nil))
+	cancelCredential()
+	if errSecret != nil {
+		return fmt.Errorf("create Realtime client secret with official SDK: %w", errSecret)
+	}
+	if secret == nil || strings.TrimSpace(secret.Value) == "" {
+		return errors.New("official SDK returned an empty Realtime client secret")
+	}
+
+	websocketURL, errWebsocketURL := realtimeWebsocketURL(cfg.baseURL, cfg.model)
+	if errWebsocketURL != nil {
+		return errWebsocketURL
+	}
+	headers := make(http.Header)
+	headers.Set("Authorization", "Bearer "+secret.Value)
+	connection, response, errDial := websocket.DefaultDialer.DialContext(ctx, websocketURL, headers)
+	if errDial != nil {
+		return websocketHandshakeError(response, errDial)
+	}
+	var closeOnce sync.Once
+	closeConnection := func() {
+		closeOnce.Do(func() {
+			if errClose := connection.Close(); errClose != nil && !websocket.IsCloseError(errClose, websocket.CloseNormalClosure, websocket.CloseGoingAway) {
+				fmt.Fprintf(output, "warning: close websocket: %v\n", errClose)
+			}
+		})
+	}
+	defer closeConnection()
+
+	connectionDone := make(chan struct{})
+	defer close(connectionDone)
+	go func() {
+		select {
+		case <-ctx.Done():
+			closeConnection()
+		case <-connectionDone:
+		}
+	}()
+
+	fmt.Fprintf(output, "Connected to %s using model %s and voice %s\n", websocketURL, cfg.model, cfg.voice)
+	if errSend := sendInputAudio(connection, inputPCM); errSend != nil {
+		return errSend
+	}
+	fmt.Fprintf(output, "Sent %s of speech audio\n", inputDuration.Round(time.Millisecond))
+
+	var responsePCM bytes.Buffer
+	fmt.Fprint(output, "Assistant transcript: ")
+	if errRead := readRealtimeResponse(ctx, connection, output, &responsePCM, cfg.debug); errRead != nil {
+		return errRead
+	}
+	if responsePCM.Len() == 0 {
+		return errors.New("Realtime response completed without audio")
+	}
+	if errWrite := writePCM16WAV(cfg.outputWAV, responsePCM.Bytes()); errWrite != nil {
+		return fmt.Errorf("write output WAV: %w", errWrite)
+	}
+	responseDuration := time.Duration(responsePCM.Len()) * time.Second / (audioSampleRate * audioBytesPerSample)
+	fmt.Fprintf(output, "Saved spoken response to %s (%s, %d PCM bytes)\n", cfg.outputWAV, responseDuration.Round(time.Millisecond), responsePCM.Len())
+	return nil
+}
+
+func sendInputAudio(connection *websocket.Conn, pcm []byte) error {
+	chunkSize := int(int64(audioSampleRate*audioBytesPerSample) * int64(audioChunkDuration) / int64(time.Second))
+	for offset := 0; offset < len(pcm); offset += chunkSize {
+		end := min(offset+chunkSize, len(pcm))
+		if errWrite := connection.WriteJSON(map[string]any{
+			"type":  "input_audio_buffer.append",
+			"audio": base64.StdEncoding.EncodeToString(pcm[offset:end]),
+		}); errWrite != nil {
+			return fmt.Errorf("append input audio: %w", errWrite)
+		}
+	}
+	if errWrite := connection.WriteJSON(map[string]any{"type": "input_audio_buffer.commit"}); errWrite != nil {
+		return fmt.Errorf("commit input audio: %w", errWrite)
+	}
+	if errWrite := connection.WriteJSON(map[string]any{
+		"type": "response.create",
+		"response": map[string]any{
+			"output_modalities": []string{"audio"},
+		},
+	}); errWrite != nil {
+		return fmt.Errorf("request spoken Realtime response: %w", errWrite)
+	}
+	return nil
+}
+
+func readRealtimeResponse(ctx context.Context, connection *websocket.Conn, output io.Writer, audioOutput *bytes.Buffer, debug bool) error {
+	for {
+		_, payload, errRead := connection.ReadMessage()
+		if errRead != nil {
+			if errContext := ctx.Err(); errContext != nil {
+				return errContext
+			}
+			if websocket.IsCloseError(errRead, websocket.CloseNormalClosure, websocket.CloseGoingAway) {
+				return errors.New("Realtime WebSocket closed before response.done")
+			}
+			return fmt.Errorf("read Realtime event: %w", errRead)
+		}
+		var event realtimeServerEvent
+		if errUnmarshal := json.Unmarshal(payload, &event); errUnmarshal != nil {
+			return fmt.Errorf("decode Realtime event: %w", errUnmarshal)
+		}
+		if debug {
+			fmt.Fprintf(output, "\n[event] %s\n", payload)
+		}
+		switch event.Type {
+		case "response.output_audio.delta", "response.audio.delta":
+			audio, errDecode := base64.StdEncoding.DecodeString(event.Delta)
+			if errDecode != nil {
+				return fmt.Errorf("decode response audio delta: %w", errDecode)
+			}
+			if audioOutput.Len()+len(audio) > maxOutputPCMBytes {
+				return fmt.Errorf("response PCM data exceeds %d bytes", maxOutputPCMBytes)
+			}
+			if _, errWrite := audioOutput.Write(audio); errWrite != nil {
+				return fmt.Errorf("buffer response audio: %w", errWrite)
+			}
+		case "response.output_audio_transcript.delta", "response.audio_transcript.delta":
+			fmt.Fprint(output, event.Delta)
+		case "response.done":
+			fmt.Fprintln(output)
+			if event.Response != nil && event.Response.Status != "" && event.Response.Status != "completed" {
+				return fmt.Errorf("Realtime response finished with status %s", event.Response.Status)
+			}
+			return nil
+		case "error":
+			if event.Error == nil {
+				return errors.New("Realtime API returned an unspecified error")
+			}
+			return fmt.Errorf("Realtime API error %s/%s: %s", event.Error.Type, event.Error.Code, event.Error.Message)
+		}
+	}
+}
+
+func normalizeBaseURL(rawURL string) (string, error) {
+	parsed, errParse := url.Parse(strings.TrimSpace(rawURL))
+	if errParse != nil {
+		return "", fmt.Errorf("parse OPENAI_BASE_URL: %w", errParse)
+	}
+	if parsed.Scheme != "http" && parsed.Scheme != "https" {
+		return "", errors.New("OPENAI_BASE_URL must use http or https")
+	}
+	if parsed.Host == "" {
+		return "", errors.New("OPENAI_BASE_URL must include a host")
+	}
+	parsed.RawQuery = ""
+	parsed.Fragment = ""
+	parsed.Path = strings.TrimRight(parsed.Path, "/")
+	if parsed.Path == "" {
+		parsed.Path = "/v1"
+	}
+	return parsed.String(), nil
+}
+
+func realtimeWebsocketURL(baseURL, model string) (string, error) {
+	parsed, errParse := url.Parse(baseURL)
+	if errParse != nil {
+		return "", fmt.Errorf("parse Realtime base URL: %w", errParse)
+	}
+	switch parsed.Scheme {
+	case "http":
+		parsed.Scheme = "ws"
+	case "https":
+		parsed.Scheme = "wss"
+	default:
+		return "", errors.New("Realtime base URL must use http or https")
+	}
+	parsed.Path = strings.TrimRight(parsed.Path, "/") + "/realtime"
+	query := parsed.Query()
+	query.Set("model", model)
+	parsed.RawQuery = query.Encode()
+	return parsed.String(), nil
+}
+
+func websocketHandshakeError(response *http.Response, errDial error) error {
+	if response == nil {
+		return fmt.Errorf("connect Realtime WebSocket: %w", errDial)
+	}
+	body, errRead := io.ReadAll(io.LimitReader(response.Body, 64<<10))
+	errClose := response.Body.Close()
+	if errRead != nil {
+		return fmt.Errorf("connect Realtime WebSocket: HTTP %d; read response: %v; dial: %w", response.StatusCode, errRead, errDial)
+	}
+	if errClose != nil {
+		return fmt.Errorf("connect Realtime WebSocket: HTTP %d; close response: %v; dial: %w", response.StatusCode, errClose, errDial)
+	}
+	message := strings.TrimSpace(string(body))
+	if message == "" {
+		message = http.StatusText(response.StatusCode)
+	}
+	return fmt.Errorf("connect Realtime WebSocket: HTTP %d: %s: %w", response.StatusCode, message, errDial)
+}
+
+func envOrDefault(name, fallback string) string {
+	if value := strings.TrimSpace(os.Getenv(name)); value != "" {
+		return value
+	}
+	return fallback
+}

backend/examples/realtime-openai-go/main_test.go

@@ -0,0 +1,226 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/gorilla/websocket"
+)
+
+func TestRunSendsAndReceivesSpeechAudio(t *testing.T) {
+	tmpDir := t.TempDir()
+	inputPath := filepath.Join(tmpDir, "input.wav")
+	outputPath := filepath.Join(tmpDir, "response.wav")
+	inputPCM := make([]byte, 9602)
+	for index := range inputPCM {
+		inputPCM[index] = byte(index % 251)
+	}
+	if errWrite := writePCM16WAV(inputPath, inputPCM); errWrite != nil {
+		t.Fatalf("write input WAV: %v", errWrite)
+	}
+	responsePCM := []byte{10, 20, 30, 40, 50, 60, 70, 80}
+
+	websocketEvents := make(chan []string, 1)
+	capturedInput := make(chan []byte, 1)
+	upgrader := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }}
+	server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+		switch request.URL.Path {
+		case "/v1/realtime/client_secrets":
+			if request.Method != http.MethodPost || request.Header.Get("Authorization") != "Bearer proxy-key" {
+				http.Error(writer, "invalid client secret request", http.StatusUnauthorized)
+				return
+			}
+			var body map[string]any
+			if errDecode := json.NewDecoder(request.Body).Decode(&body); errDecode != nil || !validAudioSession(body) {
+				http.Error(writer, "invalid audio session", http.StatusBadRequest)
+				return
+			}
+			writer.Header().Set("Content-Type", "application/json")
+			_, _ = writer.Write([]byte(`{
+				"value":"ek_test",
+				"expires_at":4102444800,
+				"session":{"id":"sess_test","object":"realtime.session","type":"realtime","model":"gpt-realtime"}
+			}`))
+		case "/v1/realtime":
+			if request.Header.Get("Authorization") != "Bearer ek_test" || request.URL.Query().Get("model") != defaultModel {
+				http.Error(writer, "invalid websocket request", http.StatusUnauthorized)
+				return
+			}
+			connection, errUpgrade := upgrader.Upgrade(writer, request, nil)
+			if errUpgrade != nil {
+				return
+			}
+			defer func() {
+				if errClose := connection.Close(); errClose != nil {
+					t.Logf("close test websocket: %v", errClose)
+				}
+			}()
+
+			types := make([]string, 0, 4)
+			var receivedPCM bytes.Buffer
+			for {
+				_, payload, errRead := connection.ReadMessage()
+				if errRead != nil {
+					return
+				}
+				var event struct {
+					Type  string `json:"type"`
+					Audio string `json:"audio"`
+				}
+				if errUnmarshal := json.Unmarshal(payload, &event); errUnmarshal != nil {
+					return
+				}
+				types = append(types, event.Type)
+				if event.Type == "input_audio_buffer.append" {
+					audio, errDecode := base64.StdEncoding.DecodeString(event.Audio)
+					if errDecode != nil {
+						return
+					}
+					_, _ = receivedPCM.Write(audio)
+				}
+				if event.Type == "response.create" {
+					break
+				}
+			}
+			websocketEvents <- types
+			capturedInput <- append([]byte(nil), receivedPCM.Bytes()...)
+			midpoint := len(responsePCM) / 2
+			for _, audio := range [][]byte{responsePCM[:midpoint], responsePCM[midpoint:]} {
+				if errWrite := connection.WriteJSON(map[string]any{
+					"type":  "response.output_audio.delta",
+					"delta": base64.StdEncoding.EncodeToString(audio),
+				}); errWrite != nil {
+					return
+				}
+			}
+			if errWrite := connection.WriteJSON(map[string]any{"type": "response.output_audio_transcript.delta", "delta": "Voice response"}); errWrite != nil {
+				return
+			}
+			_ = connection.WriteJSON(map[string]any{"type": "response.done", "response": map[string]any{"status": "completed"}})
+		default:
+			http.NotFound(writer, request)
+		}
+	}))
+	defer server.Close()
+
+	baseURL, errBaseURL := normalizeBaseURL(server.URL + "/v1/")
+	if errBaseURL != nil {
+		t.Fatalf("normalizeBaseURL() error = %v", errBaseURL)
+	}
+	var output bytes.Buffer
+	errRun := run(context.Background(), appConfig{
+		baseURL:      baseURL,
+		apiKey:       "proxy-key",
+		model:        defaultModel,
+		inputWAV:     inputPath,
+		outputWAV:    outputPath,
+		instructions: defaultInstructions,
+		voice:        defaultVoice,
+	}, &output)
+	if errRun != nil {
+		t.Fatalf("run() error = %v", errRun)
+	}
+	if !strings.Contains(output.String(), "Sent") || !strings.Contains(output.String(), "Assistant transcript: Voice response") || !strings.Contains(output.String(), "Saved spoken response") {
+		t.Fatalf("output = %q", output.String())
+	}
+	select {
+	case events := <-websocketEvents:
+		want := []string{"input_audio_buffer.append", "input_audio_buffer.append", "input_audio_buffer.commit", "response.create"}
+		if strings.Join(events, ",") != strings.Join(want, ",") {
+			t.Fatalf("client events = %v, want %v", events, want)
+		}
+	default:
+		t.Fatal("websocket events were not captured")
+	}
+	select {
+	case audio := <-capturedInput:
+		if !bytes.Equal(audio, inputPCM) {
+			t.Fatalf("input PCM mismatch: got %d bytes, want %d", len(audio), len(inputPCM))
+		}
+	default:
+		t.Fatal("input audio was not captured")
+	}
+	actualResponsePCM, errRead := readPCM16WAV(outputPath)
+	if errRead != nil {
+		t.Fatalf("read output WAV: %v", errRead)
+	}
+	if !bytes.Equal(actualResponsePCM, responsePCM) {
+		t.Fatalf("response PCM = %v, want %v", actualResponsePCM, responsePCM)
+	}
+}
+
+func validAudioSession(body map[string]any) bool {
+	session, ok := body["session"].(map[string]any)
+	if !ok || session["type"] != "realtime" || session["model"] != defaultModel {
+		return false
+	}
+	modalities, ok := session["output_modalities"].([]any)
+	if !ok || len(modalities) != 1 || modalities[0] != "audio" {
+		return false
+	}
+	audio, ok := session["audio"].(map[string]any)
+	if !ok {
+		return false
+	}
+	input, inputOK := audio["input"].(map[string]any)
+	output, outputOK := audio["output"].(map[string]any)
+	if !inputOK || !outputOK {
+		return false
+	}
+	inputFormat, inputFormatOK := input["format"].(map[string]any)
+	outputFormat, outputFormatOK := output["format"].(map[string]any)
+	if !inputFormatOK || !outputFormatOK {
+		return false
+	}
+	_, turnDetectionPresent := input["turn_detection"]
+	return inputFormat["type"] == "audio/pcm" && inputFormat["rate"] == float64(audioSampleRate) &&
+		outputFormat["type"] == "audio/pcm" && outputFormat["rate"] == float64(audioSampleRate) &&
+		output["voice"] == defaultVoice && turnDetectionPresent && input["turn_detection"] == nil
+}
+
+func TestNormalizeBaseURLAddsV1(t *testing.T) {
+	baseURL, errNormalize := normalizeBaseURL("http://127.0.0.1:8317/")
+	if errNormalize != nil {
+		t.Fatalf("normalizeBaseURL() error = %v", errNormalize)
+	}
+	if baseURL != "http://127.0.0.1:8317/v1" {
+		t.Fatalf("baseURL = %q", baseURL)
+	}
+	websocketURL, errWebsocketURL := realtimeWebsocketURL(baseURL, defaultModel)
+	if errWebsocketURL != nil {
+		t.Fatalf("realtimeWebsocketURL() error = %v", errWebsocketURL)
+	}
+	wantWebsocketURL := "ws://127.0.0.1:8317/v1/realtime?model=" + defaultModel
+	if websocketURL != wantWebsocketURL {
+		t.Fatalf("websocketURL = %q", websocketURL)
+	}
+}
+
+func TestReadPCM16WAVRejectsWrongSampleRate(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "wrong-rate.wav")
+	if errWrite := writePCM16WAV(path, []byte{1, 2, 3, 4}); errWrite != nil {
+		t.Fatalf("writePCM16WAV() error = %v", errWrite)
+	}
+	payload, errRead := os.ReadFile(path)
+	if errRead != nil {
+		t.Fatalf("read WAV: %v", errRead)
+	}
+	payload[24] = 0x80
+	payload[25] = 0xbb
+	payload[26] = 0x00
+	payload[27] = 0x00
+	if errWrite := os.WriteFile(path, payload, 0o644); errWrite != nil {
+		t.Fatalf("rewrite WAV: %v", errWrite)
+	}
+	if _, errRead = readPCM16WAV(path); errRead == nil || !strings.Contains(errRead.Error(), "24000") {
+		t.Fatalf("readPCM16WAV() error = %v", errRead)
+	}
+}

backend/examples/realtime-openai-go/wav.go

@@ -0,0 +1,147 @@
+package main
+
+import (
+	"bytes"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"os"
+)
+
+const (
+	maxInputPCMBytes  = 15 << 20
+	maxOutputPCMBytes = 64 << 20
+	wavHeaderSize     = 44
+)
+
+func readPCM16WAV(path string) ([]byte, error) {
+	fileInfo, errStat := os.Stat(path)
+	if errStat != nil {
+		return nil, errStat
+	}
+	if fileInfo.Size() > maxInputPCMBytes+(1<<20) {
+		return nil, fmt.Errorf("WAV file is too large: %d bytes", fileInfo.Size())
+	}
+	payload, errRead := os.ReadFile(path)
+	if errRead != nil {
+		return nil, errRead
+	}
+	if len(payload) < 12 || string(payload[:4]) != "RIFF" || string(payload[8:12]) != "WAVE" {
+		return nil, errors.New("input is not a RIFF/WAVE file")
+	}
+
+	var formatFound bool
+	var audioFormat uint16
+	var channels uint16
+	var sampleRate uint32
+	var bitsPerSample uint16
+	var pcm bytes.Buffer
+	for offset := 12; offset+8 <= len(payload); {
+		chunkID := string(payload[offset : offset+4])
+		chunkSize := int(binary.LittleEndian.Uint32(payload[offset+4 : offset+8]))
+		chunkStart := offset + 8
+		chunkEnd := chunkStart + chunkSize
+		if chunkSize < 0 || chunkEnd < chunkStart || chunkEnd > len(payload) {
+			return nil, fmt.Errorf("invalid WAV %q chunk size", chunkID)
+		}
+		switch chunkID {
+		case "fmt ":
+			if chunkSize < 16 {
+				return nil, errors.New("WAV fmt chunk is too short")
+			}
+			audioFormat = binary.LittleEndian.Uint16(payload[chunkStart : chunkStart+2])
+			channels = binary.LittleEndian.Uint16(payload[chunkStart+2 : chunkStart+4])
+			sampleRate = binary.LittleEndian.Uint32(payload[chunkStart+4 : chunkStart+8])
+			bitsPerSample = binary.LittleEndian.Uint16(payload[chunkStart+14 : chunkStart+16])
+			formatFound = true
+		case "data":
+			if pcm.Len()+chunkSize > maxInputPCMBytes {
+				return nil, fmt.Errorf("WAV PCM data exceeds %d bytes", maxInputPCMBytes)
+			}
+			_, _ = pcm.Write(payload[chunkStart:chunkEnd])
+		}
+		offset = chunkEnd
+		if chunkSize%2 != 0 {
+			offset++
+		}
+	}
+	if !formatFound {
+		return nil, errors.New("WAV fmt chunk is missing")
+	}
+	if audioFormat != 1 {
+		return nil, fmt.Errorf("WAV audio format must be PCM (1), got %d", audioFormat)
+	}
+	if channels != 1 {
+		return nil, fmt.Errorf("WAV must be mono, got %d channels", channels)
+	}
+	if sampleRate != audioSampleRate {
+		return nil, fmt.Errorf("WAV sample rate must be %d Hz, got %d Hz", audioSampleRate, sampleRate)
+	}
+	if bitsPerSample != 16 {
+		return nil, fmt.Errorf("WAV must use 16-bit samples, got %d bits", bitsPerSample)
+	}
+	if pcm.Len() == 0 {
+		return nil, errors.New("WAV data chunk is empty or missing")
+	}
+	if pcm.Len()%audioBytesPerSample != 0 {
+		return nil, errors.New("WAV PCM data contains an incomplete sample")
+	}
+	return append([]byte(nil), pcm.Bytes()...), nil
+}
+
+func writePCM16WAV(path string, pcm []byte) error {
+	if len(pcm) == 0 {
+		return errors.New("cannot write an empty WAV response")
+	}
+	if len(pcm) > maxOutputPCMBytes {
+		return fmt.Errorf("response PCM data exceeds %d bytes", maxOutputPCMBytes)
+	}
+	if len(pcm)%audioBytesPerSample != 0 {
+		return errors.New("response PCM data contains an incomplete sample")
+	}
+
+	var payload bytes.Buffer
+	payload.Grow(wavHeaderSize + len(pcm))
+	writeString := func(value string) error {
+		_, errWrite := payload.WriteString(value)
+		return errWrite
+	}
+	writeValue := func(value any) error {
+		return binary.Write(&payload, binary.LittleEndian, value)
+	}
+	if errWrite := writeString("RIFF"); errWrite != nil {
+		return errWrite
+	}
+	if errWrite := writeValue(uint32(36 + len(pcm))); errWrite != nil {
+		return errWrite
+	}
+	if errWrite := writeString("WAVEfmt "); errWrite != nil {
+		return errWrite
+	}
+	for _, value := range []any{
+		uint32(16),
+		uint16(1),
+		uint16(1),
+		uint32(audioSampleRate),
+		uint32(audioSampleRate * audioBytesPerSample),
+		uint16(audioBytesPerSample),
+		uint16(16),
+	} {
+		if errWrite := writeValue(value); errWrite != nil {
+			return errWrite
+		}
+	}
+	if errWrite := writeString("data"); errWrite != nil {
+		return errWrite
+	}
+	if errWrite := writeValue(uint32(len(pcm))); errWrite != nil {
+		return errWrite
+	}
+	if _, errWrite := payload.Write(pcm); errWrite != nil {
+		return errWrite
+	}
+	if errWrite := os.WriteFile(path, payload.Bytes(), 0o644); errWrite != nil {
+		return errWrite
+	}
+	return nil
+}