vibe-proxy/backend/examples/realtime-openai-go/main.go

package main

import (
	"bytes"
	"context"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/gorilla/websocket"
	"github.com/openai/openai-go/v3"
	"github.com/openai/openai-go/v3/option"
	"github.com/openai/openai-go/v3/realtime"
)

const (
	defaultBaseURL      = "http://127.0.0.1:8317/v1"
	defaultModel        = "gpt-realtime-2.1"
	defaultInstructions = "Listen to the user's speech and reply with a short spoken response."
	defaultOutputWAV    = "response.wav"
	defaultVoice        = "marin"
	audioSampleRate     = 24000
	audioBytesPerSample = 2
	audioChunkDuration  = 200 * time.Millisecond
)

type appConfig struct {
	baseURL      string
	apiKey       string
	model        string
	inputWAV     string
	outputWAV    string
	instructions string
	voice        string
	debug        bool
}

type realtimeServerEvent struct {
	Type  string `json:"type"`
	Delta string `json:"delta"`
	Error *struct {
		Message string `json:"message"`
		Type    string `json:"type"`
		Code    string `json:"code"`
	} `json:"error,omitempty"`
	Response *struct {
		Status string `json:"status"`
	} `json:"response,omitempty"`
}

func main() {
	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
	defer stop()

	cfg, errConfig := loadConfig()
	if errConfig != nil {
		fmt.Fprintf(os.Stderr, "configuration error: %v\n", errConfig)
		os.Exit(1)
	}
	if errRun := run(ctx, cfg, os.Stdout); errRun != nil {
		fmt.Fprintf(os.Stderr, "realtime example failed: %v\n", errRun)
		os.Exit(1)
	}
}

func loadConfig() (appConfig, error) {
	baseURL, errBaseURL := normalizeBaseURL(envOrDefault("OPENAI_BASE_URL", defaultBaseURL))
	if errBaseURL != nil {
		return appConfig{}, errBaseURL
	}
	apiKey := strings.TrimSpace(os.Getenv("OPENAI_API_KEY"))
	if apiKey == "" {
		return appConfig{}, errors.New("OPENAI_API_KEY is required")
	}
	inputWAV := strings.TrimSpace(os.Getenv("OPENAI_REALTIME_INPUT_WAV"))
	if inputWAV == "" {
		return appConfig{}, errors.New("OPENAI_REALTIME_INPUT_WAV is required")
	}
	return appConfig{
		baseURL:      baseURL,
		apiKey:       apiKey,
		model:        envOrDefault("OPENAI_REALTIME_MODEL", defaultModel),
		inputWAV:     inputWAV,
		outputWAV:    envOrDefault("OPENAI_REALTIME_OUTPUT_WAV", defaultOutputWAV),
		instructions: envOrDefault("OPENAI_REALTIME_INSTRUCTIONS", defaultInstructions),
		voice:        envOrDefault("OPENAI_REALTIME_VOICE", defaultVoice),
		debug:        strings.EqualFold(strings.TrimSpace(os.Getenv("OPENAI_REALTIME_DEBUG")), "true"),
	}, nil
}

func run(ctx context.Context, cfg appConfig, output io.Writer) error {
	inputPCM, errInput := readPCM16WAV(cfg.inputWAV)
	if errInput != nil {
		return fmt.Errorf("read input WAV: %w", errInput)
	}
	inputDuration := time.Duration(len(inputPCM)) * time.Second / (audioSampleRate * audioBytesPerSample)
	fmt.Fprintf(output, "Loaded %s (%s, %d PCM bytes)\n", cfg.inputWAV, inputDuration.Round(time.Millisecond), len(inputPCM))

	client := openai.NewClient(
		option.WithAPIKey(cfg.apiKey),
		option.WithBaseURL(cfg.baseURL),
	)
	pcmFormat := realtime.RealtimeAudioFormatsUnionParam{
		OfAudioPCM: &realtime.RealtimeAudioFormatsAudioPCMParam{
			Rate: audioSampleRate,
			Type: "audio/pcm",
		},
	}
	credentialCtx, cancelCredential := context.WithTimeout(ctx, 30*time.Second)
	secret, errSecret := client.Realtime.ClientSecrets.New(credentialCtx, realtime.ClientSecretNewParams{
		ExpiresAfter: realtime.ClientSecretNewParamsExpiresAfter{
			Anchor:  "created_at",
			Seconds: openai.Int(600),
		},
		Session: realtime.ClientSecretNewParamsSessionUnion{
			OfRealtime: &realtime.RealtimeSessionCreateRequestParam{
				Model:            realtime.RealtimeSessionCreateRequestModel(cfg.model),
				Instructions:     openai.String(cfg.instructions),
				OutputModalities: []string{"audio"},
				Audio: realtime.RealtimeAudioConfigParam{
					Input: realtime.RealtimeAudioConfigInputParam{
						Format: pcmFormat,
					},
					Output: realtime.RealtimeAudioConfigOutputParam{
						Format: pcmFormat,
						Voice: realtime.RealtimeAudioConfigOutputVoiceUnionParam{
							OfString: openai.String(cfg.voice),
						},
					},
				},
			},
		},
	}, option.WithJSONSet("session.audio.input.turn_detection", nil))
	cancelCredential()
	if errSecret != nil {
		return fmt.Errorf("create Realtime client secret with official SDK: %w", errSecret)
	}
	if secret == nil || strings.TrimSpace(secret.Value) == "" {
		return errors.New("official SDK returned an empty Realtime client secret")
	}

	websocketURL, errWebsocketURL := realtimeWebsocketURL(cfg.baseURL, cfg.model)
	if errWebsocketURL != nil {
		return errWebsocketURL
	}
	headers := make(http.Header)
	headers.Set("Authorization", "Bearer "+secret.Value)
	connection, response, errDial := websocket.DefaultDialer.DialContext(ctx, websocketURL, headers)
	if errDial != nil {
		return websocketHandshakeError(response, errDial)
	}
	var closeOnce sync.Once
	closeConnection := func() {
		closeOnce.Do(func() {
			if errClose := connection.Close(); errClose != nil && !websocket.IsCloseError(errClose, websocket.CloseNormalClosure, websocket.CloseGoingAway) {
				fmt.Fprintf(output, "warning: close websocket: %v\n", errClose)
			}
		})
	}
	defer closeConnection()

	connectionDone := make(chan struct{})
	defer close(connectionDone)
	go func() {
		select {
		case <-ctx.Done():
			closeConnection()
		case <-connectionDone:
		}
	}()

	fmt.Fprintf(output, "Connected to %s using model %s and voice %s\n", websocketURL, cfg.model, cfg.voice)
	if errSend := sendInputAudio(connection, inputPCM); errSend != nil {
		return errSend
	}
	fmt.Fprintf(output, "Sent %s of speech audio\n", inputDuration.Round(time.Millisecond))

	var responsePCM bytes.Buffer
	fmt.Fprint(output, "Assistant transcript: ")
	if errRead := readRealtimeResponse(ctx, connection, output, &responsePCM, cfg.debug); errRead != nil {
		return errRead
	}
	if responsePCM.Len() == 0 {
		return errors.New("Realtime response completed without audio")
	}
	if errWrite := writePCM16WAV(cfg.outputWAV, responsePCM.Bytes()); errWrite != nil {
		return fmt.Errorf("write output WAV: %w", errWrite)
	}
	responseDuration := time.Duration(responsePCM.Len()) * time.Second / (audioSampleRate * audioBytesPerSample)
	fmt.Fprintf(output, "Saved spoken response to %s (%s, %d PCM bytes)\n", cfg.outputWAV, responseDuration.Round(time.Millisecond), responsePCM.Len())
	return nil
}

func sendInputAudio(connection *websocket.Conn, pcm []byte) error {
	chunkSize := int(int64(audioSampleRate*audioBytesPerSample) * int64(audioChunkDuration) / int64(time.Second))
	for offset := 0; offset < len(pcm); offset += chunkSize {
		end := min(offset+chunkSize, len(pcm))
		if errWrite := connection.WriteJSON(map[string]any{
			"type":  "input_audio_buffer.append",
			"audio": base64.StdEncoding.EncodeToString(pcm[offset:end]),
		}); errWrite != nil {
			return fmt.Errorf("append input audio: %w", errWrite)
		}
	}
	if errWrite := connection.WriteJSON(map[string]any{"type": "input_audio_buffer.commit"}); errWrite != nil {
		return fmt.Errorf("commit input audio: %w", errWrite)
	}
	if errWrite := connection.WriteJSON(map[string]any{
		"type": "response.create",
		"response": map[string]any{
			"output_modalities": []string{"audio"},
		},
	}); errWrite != nil {
		return fmt.Errorf("request spoken Realtime response: %w", errWrite)
	}
	return nil
}

func readRealtimeResponse(ctx context.Context, connection *websocket.Conn, output io.Writer, audioOutput *bytes.Buffer, debug bool) error {
	for {
		_, payload, errRead := connection.ReadMessage()
		if errRead != nil {
			if errContext := ctx.Err(); errContext != nil {
				return errContext
			}
			if websocket.IsCloseError(errRead, websocket.CloseNormalClosure, websocket.CloseGoingAway) {
				return errors.New("Realtime WebSocket closed before response.done")
			}
			return fmt.Errorf("read Realtime event: %w", errRead)
		}
		var event realtimeServerEvent
		if errUnmarshal := json.Unmarshal(payload, &event); errUnmarshal != nil {
			return fmt.Errorf("decode Realtime event: %w", errUnmarshal)
		}
		if debug {
			fmt.Fprintf(output, "\n[event] %s\n", payload)
		}
		switch event.Type {
		case "response.output_audio.delta", "response.audio.delta":
			audio, errDecode := base64.StdEncoding.DecodeString(event.Delta)
			if errDecode != nil {
				return fmt.Errorf("decode response audio delta: %w", errDecode)
			}
			if audioOutput.Len()+len(audio) > maxOutputPCMBytes {
				return fmt.Errorf("response PCM data exceeds %d bytes", maxOutputPCMBytes)
			}
			if _, errWrite := audioOutput.Write(audio); errWrite != nil {
				return fmt.Errorf("buffer response audio: %w", errWrite)
			}
		case "response.output_audio_transcript.delta", "response.audio_transcript.delta":
			fmt.Fprint(output, event.Delta)
		case "response.done":
			fmt.Fprintln(output)
			if event.Response != nil && event.Response.Status != "" && event.Response.Status != "completed" {
				return fmt.Errorf("Realtime response finished with status %s", event.Response.Status)
			}
			return nil
		case "error":
			if event.Error == nil {
				return errors.New("Realtime API returned an unspecified error")
			}
			return fmt.Errorf("Realtime API error %s/%s: %s", event.Error.Type, event.Error.Code, event.Error.Message)
		}
	}
}

func normalizeBaseURL(rawURL string) (string, error) {
	parsed, errParse := url.Parse(strings.TrimSpace(rawURL))
	if errParse != nil {
		return "", fmt.Errorf("parse OPENAI_BASE_URL: %w", errParse)
	}
	if parsed.Scheme != "http" && parsed.Scheme != "https" {
		return "", errors.New("OPENAI_BASE_URL must use http or https")
	}
	if parsed.Host == "" {
		return "", errors.New("OPENAI_BASE_URL must include a host")
	}
	parsed.RawQuery = ""
	parsed.Fragment = ""
	parsed.Path = strings.TrimRight(parsed.Path, "/")
	if parsed.Path == "" {
		parsed.Path = "/v1"
	}
	return parsed.String(), nil
}

func realtimeWebsocketURL(baseURL, model string) (string, error) {
	parsed, errParse := url.Parse(baseURL)
	if errParse != nil {
		return "", fmt.Errorf("parse Realtime base URL: %w", errParse)
	}
	switch parsed.Scheme {
	case "http":
		parsed.Scheme = "ws"
	case "https":
		parsed.Scheme = "wss"
	default:
		return "", errors.New("Realtime base URL must use http or https")
	}
	parsed.Path = strings.TrimRight(parsed.Path, "/") + "/realtime"
	query := parsed.Query()
	query.Set("model", model)
	parsed.RawQuery = query.Encode()
	return parsed.String(), nil
}

func websocketHandshakeError(response *http.Response, errDial error) error {
	if response == nil {
		return fmt.Errorf("connect Realtime WebSocket: %w", errDial)
	}
	body, errRead := io.ReadAll(io.LimitReader(response.Body, 64<<10))
	errClose := response.Body.Close()
	if errRead != nil {
		return fmt.Errorf("connect Realtime WebSocket: HTTP %d; read response: %v; dial: %w", response.StatusCode, errRead, errDial)
	}
	if errClose != nil {
		return fmt.Errorf("connect Realtime WebSocket: HTTP %d; close response: %v; dial: %w", response.StatusCode, errClose, errDial)
	}
	message := strings.TrimSpace(string(body))
	if message == "" {
		message = http.StatusText(response.StatusCode)
	}
	return fmt.Errorf("connect Realtime WebSocket: HTTP %d: %s: %w", response.StatusCode, message, errDial)
}

func envOrDefault(name, fallback string) string {
	if value := strings.TrimSpace(os.Getenv(name)); value != "" {
		return value
	}
	return fallback
}