Add projects
This commit is contained in:
parent
2d3a9ad623
commit
8b607dd700
SSH key fingerprint:
SHA256:GBzT2DXvAuGV9XIV5W3WrzVpjU54FThmxHXdbz95J24
1802 changed files with 503346 additions and 2 deletions
455
backend/internal/runtime/executor/codex_executor_terminal.go
Normal file
455
backend/internal/runtime/executor/codex_executor_terminal.go
Normal file
|
|
@ -0,0 +1,455 @@
|
|||
package executor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/tidwall/gjson"
|
||||
"github.com/tidwall/sjson"
|
||||
)
|
||||
|
||||
const codexIncompleteStreamMessage = "stream error: stream disconnected before completion: stream closed before response.completed"
|
||||
|
||||
type codexIncompleteStreamError struct {
|
||||
statusErr
|
||||
}
|
||||
|
||||
func newCodexIncompleteStreamError() codexIncompleteStreamError {
|
||||
return codexIncompleteStreamError{statusErr: statusErr{
|
||||
code: http.StatusRequestTimeout,
|
||||
msg: codexIncompleteStreamMessage,
|
||||
}}
|
||||
}
|
||||
|
||||
func (codexIncompleteStreamError) IsRequestScoped() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// Streamed Codex responses may emit response.output_item.done events while leaving
|
||||
// response.completed.response.output empty. Keep the stream path aligned with the
|
||||
// already-patched non-stream path by reconstructing response.output from those items.
|
||||
func collectCodexOutputItemDone(eventData []byte, outputItemsByIndex map[int64][]byte, outputItemsFallback *[][]byte) {
|
||||
itemResult := gjson.GetBytes(eventData, "item")
|
||||
if !itemResult.Exists() || itemResult.Type != gjson.JSON {
|
||||
return
|
||||
}
|
||||
outputIndexResult := gjson.GetBytes(eventData, "output_index")
|
||||
if outputIndexResult.Exists() {
|
||||
outputItemsByIndex[outputIndexResult.Int()] = []byte(itemResult.Raw)
|
||||
return
|
||||
}
|
||||
*outputItemsFallback = append(*outputItemsFallback, []byte(itemResult.Raw))
|
||||
}
|
||||
|
||||
func hydrateCodexCompletedOutputItemIDs(eventData []byte, outputItems []gjson.Result, outputItemsByIndex map[int64][]byte) []byte {
|
||||
patchedData := eventData
|
||||
for outputIndex, outputItem := range outputItems {
|
||||
itemData := []byte(outputItem.Raw)
|
||||
itemID := gjson.GetBytes(itemData, "id")
|
||||
if itemID.Exists() && itemID.Type != gjson.Null && (itemID.Type != gjson.String || strings.TrimSpace(itemID.String()) != "") {
|
||||
continue
|
||||
}
|
||||
|
||||
completedItem, ok := outputItemsByIndex[int64(outputIndex)]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
completedID := gjson.GetBytes(completedItem, "id")
|
||||
if completedID.Type != gjson.String || strings.TrimSpace(completedID.String()) == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
updatedData, errSet := sjson.SetRawBytes(patchedData, "response.output."+strconv.Itoa(outputIndex)+".id", []byte(completedID.Raw))
|
||||
if errSet != nil {
|
||||
continue
|
||||
}
|
||||
patchedData = updatedData
|
||||
}
|
||||
return patchedData
|
||||
}
|
||||
|
||||
func patchCodexCompletedOutput(eventData []byte, outputItemsByIndex map[int64][]byte, outputItemsFallback [][]byte) []byte {
|
||||
outputResult := gjson.GetBytes(eventData, "response.output")
|
||||
if outputResult.Exists() && outputResult.IsArray() && len(outputResult.Array()) > 0 {
|
||||
return hydrateCodexCompletedOutputItemIDs(eventData, outputResult.Array(), outputItemsByIndex)
|
||||
}
|
||||
|
||||
shouldPatchOutput := (!outputResult.Exists() || !outputResult.IsArray() || len(outputResult.Array()) == 0) && (len(outputItemsByIndex) > 0 || len(outputItemsFallback) > 0)
|
||||
if !shouldPatchOutput {
|
||||
return eventData
|
||||
}
|
||||
|
||||
indexes := make([]int64, 0, len(outputItemsByIndex))
|
||||
for idx := range outputItemsByIndex {
|
||||
indexes = append(indexes, idx)
|
||||
}
|
||||
sort.Slice(indexes, func(i, j int) bool {
|
||||
return indexes[i] < indexes[j]
|
||||
})
|
||||
|
||||
items := make([][]byte, 0, len(outputItemsByIndex)+len(outputItemsFallback))
|
||||
for _, idx := range indexes {
|
||||
items = append(items, outputItemsByIndex[idx])
|
||||
}
|
||||
items = append(items, outputItemsFallback...)
|
||||
|
||||
outputArray := []byte("[]")
|
||||
if len(items) > 0 {
|
||||
var buf bytes.Buffer
|
||||
totalLen := 2
|
||||
for _, item := range items {
|
||||
totalLen += len(item)
|
||||
}
|
||||
if len(items) > 1 {
|
||||
totalLen += len(items) - 1
|
||||
}
|
||||
buf.Grow(totalLen)
|
||||
buf.WriteByte('[')
|
||||
for i, item := range items {
|
||||
if i > 0 {
|
||||
buf.WriteByte(',')
|
||||
}
|
||||
buf.Write(item)
|
||||
}
|
||||
buf.WriteByte(']')
|
||||
outputArray = buf.Bytes()
|
||||
}
|
||||
|
||||
completedDataPatched, _ := sjson.SetRawBytes(eventData, "response.output", outputArray)
|
||||
return completedDataPatched
|
||||
}
|
||||
|
||||
func codexTerminalStreamContextLengthErr(eventData []byte) (statusErr, bool) {
|
||||
streamErr, body, ok := codexTerminalStreamErr(eventData)
|
||||
if !ok || !codexTerminalErrorIsContextLength(body) {
|
||||
return statusErr{}, false
|
||||
}
|
||||
return streamErr, true
|
||||
}
|
||||
|
||||
func codexTerminalStreamErr(eventData []byte) (statusErr, []byte, bool) {
|
||||
body, ok := codexTerminalFailureBody(eventData)
|
||||
if !ok || !codexTerminalStreamErrShouldHandle(body) {
|
||||
return statusErr{}, nil, false
|
||||
}
|
||||
return newCodexStatusErr(http.StatusBadRequest, body), body, true
|
||||
}
|
||||
|
||||
func codexTerminalFailureErr(eventData []byte) (statusErr, []byte, bool) {
|
||||
if streamErr, body, ok := codexTerminalStreamErr(eventData); ok {
|
||||
return streamErr, body, true
|
||||
}
|
||||
body, ok := codexTerminalFailureBody(eventData)
|
||||
if !ok {
|
||||
return statusErr{}, nil, false
|
||||
}
|
||||
return newCodexStatusErr(codexTerminalFailureStatus(body), body), body, true
|
||||
}
|
||||
|
||||
func codexTerminalFailureStatus(body []byte) int {
|
||||
for _, path := range []string{"error.status_code", "error.status"} {
|
||||
if status := int(gjson.GetBytes(body, path).Int()); status >= 400 && status <= 599 {
|
||||
return status
|
||||
}
|
||||
}
|
||||
|
||||
errorType := strings.ToLower(strings.TrimSpace(gjson.GetBytes(body, "error.type").String()))
|
||||
errorCode := strings.ToLower(strings.TrimSpace(gjson.GetBytes(body, "error.code").String()))
|
||||
switch {
|
||||
case errorCode == "cyber_policy":
|
||||
return http.StatusBadRequest
|
||||
case errorType == "invalid_request_error", errorType == "bad_request_error":
|
||||
return http.StatusBadRequest
|
||||
case errorType == "authentication_error", errorCode == "invalid_api_key", errorCode == "unauthorized":
|
||||
return http.StatusUnauthorized
|
||||
case errorType == "permission_error", errorCode == "forbidden", errorCode == "permission_denied":
|
||||
return http.StatusForbidden
|
||||
case errorType == "not_found_error", errorCode == "not_found", errorCode == "model_not_found":
|
||||
return http.StatusNotFound
|
||||
case errorType == "rate_limit_error", errorCode == "rate_limit_exceeded":
|
||||
return http.StatusTooManyRequests
|
||||
default:
|
||||
return http.StatusBadGateway
|
||||
}
|
||||
}
|
||||
|
||||
func codexTerminalFailureBody(eventData []byte) ([]byte, bool) {
|
||||
eventType := gjson.GetBytes(eventData, "type").String()
|
||||
var body []byte
|
||||
switch eventType {
|
||||
case "error":
|
||||
body = codexTerminalErrorBody(eventData, "error")
|
||||
if len(body) == 0 {
|
||||
body = codexTerminalTopLevelErrorBody(eventData)
|
||||
}
|
||||
case "response.failed":
|
||||
body = codexTerminalErrorBody(eventData, "response.error")
|
||||
if len(body) == 0 {
|
||||
body = codexTerminalErrorBody(eventData, "error")
|
||||
}
|
||||
default:
|
||||
return nil, false
|
||||
}
|
||||
if len(body) == 0 {
|
||||
body = []byte(`{"error":{"message":"upstream stream failed without error details"}}`)
|
||||
}
|
||||
return body, true
|
||||
}
|
||||
|
||||
func codexTerminalStreamErrShouldHandle(body []byte) bool {
|
||||
if codexTerminalErrorIsContextLength(body) {
|
||||
return true
|
||||
}
|
||||
if isCodexUsageLimitError(body) || isCodexModelCapacityError(body) {
|
||||
return true
|
||||
}
|
||||
code, _, ok := codexStatusErrorClassification(http.StatusBadRequest, body)
|
||||
return ok && code == "thinking_signature_invalid"
|
||||
}
|
||||
|
||||
func codexTerminalErrorBody(eventData []byte, path string) []byte {
|
||||
errorResult := gjson.GetBytes(eventData, path)
|
||||
if !errorResult.Exists() {
|
||||
return nil
|
||||
}
|
||||
body := []byte(`{"error":{}}`)
|
||||
if errorResult.Type == gjson.JSON {
|
||||
body, _ = sjson.SetRawBytes(body, "error", []byte(errorResult.Raw))
|
||||
} else if message := strings.TrimSpace(errorResult.String()); message != "" {
|
||||
body, _ = sjson.SetBytes(body, "error.message", message)
|
||||
}
|
||||
if strings.TrimSpace(gjson.GetBytes(body, "error.message").String()) == "" {
|
||||
if message := strings.TrimSpace(gjson.GetBytes(eventData, "response.error.message").String()); message != "" {
|
||||
body, _ = sjson.SetBytes(body, "error.message", message)
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(gjson.GetBytes(body, "error.message").String()) == "" {
|
||||
if code := strings.TrimSpace(gjson.GetBytes(body, "error.code").String()); code != "" {
|
||||
body, _ = sjson.SetBytes(body, "error.message", code)
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(gjson.GetBytes(body, "error.message").String()) == "" {
|
||||
if errorType := strings.TrimSpace(gjson.GetBytes(body, "error.type").String()); errorType != "" {
|
||||
body, _ = sjson.SetBytes(body, "error.message", errorType)
|
||||
}
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
func codexTerminalTopLevelErrorBody(eventData []byte) []byte {
|
||||
message := strings.TrimSpace(gjson.GetBytes(eventData, "message").String())
|
||||
code := strings.TrimSpace(gjson.GetBytes(eventData, "code").String())
|
||||
errorType := strings.TrimSpace(gjson.GetBytes(eventData, "error_type").String())
|
||||
param := strings.TrimSpace(gjson.GetBytes(eventData, "param").String())
|
||||
if message == "" && code == "" && errorType == "" && param == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
body := []byte(`{"error":{}}`)
|
||||
if message != "" {
|
||||
body, _ = sjson.SetBytes(body, "error.message", message)
|
||||
}
|
||||
if code != "" {
|
||||
body, _ = sjson.SetBytes(body, "error.code", code)
|
||||
}
|
||||
if errorType != "" {
|
||||
body, _ = sjson.SetBytes(body, "error.type", errorType)
|
||||
}
|
||||
if param != "" {
|
||||
body, _ = sjson.SetBytes(body, "error.param", param)
|
||||
}
|
||||
if strings.TrimSpace(gjson.GetBytes(body, "error.message").String()) == "" {
|
||||
if code != "" {
|
||||
body, _ = sjson.SetBytes(body, "error.message", code)
|
||||
} else if errorType != "" {
|
||||
body, _ = sjson.SetBytes(body, "error.message", errorType)
|
||||
}
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
func codexTerminalErrorIsContextLength(body []byte) bool {
|
||||
errorCode := strings.ToLower(strings.TrimSpace(gjson.GetBytes(body, "error.code").String()))
|
||||
message := strings.ToLower(strings.TrimSpace(gjson.GetBytes(body, "error.message").String()))
|
||||
return errorCode == "context_length_exceeded" ||
|
||||
errorCode == "context_too_large" ||
|
||||
strings.Contains(message, "context window") ||
|
||||
strings.Contains(message, "context length") ||
|
||||
strings.Contains(message, "too many tokens")
|
||||
}
|
||||
|
||||
func newCodexStatusErr(statusCode int, body []byte) statusErr {
|
||||
errCode := statusCode
|
||||
if isCodexModelCapacityError(body) || isCodexUsageLimitError(body) {
|
||||
errCode = http.StatusTooManyRequests
|
||||
}
|
||||
body = classifyCodexStatusError(errCode, body)
|
||||
err := statusErr{code: errCode, msg: string(body)}
|
||||
if retryAfter := parseCodexRetryAfter(errCode, body, time.Now()); retryAfter != nil {
|
||||
err.retryAfter = retryAfter
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func classifyCodexStatusError(statusCode int, body []byte) []byte {
|
||||
code, errType, ok := codexStatusErrorClassification(statusCode, body)
|
||||
if !ok {
|
||||
return body
|
||||
}
|
||||
message := gjson.GetBytes(body, "error.message").String()
|
||||
if message == "" {
|
||||
message = gjson.GetBytes(body, "message").String()
|
||||
}
|
||||
if message == "" {
|
||||
message = strings.TrimSpace(string(body))
|
||||
}
|
||||
if message == "" {
|
||||
message = http.StatusText(statusCode)
|
||||
}
|
||||
out := []byte(`{"error":{}}`)
|
||||
out, _ = sjson.SetBytes(out, "error.message", message)
|
||||
out, _ = sjson.SetBytes(out, "error.type", errType)
|
||||
out, _ = sjson.SetBytes(out, "error.code", code)
|
||||
return out
|
||||
}
|
||||
|
||||
func codexStatusErrorClassification(statusCode int, body []byte) (code string, errType string, ok bool) {
|
||||
errorMessage := strings.ToLower(strings.TrimSpace(gjson.GetBytes(body, "error.message").String()))
|
||||
if errorMessage == "" {
|
||||
errorMessage = strings.ToLower(strings.TrimSpace(gjson.GetBytes(body, "message").String()))
|
||||
}
|
||||
lower := strings.ToLower(strings.TrimSpace(string(body)))
|
||||
upstreamCode := strings.ToLower(strings.TrimSpace(gjson.GetBytes(body, "error.code").String()))
|
||||
upstreamType := strings.ToLower(strings.TrimSpace(gjson.GetBytes(body, "error.type").String()))
|
||||
isInvalidRequest := upstreamType == "" || upstreamType == "invalid_request_error"
|
||||
|
||||
switch {
|
||||
case statusCode == http.StatusRequestEntityTooLarge || upstreamCode == "context_length_exceeded" || upstreamCode == "context_too_large" || isInvalidRequest && (strings.Contains(errorMessage, "context length") || strings.Contains(errorMessage, "context_length") || strings.Contains(errorMessage, "maximum context") || strings.Contains(errorMessage, "too many tokens")):
|
||||
return "context_too_large", "invalid_request_error", true
|
||||
case strings.Contains(lower, "invalid signature in thinking block") || strings.Contains(lower, "invalid_encrypted_content"):
|
||||
return "thinking_signature_invalid", "invalid_request_error", true
|
||||
case upstreamCode == "previous_response_not_found" || strings.Contains(lower, "previous_response_not_found") || strings.Contains(lower, "previous_response_id") && strings.Contains(lower, "not found"):
|
||||
return "previous_response_not_found", "invalid_request_error", true
|
||||
case statusCode == http.StatusUnauthorized || upstreamType == "authentication_error" || upstreamCode == "invalid_api_key" || strings.Contains(lower, "invalid or expired token") || strings.Contains(lower, "refresh_token_reused"):
|
||||
return "auth_unavailable", "authentication_error", true
|
||||
default:
|
||||
return "", "", false
|
||||
}
|
||||
}
|
||||
|
||||
func isCodexModelCapacityError(errorBody []byte) bool {
|
||||
if len(errorBody) == 0 {
|
||||
return false
|
||||
}
|
||||
candidates := []string{
|
||||
gjson.GetBytes(errorBody, "error.message").String(),
|
||||
gjson.GetBytes(errorBody, "message").String(),
|
||||
string(errorBody),
|
||||
}
|
||||
for _, candidate := range candidates {
|
||||
lower := strings.ToLower(strings.TrimSpace(candidate))
|
||||
if lower == "" {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(lower, "selected model is at capacity") ||
|
||||
strings.Contains(lower, "model is at capacity. please try a different model") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// isCodexUsageLimitError reports whether the error body represents a Codex
|
||||
// quota/plan-limit exhaustion (error.type == "usage_limit_reached"). This is the
|
||||
// signal Codex emits when a credential's usage quota is depleted, and it carries
|
||||
// reset timing (resets_at/resets_in_seconds) parsed by parseCodexRetryAfter.
|
||||
// Transient per-minute rate limits (rate_limit_error/rate_limit_exceeded) are
|
||||
// intentionally excluded, as they should be retried rather than cooled down.
|
||||
func isCodexUsageLimitError(errorBody []byte) bool {
|
||||
if len(errorBody) == 0 {
|
||||
return false
|
||||
}
|
||||
candidates := []string{
|
||||
gjson.GetBytes(errorBody, "error.type").String(),
|
||||
gjson.GetBytes(errorBody, "type").String(),
|
||||
}
|
||||
for _, candidate := range candidates {
|
||||
if strings.EqualFold(strings.TrimSpace(candidate), "usage_limit_reached") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func parseCodexRetryAfter(statusCode int, errorBody []byte, now time.Time) *time.Duration {
|
||||
if statusCode != http.StatusTooManyRequests || len(errorBody) == 0 {
|
||||
return nil
|
||||
}
|
||||
if strings.TrimSpace(gjson.GetBytes(errorBody, "error.type").String()) != "usage_limit_reached" {
|
||||
return nil
|
||||
}
|
||||
if resetsAt := gjson.GetBytes(errorBody, "error.resets_at").Int(); resetsAt > 0 {
|
||||
resetAtTime := time.Unix(resetsAt, 0)
|
||||
if resetAtTime.After(now) {
|
||||
retryAfter := resetAtTime.Sub(now)
|
||||
return &retryAfter
|
||||
}
|
||||
}
|
||||
if resetsInSeconds := gjson.GetBytes(errorBody, "error.resets_in_seconds").Int(); resetsInSeconds > 0 {
|
||||
retryAfter := time.Duration(resetsInSeconds) * time.Second
|
||||
return &retryAfter
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// codexBootstrapMaxBufferedEvents bounds how many handshake metadata events may be held
|
||||
// back while probing for an upstream rejection embedded in an HTTP 200 stream. The websocket
|
||||
// transport prefixes response events with codex.response.metadata and codex.rate_limits frames,
|
||||
// so the limit must comfortably exceed the four handshake frames observed in practice. Once the
|
||||
// limit is reached the stream is released and the original unbuffered semantics apply.
|
||||
const codexBootstrapMaxBufferedEvents = 16
|
||||
|
||||
// isCodexHandshakeMetadataEvent reports whether an event carries no generated output and is
|
||||
// therefore safe to hold back before the downstream response headers are committed. Keeping a type
|
||||
// allow-list rather than a fixed event count matters for the websocket transport, where the
|
||||
// handshake frames arrive before response.created and would otherwise exhaust a small counter
|
||||
// before the rejection event is seen.
|
||||
func isCodexHandshakeMetadataEvent(eventType string) bool {
|
||||
switch eventType {
|
||||
case "response.created", "response.in_progress", "codex.rate_limits", "codex.response.metadata":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// newCodexBootstrapOverloadErr reports a buffered overload rejection with its real status.
|
||||
//
|
||||
// The status is deliberately produced here instead of in codexTerminalFailureStatus: that mapping
|
||||
// is shared with the unbuffered path, where the rejection is delivered in-stream and a status
|
||||
// change would alter cooldown classification and retry-after parsing for everyone. Keeping 503
|
||||
// scoped to this path means disabling the feature restores the previous behaviour exactly.
|
||||
func newCodexBootstrapOverloadErr(body []byte) statusErr {
|
||||
return newCodexStatusErr(http.StatusServiceUnavailable, body)
|
||||
}
|
||||
|
||||
// isCodexOverloadBootstrapFailure reports whether a terminal failure delivered inside an HTTP 200
|
||||
// stream is a transient capacity rejection that a different credential may be able to serve.
|
||||
// Only these failures justify replacing the whole attempt during bootstrap; every other terminal
|
||||
// failure keeps the original in-stream delivery semantics so downstream behaviour is unchanged.
|
||||
func isCodexOverloadBootstrapFailure(body []byte) bool {
|
||||
errorType := strings.ToLower(strings.TrimSpace(gjson.GetBytes(body, "error.type").String()))
|
||||
errorCode := strings.ToLower(strings.TrimSpace(gjson.GetBytes(body, "error.code").String()))
|
||||
switch {
|
||||
case errorType == "service_unavailable_error", errorCode == "server_is_overloaded":
|
||||
return true
|
||||
case errorType == "rate_limit_error", errorCode == "rate_limit_exceeded":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue