feat(llm): openai-compatible vision and transcription provider

internal/llm/openai.go

@@ -0,0 +1,145 @@
+package llm
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+)
+
+type openAIClient struct {
+	http       *http.Client
+	base       string
+	key        string
+	model      string
+	audioModel string
+	maxTokens  int
+}
+
+func (c *openAIClient) modelOr(o Options) string {
+	if o.Model != "" {
+		return o.Model
+	}
+	return c.model
+}
+
+func (c *openAIClient) Vision(ctx context.Context, prompt string, images []Image, opts Options) (string, error) {
+	content := []map[string]any{{"type": "text", "text": prompt}}
+	for _, img := range images {
+		url := fmt.Sprintf("data:%s;base64,%s", img.MediaType, base64.StdEncoding.EncodeToString(img.Data))
+		content = append(content, map[string]any{"type": "image_url", "image_url": map[string]any{"url": url}})
+	}
+	maxTok := c.maxTokens
+	if opts.MaxTokens > 0 {
+		maxTok = opts.MaxTokens
+	}
+	reqBody := map[string]any{
+		"model":       c.modelOr(opts),
+		"max_tokens":  maxTok,
+		"temperature": opts.Temperature,
+		"messages":    []map[string]any{{"role": "user", "content": content}},
+	}
+	var out struct {
+		Choices []struct {
+			Message struct {
+				Content string `json:"content"`
+			} `json:"message"`
+		} `json:"choices"`
+	}
+	if err := c.postJSON(ctx, "/v1/chat/completions", reqBody, &out); err != nil {
+		return "", err
+	}
+	if len(out.Choices) == 0 {
+		return "", fmt.Errorf("openai: empty choices")
+	}
+	return out.Choices[0].Message.Content, nil
+}
+
+func (c *openAIClient) Transcribe(ctx context.Context, audio []byte, filename string, opts Options) (string, error) {
+	model := c.audioModel
+	if opts.Model != "" {
+		model = opts.Model
+	}
+	if model == "" {
+		model = c.model
+	}
+	var buf bytes.Buffer
+	mw := multipart.NewWriter(&buf)
+	if err := mw.WriteField("model", model); err != nil {
+		return "", err
+	}
+	if opts.Language != "" {
+		if err := mw.WriteField("language", opts.Language); err != nil {
+			return "", err
+		}
+	}
+	fw, err := mw.CreateFormFile("file", filename)
+	if err != nil {
+		return "", err
+	}
+	if _, err := fw.Write(audio); err != nil {
+		return "", err
+	}
+	if err := mw.Close(); err != nil {
+		return "", err
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.base+"/v1/audio/transcriptions", &buf)
+	if err != nil {
+		return "", err
+	}
+	req.Header.Set("Content-Type", mw.FormDataContentType())
+	if c.key != "" {
+		req.Header.Set("Authorization", "Bearer "+c.key)
+	}
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer func() { _ = resp.Body.Close() }()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+	if resp.StatusCode >= 300 {
+		return "", fmt.Errorf("openai transcribe HTTP %d: %s", resp.StatusCode, string(body))
+	}
+	var out struct {
+		Text string `json:"text"`
+	}
+	if err := json.Unmarshal(body, &out); err != nil {
+		return "", err
+	}
+	return out.Text, nil
+}
+
+func (c *openAIClient) postJSON(ctx context.Context, path string, body, out any) error {
+	b, err := json.Marshal(body)
+	if err != nil {
+		return err
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.base+path, bytes.NewReader(b))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	if c.key != "" {
+		req.Header.Set("Authorization", "Bearer "+c.key)
+	}
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = resp.Body.Close() }()
+	raw, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode >= 300 {
+		return fmt.Errorf("openai HTTP %d: %s", resp.StatusCode, string(raw))
+	}
+	return json.Unmarshal(raw, out)
+}

internal/llm/openai_test.go

@@ -0,0 +1,61 @@
+package llm
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestOpenAIVision(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/v1/chat/completions" {
+			t.Errorf("path %s", r.URL.Path)
+		}
+		if got := r.Header.Get("Authorization"); got != "Bearer key" {
+			t.Errorf("auth %q", got)
+		}
+		var body map[string]any
+		_ = json.NewDecoder(r.Body).Decode(&body)
+		raw, _ := json.Marshal(body)
+		if !strings.Contains(string(raw), "data:image/png;base64,AQ==") {
+			t.Errorf("image not embedded: %s", raw)
+		}
+		_, _ = io.WriteString(w, `{"choices":[{"message":{"content":"ab12"}}]}`)
+	}))
+	defer srv.Close()
+
+	c := &openAIClient{http: &http.Client{Timeout: 5 * time.Second}, base: srv.URL, key: "key", model: "gpt-4o", maxTokens: 100}
+	got, err := c.Vision(context.Background(), "read", []Image{{MediaType: "image/png", Data: []byte{1}}}, Options{})
+	if err != nil || got != "ab12" {
+		t.Fatalf("got %q err %v", got, err)
+	}
+}
+
+const maxMultipartMem = 1 << 20 // 1 MiB — bounded, used only in tests
+
+func TestOpenAITranscribe(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/v1/audio/transcriptions" {
+			t.Errorf("path %s", r.URL.Path)
+		}
+		if err := r.ParseMultipartForm(maxMultipartMem); err != nil { //nolint:gosec // G120: httptest server, bounded by maxMultipartMem constant, not a real HTTP handler
+			t.Errorf("parse multipart: %v", err)
+		}
+		if r.FormValue("model") != "whisper-1" {
+			t.Errorf("model %q", r.FormValue("model"))
+		}
+		_, _ = io.WriteString(w, `{"text":"the secret is 42"}`)
+	}))
+	defer srv.Close()
+
+	c := &openAIClient{http: &http.Client{Timeout: 5 * time.Second}, base: srv.URL, key: "key", model: "gpt-4o", audioModel: "whisper-1"}
+	got, err := c.Transcribe(context.Background(), []byte("RIFF"), "a.wav", Options{})
+	if err != nil || got != "the secret is 42" {
+		t.Fatalf("got %q err %v", got, err)
+	}
+}