orchard

 1package skills
 2
 3import (
 4	"bytes"
 5	"context"
 6	"encoding/json"
 7	"fmt"
 8	"net/http"
 9	"os"
10	"strings"
11	"testing"
12	"time"
13)
14
15// liveModel is a small stand-in for the site's own client, so this package can
16// be evaluated against the real model without importing main. It speaks the
17// same OpenAI compatible endpoint and sets the one option that matters, since
18// a thinking model with thinking left on returns an empty content beside a full
19// reasoning_content and looks like a silent failure.
20type liveLLM struct {
21	base   string
22	client *http.Client
23}
24
25func liveModel(t *testing.T) *liveLLM {
26	t.Helper()
27	base := os.Getenv("LLM_URL")
28	if base == "" {
29		base = "http://orchard-search-llm:8091"
30	}
31	return &liveLLM{
32		base:   strings.TrimRight(base, "/"),
33		client: &http.Client{Timeout: 2 * time.Minute},
34	}
35}
36
37func (l *liveLLM) Structured(ctx context.Context, system, user string, maxTokens int, schema any, out any) error {
38	raw, err := json.Marshal(schema)
39	if err != nil {
40		return err
41	}
42	body, _ := json.Marshal(map[string]any{
43		"model":       "local",
44		"temperature": 0.2,
45		"max_tokens":  maxTokens,
46		"messages": []map[string]string{
47			{"role": "system", "content": system},
48			{"role": "user", "content": user},
49		},
50		"chat_template_kwargs": map[string]any{"enable_thinking": false},
51		"response_format": map[string]any{
52			"type": "json_schema",
53			"json_schema": map[string]any{
54				"name": "response", "strict": true, "schema": json.RawMessage(raw),
55			},
56		},
57	})
58	req, err := http.NewRequestWithContext(ctx, "POST", l.base+"/v1/chat/completions", bytes.NewReader(body))
59	if err != nil {
60		return err
61	}
62	req.Header.Set("Content-Type", "application/json")
63	resp, err := l.client.Do(req)
64	if err != nil {
65		return err
66	}
67	defer resp.Body.Close()
68
69	var r struct {
70		Choices []struct {
71			Message struct {
72				Content   string `json:"content"`
73				Reasoning string `json:"reasoning_content"`
74			} `json:"message"`
75		} `json:"choices"`
76	}
77	if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
78		return err
79	}
80	if len(r.Choices) == 0 {
81		return fmt.Errorf("no choices")
82	}
83	text := strings.TrimSpace(r.Choices[0].Message.Content)
84	if text == "" {
85		return fmt.Errorf("empty content, reasoning was %q", r.Choices[0].Message.Reasoning)
86	}
87	if i := strings.Index(text, "{"); i > 0 {
88		text = text[i:]
89	}
90	return json.Unmarshal([]byte(text), out)
91}