orchard

  1package main
  2
  3import (
  4	"context"
  5	"encoding/json"
  6	"fmt"
  7	"io"
  8	"net/http"
  9	"net/http/httptest"
 10	"strings"
 11	"testing"
 12
 13	"chat.bythewood.me/tools"
 14)
 15
 16// A model server that answers the freshness question with whatever the test
 17// wants, and records what it was sent.
 18func fakeJudge(t *testing.T, needsFresh bool, query string) (*LLM, *[]chatReq) {
 19	t.Helper()
 20	var got []chatReq
 21	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 22		body, _ := io.ReadAll(r.Body)
 23		var req chatReq
 24		_ = json.Unmarshal(body, &req)
 25		got = append(got, req)
 26		payload, _ := json.Marshal(freshness{NeedsFresh: needsFresh, Query: query})
 27		w.Header().Set("Content-Type", "application/json")
 28		fmt.Fprintf(w, `{"choices":[{"message":{"content":%s}}]}`, mustJSONString(string(payload)))
 29	}))
 30	t.Cleanup(srv.Close)
 31	return NewLLM(srv.URL, "local", ""), &got
 32}
 33
 34func mustJSONString(s string) string {
 35	b, _ := json.Marshal(s)
 36	return string(b)
 37}
 38
 39// The failure this was written for: a draft that sounds like an answer, on a
 40// question the weights cannot answer, with nothing fetched.
 41func TestAFreshQuestionWithNoToolCallIsSentBack(t *testing.T) {
 42	llm, _ := fakeJudge(t, true, "big news today")
 43	e := NewEngine(llm, "test")
 44
 45	nudge, _ := e.gate(context.Background(), "any big news today?",
 46		"No big news today. It is a quiet Monday.", nil, nil, func(Event) {})
 47
 48	if nudge == "" {
 49		t.Fatal("a confident draft on a current question was let through")
 50	}
 51	if !strings.Contains(nudge, "big news today") {
 52		t.Errorf("the nudge carries no query to search for: %q", nudge)
 53	}
 54}
 55
 56// The check is about the question, so a turn that already fetched something is
 57// not sent back for asking about today.
 58func TestAFreshQuestionThatFetchedSomethingIsNotSentBack(t *testing.T) {
 59	llm, sent := fakeJudge(t, true, "big news today")
 60	e := NewEngine(llm, "test")
 61
 62	used := []tools.Result{{Name: "web_search", Content: map[string]any{"results": "something"}}}
 63	_, _ = e.gate(context.Background(), "any big news today?", "Here is what happened.", nil, used, func(Event) {})
 64
 65	for _, req := range *sent {
 66		for _, m := range req.Messages {
 67			if m.Role == RoleSystem && strings.Contains(m.Content, "needs_fresh") {
 68				t.Fatal("the freshness check ran on a turn that had already fetched")
 69			}
 70		}
 71	}
 72}
 73
 74// Turning the model loose again after a nudge is what let the old loop run out
 75// of attempts, so the round after one has to be a forced call.
 76func TestTheCallAfterANudgeRequiresATool(t *testing.T) {
 77	var choices []string
 78	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 79		body, _ := io.ReadAll(r.Body)
 80		var req chatReq
 81		_ = json.Unmarshal(body, &req)
 82		choices = append(choices, req.ToolChoice)
 83		w.Header().Set("Content-Type", "application/json")
 84		fmt.Fprint(w, `{"choices":[{"message":{"content":"ok"}}]}`)
 85	}))
 86	defer srv.Close()
 87	l := NewLLM(srv.URL, "local", "")
 88
 89	schemas := []map[string]any{{"type": "function"}}
 90	if _, _, err := l.CompleteStats(context.Background(), nil, schemas, 10); err != nil {
 91		t.Fatal(err)
 92	}
 93	if _, _, err := l.CompleteRequiringTool(context.Background(), nil, schemas, 10); err != nil {
 94		t.Fatal(err)
 95	}
 96	want := []string{"auto", "required"}
 97	for i, w := range want {
 98		if choices[i] != w {
 99			t.Errorf("call %d sent tool_choice %q, want %q", i, choices[i], w)
100		}
101	}
102}
103
104// With no tools on the table there is nothing to require, and asking for one
105// anyway is a request llama.cpp cannot satisfy.
106func TestNoToolsMeansNoToolChoice(t *testing.T) {
107	var seen chatReq
108	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
109		body, _ := io.ReadAll(r.Body)
110		_ = json.Unmarshal(body, &seen)
111		w.Header().Set("Content-Type", "application/json")
112		fmt.Fprint(w, `{"choices":[{"message":{"content":"ok"}}]}`)
113	}))
114	defer srv.Close()
115
116	l := NewLLM(srv.URL, "local", "")
117	if _, _, err := l.CompleteRequiringTool(context.Background(), nil, nil, 10); err != nil {
118		t.Fatal(err)
119	}
120	if seen.ToolChoice != "" {
121		t.Errorf("tool_choice = %q with no tools offered", seen.ToolChoice)
122	}
123}
124
125// A ticker or an abbreviation is too short to name on its own, so the answer
126// goes to the titler as well. "VXUS" alone came back "Video game streaming
127// service".
128func TestTheTitlerIsGivenTheAnswerAsWellAsTheQuestion(t *testing.T) {
129	var seen chatReq
130	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
131		body, _ := io.ReadAll(r.Body)
132		_ = json.Unmarshal(body, &seen)
133		w.Header().Set("Content-Type", "application/json")
134		fmt.Fprint(w, `{"choices":[{"message":{"content":"Vanguard Total International Stock ETF"}}]}`)
135	}))
136	defer srv.Close()
137
138	c := NewCompactor(NewLLM(srv.URL, "local", ""), 65536)
139	got := c.Title(context.Background(), "VXUS", "VXUS is the Vanguard Total International Stock ETF.")
140	if got != "Vanguard Total International Stock ETF" {
141		t.Errorf("title = %q", got)
142	}
143	var prompt string
144	for _, m := range seen.Messages {
145		if m.Role == RoleUser {
146			prompt = m.Content
147		}
148	}
149	if !strings.Contains(prompt, "Vanguard") {
150		t.Errorf("the answer never reached the titler: %q", prompt)
151	}
152	if !strings.Contains(prompt, "VXUS") {
153		t.Errorf("the question never reached the titler: %q", prompt)
154	}
155}
156
157// A turn with no answer to hand still has to produce a name rather than an
158// empty prompt section.
159func TestTheTitlerWorksWithNoAnswer(t *testing.T) {
160	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
161		w.Header().Set("Content-Type", "application/json")
162		fmt.Fprint(w, `{"choices":[{"message":{"content":"Sheet pan dinners"}}]}`)
163	}))
164	defer srv.Close()
165
166	c := NewCompactor(NewLLM(srv.URL, "local", ""), 65536)
167	if got := c.Title(context.Background(), "easy weeknight meals", ""); got != "Sheet pan dinners" {
168		t.Errorf("title = %q", got)
169	}
170}