orchard

 1package main
 2
 3import (
 4	"context"
 5	"os"
 6	"testing"
 7	"time"
 8
 9	"chat.bythewood.me/tools"
10)
11
12// Drives the real gate against the real model and snapshot.
13func TestGateLive(t *testing.T) {
14	if os.Getenv("PROMPT_SMOKE") == "" {
15		t.Skip("set PROMPT_SMOKE=1")
16	}
17	if base := os.Getenv("WIKI_URL"); base != "" {
18		tools.WikiBase = base
19	}
20	llm := NewLLM(env("LLM_URL", "http://orchard-llm:8000"), env("LLM_MODEL", "local"), os.Getenv("LLM_KEY"))
21	eng := NewEngine(llm, "Ornith 1.5 9B")
22
23	cases := []struct {
24		name, question, draft string
25		wantBack              bool
26	}{
27		{"wrong origin", "what is a goodyear welt",
28			"A goodyear welt is named after the Goodyear tyre company, which invented it in 1920 to make shoes resoleable.", true},
29		{"right origin", "what is a goodyear welt",
30			"A goodyear welt is a strip of leather running around the outsole. The machine behind it was invented in 1862 by Auguste Destouy and improved by Daniel Mills, both working for Charles Goodyear Jr.", false},
31		{"wrong definition", "what is postgresql",
32			"PostgreSQL is a proprietary NoSQL document database made by Oracle, and it does not support SQL or transactions.", true},
33		{"right definition", "what is postgresql",
34			"PostgreSQL is a free and open-source relational database management system that emphasises extensibility and SQL compliance, with ACID transactions.", false},
35	}
36
37	for _, c := range cases {
38		bg := eng.background(context.Background(), c.question)
39		if bg == "" {
40			t.Errorf("%s: no background found for %q", c.name, c.question)
41			continue
42		}
43		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
44		nudge, _ := eng.gate(ctx, c.question, c.draft, nil, nil, func(Event) {})
45		cancel()
46		sentBack := nudge != ""
47		status := "let through"
48		if sentBack {
49			status = "sent back: " + nudge
50		}
51		if sentBack != c.wantBack {
52			t.Errorf("%s: sentBack = %v, want %v (%s)", c.name, sentBack, c.wantBack, status)
53		} else {
54			t.Logf("%s: %s", c.name, status)
55		}
56	}
57}