orchard

  1// Citations. The model points at a source by number and never writes the
  2// address, so a link on this page is one a tool really fetched rather than one
  3// a small model half remembered. It wrote three dead hostnames under an answer
  4// before this existed.
  5//
  6// The number it wrote is then repaired here rather than trusted, and a sentence
  7// it left uncited gets one if the wording clearly came from a source. That is
  8// search's shape without search's cost: no entailment call per sentence, which
  9// is the part that makes an answer there take a minute.
 10package main
 11
 12import (
 13	"fmt"
 14	"net/url"
 15	"regexp"
 16	"strconv"
 17	"strings"
 18
 19	"chat.bythewood.me/tools"
 20)
 21
 22// How many sources the model is offered. Pages it fetched go in first, since
 23// those are what it actually read, and search hits fill the rest.
 24const (
 25	maxSources     = 14
 26	maxNewsSources = 32
 27)
 28
 29// Source is one page this turn retrieved. Text never leaves the process, it is
 30// only what a sentence is matched against.
 31type Source struct {
 32	N     int    `json:"n"`
 33	URL   string `json:"url"`
 34	Title string `json:"title"`
 35	Site  string `json:"site"`
 36
 37	Text   string          `json:"-"`
 38	tokens map[string]bool `json:"-"`
 39}
 40
 41// collectSources numbers everything the turn read. A url seen as a search hit
 42// and then fetched is one source, keeping the hit's title and the page's text.
 43func collectSources(used []tools.Result) []Source {
 44	type entry struct {
 45		url, title, text string
 46		fetched          bool
 47	}
 48	var order []*entry
 49	seen := map[string]*entry{}
 50	add := func(raw, title, text string, fetched bool) {
 51		u := tidyURL(raw)
 52		if u == "" {
 53			return
 54		}
 55		e, ok := seen[u]
 56		if !ok {
 57			e = &entry{url: u}
 58			seen[u] = e
 59			order = append(order, e)
 60		}
 61		if e.title == "" {
 62			e.title = strings.TrimSpace(title)
 63		}
 64		if len(text) > len(e.text) {
 65			e.text = text
 66		}
 67		e.fetched = e.fetched || fetched
 68	}
 69
 70	for _, r := range used {
 71		if r.Err != "" {
 72			continue
 73		}
 74		m, _ := r.Content.(map[string]any)
 75		if m == nil {
 76			continue
 77		}
 78		switch r.Name {
 79		case tools.WebFetch.Name:
 80			add(asString(m["url"]), "", asString(m["text"]), true)
 81		case tools.WebSearch.Name:
 82			hits, _ := m["results"].([]tools.SearchHit)
 83			for _, h := range hits {
 84				add(h.URL, h.Title, h.Title+" "+h.Snippet, false)
 85			}
 86		case tools.News.Name:
 87			// A rundown named its publishers in the prose and linked none of
 88			// them, so the one answer most worth clicking through was the one
 89			// with nothing to click. Each headline is a page that was read.
 90			sections, _ := m["sections"].([]tools.NewsSection)
 91			for _, sec := range sections {
 92				for _, it := range sec.Items {
 93					add(it.URL, it.Headline, it.Headline+" "+it.Summary, false)
 94				}
 95			}
 96		}
 97	}
 98
 99	// A rundown is twenty to thirty items and every one of them is a page worth
100	// a link, where an ordinary turn reads a handful. Capping both the same way
101	// left most of the news with nothing to click.
102	cap := maxSources
103	for _, r := range used {
104		if r.Name == tools.News.Name && r.Err == "" {
105			cap = maxNewsSources
106			break
107		}
108	}
109
110	var out []Source
111	for _, want := range []bool{true, false} {
112		for _, e := range order {
113			if e.fetched != want || len(out) >= cap {
114				continue
115			}
116			s := Source{
117				N: len(out) + 1, URL: e.url, Title: e.title,
118				Site: siteOf(e.url), Text: e.text,
119			}
120			if s.Title == "" {
121				s.Title = s.Site
122			}
123			s.tokens = tokenSet(s.Text + " " + s.Title)
124			out = append(out, s)
125		}
126	}
127	return out
128}
129
130// prompt is the numbered list the final turn is handed. Titles are trimmed hard
131// because a page title runs to a headline and a sentence of standfirst, and the
132// model only needs enough to tell one source from another.
133func sourcePrompt(srcs []Source) string {
134	if len(srcs) == 0 {
135		return ""
136	}
137	var b strings.Builder
138	b.WriteString("\n\nSources, by number:\n")
139	for _, s := range srcs {
140		fmt.Fprintf(&b, "[%d] %s, %s\n", s.N, s.Site, trimLine(s.Title, 90))
141	}
142	b.WriteString("\nEnd a sentence that came from one of these with its number, like [2]. " +
143		"Write the number and never the address, and do not list the sources at the end, since they are shown under your answer.")
144	return b.String()
145}
146
147var citeMark = regexp.MustCompile(`\[(\d{1,3})\]`)
148
149// A marker the model put after the full stop belongs to the sentence in front
150// of it, so it moves inside before anything is split.
151var markAfterStop = regexp.MustCompile(`([.!?:])((?:\s*\[\d{1,3}\])+)`)
152
153// attach repairs the citations in a block of markdown. A number with no source
154// behind it is dropped, a sentence that cites nothing is matched against what
155// the turn read, and every marker ends up in the same place.
156func attach(text string, srcs []Source) string {
157	known := make(map[int]bool, len(srcs))
158	for _, s := range srcs {
159		known[s.N] = true
160	}
161	lines := strings.Split(text, "\n")
162	fenced := false
163	for i, line := range lines {
164		t := strings.TrimSpace(line)
165		if strings.HasPrefix(t, "```") || strings.HasPrefix(t, "~~~") {
166			fenced = !fenced
167			continue
168		}
169		// A pill in a heading, a quote or a table cell is in the way rather
170		// than in the margin, and a fence is code somebody is about to copy.
171		if fenced || t == "" || strings.HasPrefix(t, "#") ||
172			strings.HasPrefix(t, ">") || strings.HasPrefix(t, "|") {
173			continue
174		}
175		lines[i] = citeLine(line, srcs, known)
176	}
177	return strings.Join(lines, "\n")
178}
179
180func citeLine(line string, srcs []Source, known map[int]bool) string {
181	indent := line[:len(line)-len(strings.TrimLeft(line, " \t"))]
182	body := line[len(indent):]
183	marker := ""
184	if item, ok := listItem(body); ok {
185		marker, body = body[:len(body)-len(item)], item
186	}
187	body = markAfterStop.ReplaceAllString(body, "$2$1")
188
189	var out []string
190	for _, s := range sentences(body) {
191		if c := citeSentence(s, srcs, known); c != "" {
192			out = append(out, c)
193		}
194	}
195	if len(out) == 0 {
196		return line
197	}
198	return indent + marker + strings.Join(out, " ")
199}
200
201// citeSentence keeps the numbers that name a real source, drops the rest, and
202// looks one up when the model gave none.
203func citeSentence(s string, srcs []Source, known map[int]bool) string {
204	var ids []int
205	seen := map[int]bool{}
206	// Outside code only. In a code span [1] is an index, and both reading it
207	// as a citation and deleting it break something a reader is about to copy.
208	body := eachOutsideCode(s, func(part string) string {
209		for _, m := range citeMark.FindAllStringSubmatch(part, -1) {
210			n, _ := strconv.Atoi(m[1])
211			if known[n] && !seen[n] {
212				seen[n] = true
213				ids = append(ids, n)
214			}
215		}
216		return citeMark.ReplaceAllString(part, "")
217	})
218	body = spaceBeforePunct.ReplaceAllString(spaceRun.ReplaceAllString(body, " "), "$1")
219	body = strings.TrimSpace(body)
220	if body == "" {
221		return ""
222	}
223	if len(ids) == 0 {
224		if n := bestSource(body, srcs); n > 0 {
225			ids = append(ids, n)
226		}
227	}
228	var b strings.Builder
229	b.WriteString(body)
230	for _, n := range ids {
231		if !endsSentence(body) {
232			b.WriteByte(' ')
233		}
234		fmt.Fprintf(&b, "[%d]", n)
235	}
236	return b.String()
237}
238
239var (
240	spaceRun = regexp.MustCompile(`[ \t]{2,}`)
241	// Lifting a marker out from in front of the full stop leaves the space it
242	// was sitting on.
243	spaceBeforePunct = regexp.MustCompile(` +([,.;:!?])`)
244)
245
246func endsSentence(s string) bool {
247	if s == "" {
248		return false
249	}
250	switch s[len(s)-1] {
251	case '.', '!', '?', ':', ')', '"', '\'', '*', ']':
252		return true
253	}
254	return false
255}
256
257// How much of a sentence has to be in a source before it is cited, and how many
258// of its distinctive words have to be. The share alone is not enough, since two
259// pages about one story share most of their ordinary vocabulary and the thing
260// that tells them apart is the figure or the name only one of them carries.
261const (
262	citeFloor      = 0.5
263	minDistinctive = 2
264)
265
266// bestSource picks what a sentence came from, or 0 when nothing is close
267// enough. Silence is the right answer here: a pill on a sentence the page does
268// not support is worse than no pill, since it is read as a check that passed.
269func bestSource(sentence string, srcs []Source) int {
270	words := tokenSet(sentence)
271	if len(words) < 4 {
272		return 0
273	}
274	rare := distinctive(sentence)
275	best, bestScore, bestRare := 0, 0.0, 0
276	for _, s := range srcs {
277		var hit, total float64
278		matched := 0
279		for w := range words {
280			weight := 1.0
281			if rare[w] {
282				weight = 3
283			}
284			total += weight
285			if s.tokens[w] {
286				hit += weight
287				if rare[w] {
288					matched++
289				}
290			}
291		}
292		if total == 0 {
293			continue
294		}
295		score := hit / total
296		if matched < minDistinctive || score < citeFloor {
297			continue
298		}
299		if matched > bestRare || (matched == bestRare && score > bestScore) {
300			best, bestScore, bestRare = s.N, score, matched
301		}
302	}
303	return best
304}
305
306var wordRun = regexp.MustCompile(`[a-z0-9]+`)
307
308// tokenSet is the words worth matching on. Commas go first so 48,000 in a
309// sentence and 48,000 on the page are the same token.
310func tokenSet(s string) map[string]bool {
311	s = strings.ToLower(strings.ReplaceAll(s, ",", ""))
312	out := map[string]bool{}
313	for _, w := range wordRun.FindAllString(s, -1) {
314		if stopword[w] || (len(w) < 3 && !hasDigit(w)) {
315			continue
316		}
317		out[w] = true
318	}
319	return out
320}
321
322// distinctive is the half of a sentence that identifies which page it came
323// from: the figures, the dates and the names. A word is a name here if it is
324// capitalised anywhere but the opening of the sentence.
325func distinctive(sentence string) map[string]bool {
326	out := map[string]bool{}
327	for _, f := range strings.Fields(strings.ReplaceAll(sentence, ",", "")) {
328		clean := strings.Trim(f, `.,;:!?"'()[]*_`)
329		low := strings.ToLower(strings.Join(wordRun.FindAllString(strings.ToLower(clean), -1), ""))
330		if low == "" || stopword[low] {
331			continue
332		}
333		if hasDigit(low) {
334			out[low] = true
335			continue
336		}
337		// A capital anywhere counts, the opening word included. The stopword
338		// list already covers the ordinary openers, and refusing the first word
339		// loses the name in "Hampshire Police opened an investigation", which
340		// is the whole of what identifies the page it came from.
341		if len(clean) > 2 && clean[0] >= 'A' && clean[0] <= 'Z' {
342			out[low] = true
343		}
344	}
345	return out
346}
347
348func hasDigit(s string) bool {
349	return strings.ContainsAny(s, "0123456789")
350}
351
352// sentences splits a line into the units a citation can hang off. It is crude
353// on purpose, since a split inside an abbreviation costs nothing here. A code
354// span is stepped over, because a full stop in `fmt.Println` is not one.
355func sentences(s string) []string {
356	rs := []rune(s)
357	var out []string
358	start, tick := 0, false
359	for i := 0; i < len(rs); i++ {
360		switch {
361		case rs[i] == '`':
362			tick = !tick
363			continue
364		case tick, rs[i] != '.' && rs[i] != '!' && rs[i] != '?':
365			continue
366		}
367		j := i + 1
368		for j < len(rs) && strings.ContainsRune(`.!?")']*`, rs[j]) {
369			j++
370		}
371		k := j
372		for k < len(rs) && rs[k] == ' ' {
373			k++
374		}
375		if k == j || k >= len(rs) || !opensSentence(rs[k]) {
376			continue
377		}
378		if piece := strings.TrimSpace(string(rs[start:j])); piece != "" {
379			out = append(out, piece)
380		}
381		start, i = k, k-1
382	}
383	if rest := strings.TrimSpace(string(rs[start:])); rest != "" {
384		out = append(out, rest)
385	}
386	return out
387}
388
389func opensSentence(r rune) bool {
390	return r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' ||
391		strings.ContainsRune("*_`\"([", r)
392}
393
394func listItem(t string) (string, bool) {
395	if len(t) > 2 && (t[0] == '-' || t[0] == '*' || t[0] == '+') && t[1] == ' ' {
396		return strings.TrimSpace(t[2:]), true
397	}
398	for i := 0; i < len(t) && i < 3; i++ {
399		if t[i] >= '0' && t[i] <= '9' {
400			continue
401		}
402		if i > 0 && (t[i] == '.' || t[i] == ')') && i+1 < len(t) && t[i+1] == ' ' {
403			return strings.TrimSpace(t[i+2:]), true
404		}
405		break
406	}
407	return "", false
408}
409
410// cited reports which sources an answer actually points at, in number order, so
411// the row under a message lists what was used rather than everything read.
412func cited(text string, srcs []Source) []Source {
413	used := map[int]bool{}
414	for _, m := range citeMark.FindAllStringSubmatch(text, -1) {
415		n, _ := strconv.Atoi(m[1])
416		used[n] = true
417	}
418	var out []Source
419	for _, s := range srcs {
420		if used[s.N] {
421			out = append(out, s)
422		}
423	}
424	return out
425}
426
427// linkCitations turns [3] into an anchor after rendering rather than before,
428// since goldmark drops raw HTML written into the markdown. It substitutes only
429// in text, never inside a tag, and never inside code, where [0] is an index
430// somebody is about to copy.
431func linkCitations(h string, srcs []Source) string {
432	byN := make(map[int]Source, len(srcs))
433	for _, s := range srcs {
434		byN[s.N] = s
435	}
436	anchor := func(text string) string {
437		return citeMark.ReplaceAllStringFunc(text, func(m string) string {
438			n, _ := strconv.Atoi(strings.Trim(m, "[]"))
439			s, ok := byN[n]
440			if !ok {
441				return m
442			}
443			return fmt.Sprintf(
444				`<a class="cite" href="%s" target="_blank" rel="noopener noreferrer" title="%s">%d</a>`,
445				escapeHTML(s.URL), escapeHTML(s.Site+", "+s.Title), n)
446		})
447	}
448
449	var out strings.Builder
450	out.Grow(len(h) + 64)
451	for {
452		open := strings.IndexByte(h, '<')
453		if open < 0 {
454			out.WriteString(anchor(h))
455			break
456		}
457		out.WriteString(anchor(h[:open]))
458		shut := strings.IndexByte(h[open:], '>')
459		if shut < 0 {
460			out.WriteString(h[open:])
461			break
462		}
463		tag := h[open : open+shut+1]
464		out.WriteString(tag)
465		h = h[open+shut+1:]
466		if name, ok := verbatimTag(tag); ok {
467			end := strings.Index(h, "</"+name)
468			if end < 0 {
469				out.WriteString(h)
470				break
471			}
472			out.WriteString(h[:end])
473			h = h[end:]
474		}
475	}
476	return out.String()
477}
478
479func verbatimTag(tag string) (string, bool) {
480	for _, name := range []string{"pre", "code"} {
481		if tag == "<"+name+">" || strings.HasPrefix(tag, "<"+name+" ") {
482			return name, true
483		}
484	}
485	return "", false
486}
487
488func escapeHTML(s string) string {
489	return strings.NewReplacer(
490		"&", "&amp;", "<", "&lt;", ">", "&gt;", `"`, "&#34;", "'", "&#39;",
491	).Replace(s)
492}
493
494// A trailing list of addresses, which is what the model wrote before it had
495// numbers to write instead. The row under the message replaces it, and half of
496// them arrived with no scheme and so were never links at all.
497var (
498	sourceHeading = regexp.MustCompile(`(?i)^\**(sources?|references?|links?)\**\s*:?\s*`)
499	bareAddress   = regexp.MustCompile(`^(?:https?://)?(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z]{2,24}(?:/\S*)?$`)
500)
501
502// dropSourceList removes an address dump from the end of an answer. It works
503// backwards and stops at the first line that is prose, so an answer ending on a
504// real sentence is untouched.
505//
506// A bulleted address only goes when a Sources heading sits above it, because a
507// list of links in the middle of an answer is a list of links the reader asked
508// for and dropping it loses the answer.
509func dropSourceList(text string) string {
510	lines := strings.Split(text, "\n")
511	plain, any := len(lines), len(lines)
512	for i := len(lines) - 1; i >= 0; i-- {
513		t := strings.TrimSpace(lines[i])
514		if t == "" {
515			continue
516		}
517		bullet := false
518		rest := sourceHeading.ReplaceAllString(t, "")
519		if trimmed, ok := listItem(rest); ok {
520			rest, bullet = trimmed, true
521		}
522		if rest == "" && t != rest {
523			// A heading on its own line, with the addresses below it.
524			any, plain = i, i
525			continue
526		}
527		if !onlyAddresses(rest) {
528			break
529		}
530		any = i
531		if !bullet {
532			plain = i
533		}
534	}
535	cut := plain
536	// The heading is what makes a bulleted run a source list rather than part
537	// of the answer.
538	if any < plain {
539		for i := any - 1; i >= 0; i-- {
540			t := strings.TrimSpace(lines[i])
541			if t == "" {
542				continue
543			}
544			if sourceHeading.MatchString(t) && strings.TrimSpace(sourceHeading.ReplaceAllString(t, "")) == "" {
545				cut = i
546			}
547			break
548		}
549	}
550	return strings.TrimRight(strings.Join(lines[:cut], "\n"), "\n")
551}
552
553// onlyAddresses reports whether a line is nothing but addresses.
554func onlyAddresses(s string) bool {
555	fields := strings.FieldsFunc(s, func(r rune) bool { return r == ' ' || r == ',' })
556	if len(fields) == 0 {
557		return false
558	}
559	for _, f := range fields {
560		if f != "and" && !bareAddress.MatchString(f) {
561			return false
562		}
563	}
564	return true
565}
566
567// An address written without a scheme is not a link to anything, and this model
568// writes them that way about half the time. Adding the scheme is what makes
569// goldmark's autolinker see it.
570var schemeless = regexp.MustCompile(`(^|[\s(])((?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z]{2,24}/[^\s)<>]*)`)
571
572func linkBareAddresses(text string) string {
573	lines := strings.Split(text, "\n")
574	fenced := false
575	for i, line := range lines {
576		t := strings.TrimSpace(line)
577		if strings.HasPrefix(t, "```") || strings.HasPrefix(t, "~~~") {
578			fenced = !fenced
579			continue
580		}
581		if fenced || !strings.Contains(line, "/") {
582			continue
583		}
584		lines[i] = eachOutsideCode(line, func(part string) string {
585			return schemeless.ReplaceAllString(part, "${1}https://${2}")
586		})
587	}
588	return strings.Join(lines, "\n")
589}
590
591// eachOutsideCode applies f to the parts of a line that are not in backticks.
592func eachOutsideCode(line string, f func(string) string) string {
593	parts := strings.Split(line, "`")
594	for i := 0; i < len(parts); i += 2 {
595		parts[i] = f(parts[i])
596	}
597	return strings.Join(parts, "`")
598}
599
600// tidyURL drops the fragment and the tracking parameters, so the same page
601// reached from two searches is one source.
602func tidyURL(raw string) string {
603	raw = strings.TrimSpace(raw)
604	if raw == "" {
605		return ""
606	}
607	u, err := url.Parse(raw)
608	if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") {
609		return ""
610	}
611	u.Fragment = ""
612	if q := u.Query(); len(q) > 0 {
613		for k := range q {
614			if strings.HasPrefix(k, "utm_") || k == "fbclid" || k == "gclid" {
615				q.Del(k)
616			}
617		}
618		u.RawQuery = q.Encode()
619	}
620	return u.String()
621}
622
623func siteOf(raw string) string {
624	u, err := url.Parse(raw)
625	if err != nil {
626		return raw
627	}
628	return strings.TrimPrefix(u.Host, "www.")
629}
630
631func asString(v any) string {
632	s, _ := v.(string)
633	return s
634}
635
636// The words that say nothing about which page a sentence came from.
637var stopword = map[string]bool{
638	"the": true, "and": true, "but": true, "for": true, "not": true, "you": true,
639	"are": true, "was": true, "were": true, "has": true, "had": true, "have": true,
640	"his": true, "her": true, "its": true, "our": true, "their": true, "them": true,
641	"they": true, "this": true, "that": true, "these": true, "those": true,
642	"with": true, "from": true, "into": true, "over": true, "under": true,
643	"than": true, "then": true, "when": true, "what": true, "which": true,
644	"who": true, "whom": true, "will": true, "would": true, "could": true,
645	"should": true, "been": true, "being": true, "there": true, "here": true,
646	"also": true, "more": true, "most": true, "some": true, "such": true,
647	"only": true, "just": true, "very": true, "much": true, "many": true,
648	"each": true, "every": true, "any": true, "all": true, "one": true,
649	"two": true, "about": true, "after": true, "before": true, "because": true,
650	"while": true, "where": true, "how": true, "why": true, "can": true,
651	"said": true, "says": true, "say": true, "make": true, "made": true,
652	"does": true, "did": true, "done": true, "get": true, "got": true,
653	"out": true, "off": true, "own": true, "same": true, "still": true,
654	"other": true, "another": true, "between": true, "through": true,
655	"during": true, "against": true, "both": true, "well": true, "back": true,
656	"even": true, "way": true, "new": true, "now": true, "day": true,
657	"like": true, "time": true, "year": true, "years": true, "people": true,
658}