orchard

  1package main
  2
  3import (
  4	"bytes"
  5	"regexp"
  6	"strings"
  7
  8	"github.com/yuin/goldmark"
  9	"github.com/yuin/goldmark/extension"
 10	"github.com/yuin/goldmark/renderer/html"
 11)
 12
 13// md renders the answer. The model writes markdown so an answer can be skimmed
 14// rather than read as one block of prose.
 15//
 16// Raw HTML stays disabled. The input is model output built from pages this
 17// server fetched, so it is not trusted, and nothing an answer needs requires
 18// passing HTML through.
 19var md = goldmark.New(
 20	goldmark.WithExtensions(extension.GFM),
 21	goldmark.WithRendererOptions(html.WithHardWraps()),
 22)
 23
 24var citation = regexp.MustCompile(`\[(\d{1,3})\]`)
 25
 26var (
 27	doubleSpace      = regexp.MustCompile(`[ \t]{2,}`)
 28	spaceBeforePunct = regexp.MustCompile(` +([,.;:!?])`)
 29)
 30
 31const citeAnchor = `<a class="cite" href="#p$1" data-passage="$1">[$1]</a>`
 32
 33func renderMarkdown(src string) string {
 34	if strings.TrimSpace(src) == "" {
 35		return ""
 36	}
 37	var buf bytes.Buffer
 38	if err := md.Convert([]byte(src), &buf); err != nil {
 39		return "<p>" + escapeHTML(src) + "</p>"
 40	}
 41	return linkCitations(buf.String())
 42}
 43
 44// linkCitations turns [3] into an anchor after rendering rather than before,
 45// because goldmark drops raw HTML written into the markdown. It substitutes
 46// only in text, never inside a tag, so an attribute holding digits in brackets
 47// is left alone.
 48//
 49// Code is left alone too. `rows[0]` is an index and not a citation, and turning
 50// it into a link puts an anchor in the middle of something a reader is about to
 51// copy.
 52func linkCitations(h string) string {
 53	var out strings.Builder
 54	out.Grow(len(h) + 64)
 55	for {
 56		open := strings.IndexByte(h, '<')
 57		if open < 0 {
 58			out.WriteString(citation.ReplaceAllString(h, citeAnchor))
 59			break
 60		}
 61		out.WriteString(citation.ReplaceAllString(h[:open], citeAnchor))
 62		shut := strings.IndexByte(h[open:], '>')
 63		if shut < 0 {
 64			out.WriteString(h[open:])
 65			break
 66		}
 67		tag := h[open : open+shut+1]
 68		out.WriteString(tag)
 69		h = h[open+shut+1:]
 70		if name, ok := verbatimTag(tag); ok {
 71			end := strings.Index(h, "</"+name)
 72			if end < 0 {
 73				out.WriteString(h)
 74				break
 75			}
 76			out.WriteString(h[:end])
 77			h = h[end:]
 78		}
 79	}
 80	return out.String()
 81}
 82
 83// verbatimTag names the elements whose contents are copied straight through.
 84func verbatimTag(tag string) (string, bool) {
 85	for _, name := range []string{"pre", "code"} {
 86		if strings.HasPrefix(tag, "<"+name+" ") || tag == "<"+name+">" {
 87			return name, true
 88		}
 89	}
 90	return "", false
 91}
 92
 93func escapeHTML(s string) string {
 94	return strings.NewReplacer(
 95		"&", "&amp;", "<", "&lt;", ">", "&gt;", `"`, "&#34;", "'", "&#39;",
 96	).Replace(s)
 97}
 98
 99// tidyCitations collapses a line that cites the same passage over and over.
100//
101// The recipe shape produced "adding the cooked bacon [14], sausage [14], eggs
102// [14], and hash browns [14] [14]" from an instruction to cite what each part
103// came from. Repeating one ID inside a sentence adds nothing a reader can use,
104// and the prompt alone does not hold on a 4B, so the duplicates are removed
105// here instead. Distinct IDs on one line are kept, since those really are
106// different evidence.
107func tidyCitations(text string) string {
108	lines := strings.Split(text, "\n")
109	fenced := false
110	for i, line := range lines {
111		// Collapsing repeats inside a code block would move an array index to
112		// the end of the line, which is a working file turned into a broken one.
113		if fenceOpen.MatchString(line) && !fenced {
114			fenced = true
115			continue
116		} else if fenced {
117			fenced = !fenceShut.MatchString(line)
118			continue
119		}
120		found := citation.FindAllString(line, -1)
121		if len(found) < 2 {
122			continue
123		}
124		seen := map[string]bool{}
125		var order []string
126		for _, c := range found {
127			if !seen[c] {
128				seen[c] = true
129				order = append(order, c)
130			}
131		}
132		if len(order) == len(found) {
133			continue // every citation on the line is a different one
134		}
135		// Removing a marker from the middle of a sentence leaves a double
136		// space behind it and a space in front of the punctuation after it.
137		stripped := citation.ReplaceAllString(line, "")
138		stripped = doubleSpace.ReplaceAllString(stripped, " ")
139		stripped = spaceBeforePunct.ReplaceAllString(stripped, "$1")
140		stripped = strings.TrimRight(stripped, " ")
141
142		tail := strings.Join(order, "")
143		if strings.HasSuffix(stripped, ".") {
144			lines[i] = strings.TrimSuffix(stripped, ".") + " " + tail + "."
145		} else {
146			lines[i] = stripped + " " + tail
147		}
148	}
149	return strings.Join(lines, "\n")
150}