orchard

  1package main
  2
  3import (
  4	"encoding/json"
  5	"fmt"
  6	"go/parser"
  7	"go/token"
  8	"regexp"
  9	"strings"
 10)
 11
 12// The code shape is the one where the reader does not read the answer, they
 13// paste it, so the things worth checking are different. A sentence can be a
 14// little vague and still be useful. A file that does not parse is worth
 15// nothing, and neither is one with a hole in the middle where the model ran
 16// out of passages.
 17//
 18// Everything here is static and runs in process, because the image is FROM
 19// scratch and there is no interpreter in it to run anything against.
 20
 21// CodeBlock is one fenced block lifted out of an answer.
 22type CodeBlock struct {
 23	Lang string
 24	// File is the bold file name on the line above the fence, when the model
 25	// wrote one, since the contract asks for **app.py** above each block.
 26	File string
 27	Code string
 28	Line int
 29	// Closed is false when the fence was never closed, which is what a block
 30	// truncated by the token budget looks like.
 31	Closed bool
 32}
 33
 34// CodeCheck is one verdict, one per block, shown to the reader.
 35type CodeCheck struct {
 36	File  string
 37	Lang  string
 38	Lines int
 39	OK    bool
 40	Note  string
 41	// Truncated separates running out of tokens from writing something wrong.
 42	// Searching again cannot fix a length problem, so it is not worth the
 43	// three searches a second round costs.
 44	Truncated bool
 45}
 46
 47var (
 48	fenceOpen = regexp.MustCompile("^\\s{0,3}```+\\s*([A-Za-z0-9_+#.-]*)\\s*$")
 49	fenceShut = regexp.MustCompile("^\\s{0,3}```+\\s*$")
 50	boldFile  = regexp.MustCompile(`^\s*\*\*([^*]+?)\*\*\s*:?\s*$`)
 51	// A file name is a name with an extension, or one of the few that carry
 52	// none. Without this the heading above a block ("**Run it**") is read as a
 53	// file name and the check panel lists a file nobody wrote.
 54	looksLikeFile = regexp.MustCompile(`^[\w./-]+\.[A-Za-z0-9]{1,10}$|^(Dockerfile|Makefile|Procfile|\.env|\.gitignore|docker-compose\.ya?ml)$`)
 55)
 56
 57// codeBlocks pulls the fenced blocks out of an answer, keeping the file name
 58// written above each one.
 59func codeBlocks(md string) []CodeBlock {
 60	lines := strings.Split(md, "\n")
 61	var out []CodeBlock
 62	for i := 0; i < len(lines); i++ {
 63		m := fenceOpen.FindStringSubmatch(lines[i])
 64		if m == nil {
 65			continue
 66		}
 67		b := CodeBlock{Lang: strings.ToLower(m[1]), Line: i + 1, File: fileAbove(lines, i)}
 68		var body []string
 69		j := i + 1
 70		for ; j < len(lines); j++ {
 71			if fenceShut.MatchString(lines[j]) {
 72				b.Closed = true
 73				break
 74			}
 75			body = append(body, lines[j])
 76		}
 77		// The name comes off a leading comment when it was not written above
 78		// the fence, and the comment goes with it, so what is checked here is
 79		// what liftFileComments leaves for the reader to copy.
 80		if b.File == "" && len(body) > 0 {
 81			if name := fileInComment(body[0]); name != "" {
 82				b.File, body = name, body[1:]
 83			}
 84		}
 85		b.Code = strings.Join(body, "\n")
 86		out = append(out, b)
 87		i = j
 88	}
 89	return out
 90}
 91
 92// fileAbove reads back over blank lines for the bold file name the contract
 93// asks for above each block.
 94func fileAbove(lines []string, fence int) string {
 95	for i := fence - 1; i >= 0 && i >= fence-3; i-- {
 96		t := strings.TrimSpace(lines[i])
 97		if t == "" {
 98			continue
 99		}
100		m := boldFile.FindStringSubmatch(t)
101		if m == nil {
102			return ""
103		}
104		// "**File: server.py**" is as common as the bare name and means the
105		// same thing.
106		name := strings.TrimSpace(m[1])
107		if i := strings.IndexByte(name, ':'); i >= 0 {
108			name = strings.TrimSpace(name[i+1:])
109		}
110		if looksLikeFile.MatchString(name) {
111			return name
112		}
113		return ""
114	}
115	return ""
116}
117
118// fileInComment reads a file name off a comment on the first line, which is
119// where the model puts it about half the time whatever the contract says.
120func fileInComment(line string) string {
121	first := strings.TrimSpace(strings.SplitN(line, "\n", 2)[0])
122	for _, marker := range []string{"#", "//", "--", "<!--"} {
123		if !strings.HasPrefix(first, marker) {
124			continue
125		}
126		name := strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(strings.TrimPrefix(first, marker)), "-->"))
127		if looksLikeFile.MatchString(name) {
128			return name
129		}
130	}
131	return ""
132}
133
134// proseOnly is the answer with the code taken out.
135//
136// Validation checks a sentence against the passage it cites, and a line of
137// Python is not a sentence: splitting it produces claims nothing can entail,
138// each one a model call, and `data[0]` reads as a citation of passage 0. So
139// the checker sees the prose and the code is checked by parsing it instead.
140func proseOnly(md string) string {
141	lines := strings.Split(md, "\n")
142	var out []string
143	skip := false
144	for _, l := range lines {
145		switch {
146		case skip:
147			if fenceShut.MatchString(l) || fenceOpen.MatchString(l) {
148				skip = false
149			}
150		case fenceOpen.MatchString(l):
151			skip = true
152		default:
153			out = append(out, l)
154		}
155	}
156	return strings.Join(out, "\n")
157}
158
159// eachProseLine runs fn over the lines outside code blocks and keeps the rest
160// byte for byte. Every tidy-up here is written for prose, and a line of Python
161// that happens to match one of those patterns is not prose.
162func eachProseLine(md string, fn func(string) (string, bool)) string {
163	lines := strings.Split(md, "\n")
164	out := make([]string, 0, len(lines))
165	in := false
166	for _, l := range lines {
167		switch {
168		case fenceOpen.MatchString(l) && !in:
169			in = true
170		case in && fenceShut.MatchString(l):
171			in = false
172		case !in:
173			kept, ok := fn(l)
174			if !ok {
175				continue
176			}
177			l = kept
178		}
179		out = append(out, l)
180	}
181	return strings.Join(out, "\n")
182}
183
184// dropMeta removes a sentence that talks about the answer rather than
185// answering. A 4B writes "This response provides a Dockerfile, citing the
186// relevant passages" however plainly the contract forbids it, and it is the
187// first line a reader sees.
188//
189// A sentence ends at a full stop with a space after it. Splitting on the stop
190// alone cut "the `leaflet.webgl-temperature-map` library" in half and left the
191// answer opening mid-word, which is worse than the sentence it removed.
192var metaSentence = regexp.MustCompile(`(?i)(^|[.!?]\s+)(?:[^.!?]|[.!?]\S)*\b(?:this (?:response|answer|reply)|the following (?:response|answer)|as an ai|below (?:you will find|is the answer))\b(?:[^.!?]|[.!?]\S)*[.!?](\s+|$)`)
193
194func dropMeta(text string) string {
195	return eachProseLine(text, func(l string) (string, bool) {
196		if !metaSentence.MatchString(l) {
197			return l, true
198		}
199		cleaned := strings.TrimSpace(metaSentence.ReplaceAllString(l, "$1 "))
200		// A line that was only meta goes entirely, and a list marker left on
201		// its own goes with it.
202		if cleaned == "" || cleaned == "-" || cleaned == "*" {
203			return "", false
204		}
205		return cleaned, true
206	})
207}
208
209// stripCodeCitations removes citation markers from inside code blocks.
210//
211// The prompt says not to put them there and a 4B does it anyway, and unlike a
212// stray marker in prose this one is pasted into a file and stops it running.
213// Prose keeps its markers, which is the whole point of them.
214func stripCodeCitations(md string) string {
215	lines := strings.Split(md, "\n")
216	in := false
217	for i, l := range lines {
218		switch {
219		case fenceOpen.MatchString(l) && !in:
220			in = true
221		case in && fenceShut.MatchString(l):
222			in = false
223		case in && citation.MatchString(l):
224			// Only where it reads as a marker rather than as an index, so
225			// `items[0]` and `x = [1]` survive and `print(x) [3]` does not.
226			cleaned := trailingCitation.ReplaceAllString(l, "")
227			if c := commentAt(cleaned); c >= 0 {
228				cleaned = cleaned[:c] + citation.ReplaceAllString(cleaned[c:], "")
229			}
230			lines[i] = strings.TrimRight(cleaned, " \t")
231		}
232	}
233	return strings.Join(lines, "\n")
234}
235
236// commentAt is where a line comment starts, or -1. Rough on purpose: it is
237// used to decide whether a citation on this line is prose, and a "#" inside a
238// string is not worth a lexer.
239func commentAt(line string) int {
240	best := -1
241	for _, marker := range []string{"#", "//", "--", "/*", "<!--"} {
242		if i := strings.Index(line, marker); i >= 0 && (best < 0 || i < best) {
243			best = i
244		}
245	}
246	return best
247}
248
249// A marker after code and before the end of the line, or after a comment
250// character. An index is always attached to the name in front of it.
251var trailingCitation = regexp.MustCompile(`(\s+\[\d{1,3}\])+\s*$`)
252
253// liftFileComments removes the "# app.py" line from the top of a block whose
254// name is already shown above it. The name belongs in one place, and inside
255// the block it is a line the reader pastes into their own file.
256func liftFileComments(md string) string {
257	lines := strings.Split(md, "\n")
258	out := make([]string, 0, len(lines))
259	in, first := false, false
260	for _, l := range lines {
261		switch {
262		case fenceOpen.MatchString(l) && !in:
263			in, first = true, true
264		case in && fenceShut.MatchString(l):
265			in = false
266		case in && first:
267			first = false
268			if fileInComment(l) != "" {
269				continue
270			}
271		}
272		out = append(out, l)
273	}
274	return strings.Join(out, "\n")
275}
276
277// checkCode parses what can be parsed and looks for the holes a model leaves
278// when it is writing from fragments.
279func checkCode(blocks []CodeBlock) []CodeCheck {
280	var out []CodeCheck
281	for _, b := range blocks {
282		if strings.TrimSpace(b.Code) == "" {
283			continue
284		}
285		c := CodeCheck{File: b.File, Lang: b.Lang, Lines: len(strings.Split(b.Code, "\n")), OK: true}
286		if c.Lang == "" {
287			c.Lang = "text"
288		}
289		switch {
290		case !b.Closed:
291			c.OK, c.Truncated = false, true
292			c.Note = "ran out of room before the block ended, so it stops partway"
293		default:
294			if note := inspect(b); note != "" {
295				c.OK, c.Note = false, note
296			}
297		}
298		if c.OK {
299			c.Note = passNote(b)
300		}
301		out = append(out, c)
302	}
303	return out
304}
305
306// inspect returns what is wrong with a block, or an empty string.
307func inspect(b CodeBlock) string {
308	if note := metaInCode(b.Code); note != "" {
309		return note
310	}
311	if note := placeholderIn(b.Code); note != "" {
312		return note
313	}
314	switch normalLang(b.Lang, b.File) {
315	case "go":
316		return goSyntax(b.Code)
317	case "json":
318		if !json.Valid([]byte(b.Code)) {
319			return "not valid JSON"
320		}
321	case "python":
322		if note := balance(b.Code, pythonScan); note != "" {
323			return note
324		}
325		if note := pythonIndent(b.Code); note != "" {
326			return note
327		}
328		return undefinedInFString(b.Code)
329	case "javascript":
330		return balance(b.Code, cLikeScan)
331	case "html":
332		if note := htmlBalance(b.Code); note != "" {
333			return note
334		}
335		return missingLibrary(b.Code)
336	case "yaml":
337		return yamlTabs(b.Code)
338	case "dockerfile":
339		return dockerfileSyntax(b.Code)
340	}
341	return ""
342}
343
344// passNote says what was actually done, since "checked" with no detail invites
345// the reader to think more was checked than was.
346func passNote(b CodeBlock) string {
347	switch normalLang(b.Lang, b.File) {
348	case "go":
349		return "parses"
350	case "json":
351		return "valid JSON"
352	case "python", "javascript":
353		return "brackets balance"
354	case "html":
355		return "tags balance"
356	case "yaml":
357		return "indented with spaces"
358	case "dockerfile":
359		return "instructions are real ones"
360	}
361	return "read, not parsed"
362}
363
364// normalLang folds the tags people write onto the ones checked here, and falls
365// back to the file name when the fence carries no language.
366func normalLang(lang, file string) string {
367	switch lang {
368	case "go", "golang":
369		return "go"
370	case "py", "python", "python3":
371		return "python"
372	case "js", "javascript", "jsx", "mjs", "node", "ts", "typescript", "tsx":
373		return "javascript"
374	case "json":
375		return "json"
376	case "html", "htm":
377		return "html"
378	case "yaml", "yml":
379		return "yaml"
380	case "dockerfile", "docker":
381		return "dockerfile"
382	}
383	switch {
384	case file == "":
385		return lang
386	case strings.HasPrefix(file, "Dockerfile"), file == "Dockerfile":
387		return "dockerfile"
388	case strings.HasSuffix(file, ".go"):
389		return "go"
390	case strings.HasSuffix(file, ".py"):
391		return "python"
392	case strings.HasSuffix(file, ".js"), strings.HasSuffix(file, ".ts"):
393		return "javascript"
394	case strings.HasSuffix(file, ".json"):
395		return "json"
396	case strings.HasSuffix(file, ".html"):
397		return "html"
398	case strings.HasSuffix(file, ".yml"), strings.HasSuffix(file, ".yaml"):
399		return "yaml"
400	}
401	return lang
402}
403
404// goSyntax runs the real parser, which is the one language here that can be
405// checked properly without leaving the standard library. A snippet is not a
406// file, so a bare fragment gets wrapped before it is rejected.
407func goSyntax(src string) string {
408	tries := []string{src}
409	if !strings.Contains(src, "package ") {
410		tries = append(tries, "package p\n"+src)
411		if !strings.Contains(src, "func ") {
412			tries = append(tries, "package p\nfunc p() {\n"+src+"\n}")
413		}
414	}
415	var first error
416	for _, try := range tries {
417		_, err := parser.ParseFile(token.NewFileSet(), "x.go", try, parser.AllErrors)
418		if err == nil {
419			return ""
420		}
421		if first == nil {
422			first = err
423		}
424	}
425	msg := first.Error()
426	if i := strings.Index(msg, "\n"); i > 0 {
427		msg = msg[:i]
428	}
429	return "does not parse: " + strings.TrimPrefix(msg, "x.go:")
430}
431
432// A model that cannot find how a library works argues with itself in a comment
433// and then writes a stand-in value, which is the worst thing a code answer can
434// do: it is complete, it runs, and it does nothing. Worth failing the block
435// over, since the reader would have to read every comment to notice.
436var metaComment = regexp.MustCompile(`(?i)(provided (?:text|passage|context|basic usage)|the passages?\b|in the (?:provided|given) \w+|based on the (?:provided|given)|not explicitly (?:defined|mentioned|in)|for the purpose of (?:the|this) demo|placeholder for)`)
437
438func metaInCode(code string) string {
439	if metaComment.MatchString(code) {
440		return "argues about what the sources said in its own comments instead of doing the work"
441	}
442	return ""
443}
444
445var placeholders = []struct {
446	re   *regexp.Regexp
447	note string
448}{
449	{regexp.MustCompile(`(?im)^\s*(?:#|//|--|/\*|<!--)?\s*\.\.\.\s*(?:\*/|-->)?\s*$`), "has an ellipsis standing in for code that was not written"},
450	{regexp.MustCompile(`(?i)(rest of (?:the |your )?(?:code|file|implementation)|your code here|code goes here|implement(?:ation)? (?:this|here|goes here)|same as (?:above|before)|remaining \w+ (?:here|omitted)|\bomitted for brevity\b|\betc\.\.\.)`), "says the rest of the code goes here rather than writing it"},
451	{regexp.MustCompile(`(?i)TODO:?\s*(implement|fill|add your|complete)`), "leaves a TODO where working code should be"},
452	// A file that describes itself as an example of what a real one would do
453	// is not one, and it is what a model writes when the passages never showed
454	// it the real thing.
455	{regexp.MustCompile(`(?i)(conceptual (?:example|implementation)|in a real (?:environment|system|setup|deployment)|this is pseudo|pseudo-?code|for illustration only|does not actually|would actually (?:run|execute|invoke)|represents the logic)`), "describes what a working version would do rather than being one"},
456}
457
458func placeholderIn(code string) string {
459	for _, p := range placeholders {
460		if p.re.MatchString(code) {
461			return p.note
462		}
463	}
464	return ""
465}
466
467// scan describes how to skip the parts of a language where a bracket is not a
468// bracket. Counting them naively calls every string holding a "(" unbalanced,
469// which is a false alarm on almost every real file.
470type scan struct {
471	lineComment []string
472	blockOpen   string
473	blockClose  string
474	quotes      []string
475	// triples are Python's, and they have to be tried before the single
476	// character quotes or the first two characters close an empty string.
477	triples []string
478	escape  bool
479}
480
481var (
482	cLikeScan  = scan{lineComment: []string{"//"}, blockOpen: "/*", blockClose: "*/", quotes: []string{`"`, "'", "`"}, escape: true}
483	pythonScan = scan{lineComment: []string{"#"}, triples: []string{`"""`, "'''"}, quotes: []string{`"`, "'"}, escape: true}
484)
485
486// balance walks the code outside strings and comments and reports the first
487// bracket that does not close.
488func balance(code string, s scan) string {
489	var stack []byte
490	pair := map[byte]byte{')': '(', ']': '[', '}': '{'}
491	line := 1
492
493	for i := 0; i < len(code); {
494		c := code[i]
495		if c == '\n' {
496			line++
497			i++
498			continue
499		}
500		if skip, n := skipNonCode(code[i:], s); skip {
501			for _, r := range code[i : i+n] {
502				if r == '\n' {
503					line++
504				}
505			}
506			i += n
507			continue
508		}
509		switch c {
510		case '(', '[', '{':
511			stack = append(stack, c)
512		case ')', ']', '}':
513			if len(stack) == 0 || stack[len(stack)-1] != pair[c] {
514				return fmt.Sprintf("bracket mismatch on line %d, a %q closes nothing", line, string(c))
515			}
516			stack = stack[:len(stack)-1]
517		}
518		i++
519	}
520	if len(stack) > 0 {
521		return fmt.Sprintf("%d bracket%s never closed, so the block is incomplete",
522			len(stack), map[bool]string{true: "", false: "s"}[len(stack) == 1])
523	}
524	return ""
525}
526
527// skipNonCode reports how many bytes to skip when the text starts a comment or
528// a string.
529func skipNonCode(s string, sc scan) (bool, int) {
530	for _, lc := range sc.lineComment {
531		if strings.HasPrefix(s, lc) {
532			if n := strings.IndexByte(s, '\n'); n >= 0 {
533				return true, n
534			}
535			return true, len(s)
536		}
537	}
538	if sc.blockOpen != "" && strings.HasPrefix(s, sc.blockOpen) {
539		if n := strings.Index(s[len(sc.blockOpen):], sc.blockClose); n >= 0 {
540			return true, len(sc.blockOpen) + n + len(sc.blockClose)
541		}
542		return true, len(s)
543	}
544	for _, t := range sc.triples {
545		if strings.HasPrefix(s, t) {
546			if n := strings.Index(s[len(t):], t); n >= 0 {
547				return true, len(t) + n + len(t)
548			}
549			return true, len(s)
550		}
551	}
552	for _, q := range sc.quotes {
553		if !strings.HasPrefix(s, q) {
554			continue
555		}
556		for i := len(q); i < len(s); i++ {
557			if sc.escape && s[i] == '\\' {
558				i++
559				continue
560			}
561			// An unterminated string ends at the newline rather than eating
562			// the whole file, which keeps one bad quote from hiding every
563			// bracket after it.
564			if s[i] == '\n' {
565				return true, i
566			}
567			if strings.HasPrefix(s[i:], q) {
568				return true, i + len(q)
569			}
570		}
571		return true, len(s)
572	}
573	return false, 0
574}
575
576// fstring is an interpolated string and fslot a bare name inside one. Two
577// alternatives rather than a back reference, which RE2 does not have. Only a
578// plain identifier is read out, since an attribute, a call or a comprehension
579// has too many ways to be defined elsewhere.
580var (
581	fstring = regexp.MustCompile(`\bf"([^"\n]*)"|\bf'([^'\n]*)'`)
582	fslot   = regexp.MustCompile(`\{([A-Za-z_]\w*)\}`)
583)
584
585// undefinedInFString catches a name that appears only inside an f-string, which
586// is a NameError the moment the line runs.
587//
588// It is the shape of mistake a model makes filling in a connection string it
589// half remembers, and unlike a missing import nothing else in the file hints
590// at it.
591func undefinedInFString(code string) string {
592	for _, m := range fstring.FindAllStringSubmatch(code, -1) {
593		for _, slot := range fslot.FindAllStringSubmatch(m[1]+m[2], -1) {
594			name := slot[1]
595			if pyBuiltin[name] {
596				continue
597			}
598			if strings.Count(code, name) > 1 {
599				continue
600			}
601			return "uses " + name + " in an f-string and never defines it, so it fails as soon as it runs"
602		}
603	}
604	return ""
605}
606
607var pyBuiltin = map[string]bool{
608	"self": true, "cls": true, "True": true, "False": true, "None": true,
609	"e": true, "i": true, "x": true,
610}
611
612// pythonIndent catches the one whitespace mistake that actually stops Python
613// running, a file mixing tabs and spaces for its indentation.
614func pythonIndent(code string) string {
615	tabs, spaces := false, false
616	for _, l := range strings.Split(code, "\n") {
617		if strings.TrimSpace(l) == "" {
618			continue
619		}
620		switch {
621		case strings.HasPrefix(l, "\t"):
622			tabs = true
623		case strings.HasPrefix(l, " "):
624			spaces = true
625		}
626	}
627	if tabs && spaces {
628		return "mixes tabs and spaces for indentation, which Python rejects"
629	}
630	return ""
631}
632
633func yamlTabs(code string) string {
634	for i, l := range strings.Split(code, "\n") {
635		if strings.HasPrefix(l, "\t") {
636			return fmt.Sprintf("line %d is indented with a tab, which YAML does not allow", i+1)
637		}
638	}
639	return ""
640}
641
642var (
643	htmlTag  = regexp.MustCompile(`(?is)<(/?)([a-z][a-z0-9-]*)\b[^>]*?(/?)>`)
644	voidTags = map[string]bool{
645		"area": true, "base": true, "br": true, "col": true, "embed": true,
646		"hr": true, "img": true, "input": true, "link": true, "meta": true,
647		"param": true, "source": true, "track": true, "wbr": true,
648		"!doctype": true,
649	}
650)
651
652// htmlBalance is a tag counter rather than a parser. It only has to catch a
653// page that stops halfway, which is what a truncated answer looks like.
654func htmlBalance(code string) string {
655	var stack []string
656	for _, m := range htmlTag.FindAllStringSubmatch(code, -1) {
657		closing, name, self := m[1] == "/", strings.ToLower(m[2]), m[3] == "/"
658		if voidTags[name] || self {
659			continue
660		}
661		if !closing {
662			stack = append(stack, name)
663			continue
664		}
665		// Unwind to the matching open rather than demanding the top of the
666		// stack, since a real page leaves the odd <p> and <li> unclosed and
667		// that is legal HTML.
668		found := -1
669		for i := len(stack) - 1; i >= 0; i-- {
670			if stack[i] == name {
671				found = i
672				break
673			}
674		}
675		if found < 0 {
676			return fmt.Sprintf("closes </%s> without opening it", name)
677		}
678		stack = stack[:found]
679	}
680	for _, name := range stack {
681		switch name {
682		case "html", "head", "body", "script", "style", "div", "svg", "table":
683			return fmt.Sprintf("<%s> is never closed, so the page is incomplete", name)
684		}
685	}
686	return ""
687}
688
689var (
690	scriptSrc  = regexp.MustCompile(`(?i)<script[^>]+src\s*=`)
691	newGlobal  = regexp.MustCompile(`\bnew\s+(?:window\.)?([A-Z][A-Za-z0-9_]*)\s*[.(]`)
692	callGlobal = regexp.MustCompile(`(?:^|[^\w.])(?:window\.)?([A-Z][A-Za-z0-9_]*)\.[a-z]\w*\s*\(`)
693)
694
695// browserGlobals is what a page already has without loading anything.
696var browserGlobals = map[string]bool{
697	"Array": true, "Boolean": true, "Date": true, "Error": true, "Function": true,
698	"Image": true, "Intl": true, "JSON": true, "Map": true, "Math": true,
699	"Number": true, "Object": true, "Promise": true, "Proxy": true, "Reflect": true,
700	"RegExp": true, "Set": true, "String": true, "Symbol": true, "URL": true,
701	"URLSearchParams": true, "WeakMap": true, "WeakSet": true, "XMLHttpRequest": true,
702	"Audio": true, "Blob": true, "FormData": true, "Headers": true, "Request": true,
703	"Response": true, "AbortController": true, "Event": true, "CustomEvent": true,
704	"Worker": true, "WebSocket": true, "FileReader": true, "TextEncoder": true,
705	"TextDecoder": true, "BigInt": true, "Notification": true, "ResizeObserver": true,
706	"IntersectionObserver": true, "MutationObserver": true, "OffscreenCanvas": true,
707	"Path2D": true, "DOMParser": true, "Option": true,
708}
709
710// missingLibrary catches a page calling into a library it never loads.
711//
712// The check only runs on a page with no external script at all, which is the
713// case it can be sure about: the model was asked for a map, could not find a
714// real library in the passages, and called `new SatMeteo.Map(...)` off a
715// website that has no such thing. The page renders blank and looks finished.
716func missingLibrary(code string) string {
717	if scriptSrc.MatchString(code) {
718		return ""
719	}
720	for _, m := range append(newGlobal.FindAllStringSubmatch(code, -1), callGlobal.FindAllStringSubmatch(code, -1)...) {
721		name := m[1]
722		if browserGlobals[name] || definedIn(code, name) {
723			continue
724		}
725		return fmt.Sprintf("uses %s but loads no script that defines it, so the page will not do anything", name)
726	}
727	return ""
728}
729
730func definedIn(code, name string) bool {
731	for _, form := range []string{
732		"var " + name, "let " + name, "const " + name, "function " + name,
733		"class " + name, "window." + name + " =", name + " =",
734	} {
735		if strings.Contains(code, form) {
736			return true
737		}
738	}
739	return false
740}
741
742var dockerInstructions = map[string]bool{
743	"FROM": true, "RUN": true, "CMD": true, "LABEL": true, "MAINTAINER": true,
744	"EXPOSE": true, "ENV": true, "ADD": true, "COPY": true, "ENTRYPOINT": true,
745	"VOLUME": true, "USER": true, "WORKDIR": true, "ARG": true, "ONBUILD": true,
746	"STOPSIGNAL": true, "HEALTHCHECK": true, "SHELL": true,
747}
748
749// dockerfileSyntax checks the two things that are always true of one: it opens
750// on FROM, and every line is an instruction docker knows.
751func dockerfileSyntax(code string) string {
752	seenFrom := false
753	continued := false
754	for i, raw := range strings.Split(code, "\n") {
755		l := strings.TrimSpace(raw)
756		wasContinued := continued
757		continued = strings.HasSuffix(l, "\\")
758		if l == "" || strings.HasPrefix(l, "#") || wasContinued {
759			continue
760		}
761		word := strings.ToUpper(strings.Fields(l)[0])
762		if !dockerInstructions[word] {
763			return fmt.Sprintf("line %d starts with %q, which is not a Dockerfile instruction", i+1, strings.Fields(l)[0])
764		}
765		if word == "FROM" {
766			seenFrom = true
767		}
768		if !seenFrom && word != "ARG" {
769			return fmt.Sprintf("line %d runs before any FROM, so there is no image to build on", i+1)
770		}
771	}
772	if !seenFrom {
773		return "has no FROM, so there is nothing to build"
774	}
775	return ""
776}
777
778// unusedConstants names a config knob the code defines and never reads.
779//
780// A model writing from fragments puts CACHE_AGE at the top because every
781// example it saw had one, and then never uses it, which reads as a setting a
782// person can change and is not. Upper case only, since a lower case name has
783// too many ways to be used indirectly.
784var constLine = regexp.MustCompile(`(?m)^([A-Z][A-Z0-9_]{2,})\s*(?::[^=\n]+)?=`)
785
786func unusedConstants(blocks []CodeBlock) []string {
787	var out []string
788	for _, b := range blocks {
789		switch normalLang(b.Lang, b.File) {
790		case "python", "javascript", "go":
791		default:
792			continue
793		}
794		for _, m := range constLine.FindAllStringSubmatch(b.Code, -1) {
795			name := m[1]
796			if strings.Count(b.Code, name) > 1 {
797				continue
798			}
799			where := b.File
800			if where == "" {
801				where = "the " + b.Lang + " block"
802			}
803			out = append(out, fmt.Sprintf("%s sets %s and never uses it, so changing it does nothing", where, name))
804		}
805	}
806	return out
807}
808
809// codeWarnings turns failed checks into the lines shown above the answer. Only
810// the failures, since a reader does not need telling that six files were fine
811// when the panel below already says so.
812func codeWarnings(checks []CodeCheck) []string {
813	var out []string
814	for _, c := range checks {
815		if c.OK {
816			continue
817		}
818		name := c.File
819		if name == "" {
820			name = "the " + c.Lang + " block"
821		}
822		out = append(out, fmt.Sprintf("%s %s", name, c.Note))
823	}
824	return out
825}
826
827// codeWeight scores a page for a code question. A page that shows code is
828// worth more than one that talks about it, and the docs are worth more than a
829// tutorial farm, which is most of what a search for a library name returns.
830func codeWeight(p *Page) int {
831	if p == nil {
832		return 0
833	}
834	score := strings.Count(p.Markdown, "\n```")
835	if score > 6 {
836		score = 6
837	}
838	host := strings.ToLower(p.Site)
839	switch {
840	case strings.HasPrefix(host, "docs."), strings.Contains(host, "readthedocs"),
841		strings.Contains(host, "developer."), strings.HasSuffix(host, ".dev"),
842		strings.Contains(host, "github.com"), strings.Contains(host, "gitlab.com"),
843		strings.Contains(host, "mozilla.org"), strings.Contains(host, "pkg.go.dev"),
844		strings.Contains(host, "docker.com"), strings.Contains(host, "python.org"),
845		strings.Contains(host, "ibm.com"), strings.Contains(host, "microsoft.com"),
846		strings.Contains(host, "stackoverflow.com"):
847		score += 4
848	}
849	return score
850}
851
852// codeFailed says whether the checks found something a second search could fix.
853// A file that does not parse and a package that is not in its registry are both
854// evidence the pages found did not show how this is really written.
855func codeFailed(checks []CodeCheck, deps []Dependency) bool {
856	for _, c := range checks {
857		if !c.OK && !c.Truncated {
858			return true
859		}
860	}
861	for _, d := range deps {
862		if d.Checked && !d.Found {
863			return true
864		}
865	}
866	return false
867}
868
869// codeHint tells the second plan what went wrong, in the terms the first one
870// got wrong, so it searches for the library rather than rewording the answer.
871func codeHint(checks []CodeCheck, deps []Dependency) string {
872	var bad []string
873	for _, c := range checks {
874		if !c.OK {
875			name := c.File
876			if name == "" {
877				name = "the " + c.Lang + " block"
878			}
879			bad = append(bad, name+" "+c.Note)
880		}
881	}
882	for _, d := range deps {
883		if d.Checked && !d.Found {
884			bad = append(bad, "it used the package "+d.Name+", which does not exist on "+ecoName(d.Eco))
885		}
886	}
887	return "A previous attempt wrote code with these problems: " + strings.Join(bad, "; ") +
888		". Search for the official documentation and a working example of the library or tool this actually needs."
889}