orchard
mirrorEvery site I host, in one repo, along with the Cloudflare Tunnel and Caddy that front them. It's all Go, Vite, and SQLite, and it runs on a desktop at home with nothing listening on an inbound port.
blogbuncaddycloudflare-tunneldockergogolanghomelabhtml-templatemonorepoself-hostedseosqlitestatic-sitetypstuptime-monitoringviteweb-analytics
1package tools
2
3import (
4 "context"
5 "encoding/xml"
6 "fmt"
7 neturl "net/url"
8 "sort"
9 "strings"
10 "sync"
11 "time"
12 "unicode"
13)
14
15// The news reader. A question like "any big news today?" used to go to
16// web_search, which returns whatever a search engine felt like that minute, so
17// the same question twice gave two different days of news. This reads a fixed
18// list of publishers instead, so the shape of an answer is the same every time
19// and the only thing that varies is what happened.
20//
21// Two kinds of source, and they are ranked differently. Hacker News and
22// Lobsters publish a score, so what got attention is a number and the cut is a
23// threshold. A newsroom feed has no score, and the order it publishes in is the
24// desk's own judgement of what leads, so position is the signal there.
25
26const (
27 // What a front page is worth reading down to. Below this an item is a
28 // story a handful of people upvoted, which is not what "big news" means.
29 hnMinPoints = 150
30 lobstersMinPoints = 25
31 // How far down a newsroom feed still counts as the front page.
32 deskDepth = 12
33)
34
35// newsWindows are the phrasings Isaac actually uses. Each resolves against a
36// New York clock, since "today" means the day he is having and not UTC's.
37var newsWindows = []string{"today", "yesterday", "weekend", "weekend-and-today", "week", "month"}
38
39var News = Tool{
40 Name: "news",
41 Description: "Read the news from a fixed list of publishers rather than searching the web. " +
42 "Use this for any question asking what is happening or what happened over a period, " +
43 "like news today, big stories this weekend, what did I miss this week, or any tech or AI news. " +
44 "For one named story that is already known, use web_search instead. " +
45 "It reads publisher front pages and nothing else, so it cannot tell you whether people are " +
46 "complaining about something, whether a service is degraded, or what is being said about a " +
47 "company outside the newsroom. Those are web_search.",
48 Schema: obj(map[string]any{
49 "window": map[string]any{"type": "string",
50 "description": "the period the question asked for, taken literally. today means today, " +
51 "weekend means Saturday and Sunday only, and weekend-and-today is for a question " +
52 "that asks for both, like over the weekend including today",
53 "enum": newsWindows},
54 "topic": map[string]any{"type": "string",
55 "description": "the subject asked for, or general when the question named none",
56 "enum": []string{"general", "world", "us", "tech", "ai", "politics",
57 "business", "science"}},
58 }, "window"),
59 Run: func(ctx context.Context, d *Deps, a map[string]any) (any, error) {
60 window := strings.ToLower(argStr(a, "window"))
61 if window == "" {
62 window = "today"
63 }
64 topic := strings.ToLower(argStr(a, "topic"))
65 if topic == "" {
66 topic = "general"
67 }
68
69 now := d.Now().In(newYork())
70 since, until, label := resolveWindow(now, window)
71
72 sections, tried, failed := gatherNews(ctx, d, topic, since, until)
73 if len(sections) == 0 {
74 if failed >= tried && tried > 0 {
75 return nil, fmt.Errorf("none of the %d news sources answered", tried)
76 }
77 return map[string]any{
78 "window": label, "topic": topic, "sections": []any{},
79 "note": "Nothing was published in that window by any of the sources read. " +
80 "Say so plainly rather than widening the window on your own or " +
81 "answering from memory.",
82 }, nil
83 }
84
85 return map[string]any{
86 "window": label,
87 "topic": topic,
88 "sections": sections,
89 "sources": sectionSources(sections),
90 "count": countItems(sections),
91 "note": itemBudget(sections) +
92 "Answer in exactly this shape and nothing else:\n\n" +
93 "### <section name>\n" +
94 "- **<the fact>** rest of the plain sentence. (Publisher: \"their headline\")\n" +
95 "- **<the fact>** rest of the plain sentence. (Publisher: \"their headline\")\n\n" +
96 "### <next section name>\n" +
97 "- ...\n\n" +
98 "Worked example of one bullet:\n" +
99 "- **Five died** when a cargo plane overran the runway at **Miami International**. " +
100 "(NPR: \"5 dead after crash at Miami Airport\")\n\n" +
101 "Rules:\n" +
102 "- One heading per section above, in the order given, keeping its name.\n" +
103 "- Every item gets a bullet, including the sections further down. Dropping a " +
104 "whole section is the failure to avoid here. This is a rundown, so do not pick " +
105 "one and write it up, and do not collapse the list into a paragraph.\n" +
106 "- One bullet per item and no more. The counts above are what is in the list, " +
107 "so never split one story into two bullets or repeat one across sections to " +
108 "reach a number. Every headline above is already distinct.\n" +
109 "- Bold only the few words carrying the news, the number, the name, the place or " +
110 "what changed, so it can be skimmed. Never bold a whole sentence.\n" +
111 "- The publisher's headline goes after your sentence, word for word, in quotes. " +
112 "Never drop it, it is what shows how the story was sold.\n" +
113 "- Merge a story two publishers ran into one bullet and name both.\n" +
114 "- Strip the loaded verbs and the party line, and put back the specifics they " +
115 "hid, so \"SLAMS\" becomes what was actually said and a tariff names the rate " +
116 "and the goods. Take no side and invent no detail.\n" +
117 "- Do not research any of these. The rundown is the answer.",
118 }, nil
119 },
120}
121
122func countItems(sections []NewsSection) int {
123 n := 0
124 for _, sec := range sections {
125 n += len(sec.Items)
126 }
127 return n
128}
129
130// itemBudget opens the note with the arithmetic, because "every item gets a
131// bullet" on its own got four of twenty one and two sections of four. A model
132// told it owes twenty one bullets under four headings can count what it wrote.
133func itemBudget(sections []NewsSection) string {
134 var b strings.Builder
135 fmt.Fprintf(&b, "There are %d items here across %d sections. Your answer has to carry all %d, "+
136 "as %d bullets under %d headings:\n", countItems(sections), len(sections),
137 countItems(sections), countItems(sections), len(sections))
138 for _, sec := range sections {
139 fmt.Fprintf(&b, " %s: %d bullets\n", sec.Name, len(sec.Items))
140 }
141 b.WriteString("\n")
142 return b.String()
143}
144
145// sectionSources says which publishers are in the answer, so a reader can see
146// at a glance whether a source they expected was reachable.
147func sectionSources(sections []NewsSection) []string {
148 seen := map[string]bool{}
149 var out []string
150 for _, sec := range sections {
151 for _, it := range sec.Items {
152 if !seen[it.Source] {
153 seen[it.Source] = true
154 out = append(out, it.Source)
155 }
156 }
157 }
158 sort.Strings(out)
159 return out
160}
161
162// newYork is the clock every window is resolved against. A fixed offset would
163// drift twice a year, and the answer to "today" would then be wrong for an hour
164// on two mornings.
165func newYork() *time.Location {
166 loc, err := time.LoadLocation("America/New_York")
167 if err != nil {
168 return time.UTC
169 }
170 return loc
171}
172
173// resolveWindow turns a word into a half open range and a label saying what it
174// decided, because a reader who asked for the weekend deserves to see which
175// days that was rather than trusting it.
176func resolveWindow(now time.Time, window string) (time.Time, time.Time, string) {
177 midnight := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
178 day := func(t time.Time) string { return t.Format("Mon 2 Jan 2006") }
179
180 switch window {
181 case "yesterday":
182 start := midnight.AddDate(0, 0, -1)
183 return start, midnight, "yesterday, " + day(start)
184 case "weekend-and-today":
185 // "over the weekend including today" has no answer in either of the
186 // other two, and asked on a Monday the plain weekend window excludes
187 // today by definition, which quietly dropped the day he asked about.
188 back := (int(now.Weekday()) + 1) % 7
189 sat := midnight.AddDate(0, 0, -back)
190 return sat, now, "the weekend of " + day(sat) + " and " + day(sat.AddDate(0, 0, 1)) +
191 ", plus today, " + day(now)
192 case "weekend":
193 // The most recent Saturday and Sunday. Asked on one of them it means
194 // the one being had, and asked on a Wednesday it means the one just
195 // gone, which is what a person means by "this weekend" either way.
196 back := (int(now.Weekday()) + 1) % 7
197 sat := midnight.AddDate(0, 0, -back)
198 end := sat.AddDate(0, 0, 2)
199 if end.After(now) {
200 end = now
201 }
202 return sat, end, "the weekend of " + day(sat) + " and " + day(sat.AddDate(0, 0, 1))
203 case "week":
204 start := midnight.AddDate(0, 0, -6)
205 return start, now, "the week from " + day(start) + " to " + day(now)
206 case "month":
207 start := midnight.AddDate(0, 0, -29)
208 return start, now, "the thirty days from " + day(start) + " to " + day(now)
209 default:
210 return midnight, now, "today, " + day(now)
211 }
212}
213
214// NewsItem is one story. Points is left out of the JSON when there is none,
215// since a newsroom feed has no score and a zero would read as unpopular rather
216// than unscored.
217type NewsItem struct {
218 Source string `json:"source"`
219 Headline string `json:"headline"`
220 URL string `json:"url"`
221 Published string `json:"published"`
222 Summary string `json:"summary,omitempty"`
223 Points int `json:"points,omitempty"`
224 Comments int `json:"comments,omitempty"`
225 Discuss string `json:"discuss,omitempty"`
226
227 at time.Time
228 rank int
229}
230
231// feed is one newsroom source. The desk orders its own front page, so rank is
232// the position an item arrived in.
233type feed struct {
234 name, url string
235}
236
237// section is one heading in the answer. Each keeps its own slots, which is what
238// stops the newsrooms crowding the aggregators out: a flat cap over everything
239// sorted desks first took 28 of 45 items and left all fourteen Hacker News
240// stories and both Lobsters ones on the floor.
241type section struct {
242 name string
243 feeds []feed
244 aggregators bool
245 slots int
246}
247
248var (
249 nprNews = feed{"NPR", "https://feeds.npr.org/1001/rss.xml"}
250 nprWorld = feed{"NPR World", "https://feeds.npr.org/1004/rss.xml"}
251 nprBusiness = feed{"NPR Business", "https://feeds.npr.org/1006/rss.xml"}
252 nprScience = feed{"NPR Science", "https://feeds.npr.org/1007/rss.xml"}
253 nprPolitics = feed{"NPR Politics", "https://feeds.npr.org/1014/rss.xml"}
254 nprTech = feed{"NPR Technology", "https://feeds.npr.org/1019/rss.xml"}
255
256 bbcNews = feed{"BBC", "https://feeds.bbci.co.uk/news/rss.xml"}
257 bbcWorld = feed{"BBC World", "https://feeds.bbci.co.uk/news/world/rss.xml"}
258 bbcTech = feed{"BBC Technology", "https://feeds.bbci.co.uk/news/technology/rss.xml"}
259 bbcBusiness = feed{"BBC Business", "https://feeds.bbci.co.uk/news/business/rss.xml"}
260 bbcScience = feed{"BBC Science", "https://feeds.bbci.co.uk/news/science_and_environment/rss.xml"}
261
262 ars = feed{"Ars Technica", "https://feeds.arstechnica.com/arstechnica/index"}
263 techcrunch = feed{"TechCrunch", "https://techcrunch.com/feed/"}
264 marketch = feed{"MarketWatch", "https://feeds.content.dowjones.io/public/rss/mw_topstories"}
265)
266
267// newsSections is the whole source list. Reuters and AP are missing because
268// both retired their public feeds: every documented Reuters path is dead at the
269// connection and apnews.com answers 401 or 404 to all of theirs. Reddit is
270// missing because old.reddit.com redirects every logged out request to a login
271// page and www.reddit.com rate limits its rss after a handful of calls, and
272// carries no score, so there is no way to ask it what got big numbers.
273var newsSections = map[string][]section{
274 "general": {
275 {name: "Top stories", feeds: []feed{nprNews, bbcNews}, slots: 10},
276 {name: "World", feeds: []feed{nprWorld, bbcWorld}, slots: 7},
277 {name: "Business", feeds: []feed{nprBusiness, bbcBusiness}, slots: 4},
278 {name: "Tech and what the forums are reading",
279 feeds: []feed{bbcTech, ars, techcrunch}, aggregators: true, slots: 9},
280 },
281 "us": {
282 {name: "Top stories", feeds: []feed{nprNews, bbcNews}, slots: 12},
283 {name: "Politics", feeds: []feed{nprPolitics}, slots: 8},
284 },
285 "world": {
286 {name: "World", feeds: []feed{nprWorld, bbcWorld}, slots: 20},
287 },
288 "politics": {
289 {name: "Politics", feeds: []feed{nprPolitics, bbcNews}, slots: 20},
290 },
291 "business": {
292 {name: "Business", feeds: []feed{nprBusiness, bbcBusiness, marketch}, slots: 20},
293 },
294 "science": {
295 {name: "Science", feeds: []feed{nprScience, bbcScience}, slots: 20},
296 },
297 "tech": {
298 {name: "What the forums are reading", aggregators: true, slots: 12},
299 {name: "Tech press", feeds: []feed{bbcTech, nprTech, ars, techcrunch}, slots: 10},
300 },
301}
302
303// NewsSection is one heading and what belongs under it.
304type NewsSection struct {
305 Name string `json:"section"`
306 Items []NewsItem `json:"items"`
307}
308
309// gatherNews reads every source for a topic at once and keeps what landed in
310// the window. A source that fails is counted and skipped rather than failing
311// the tool, because eight publishers answering out of nine is still the news.
312//
313// The aggregators are fetched once however many sections asked for them, since
314// two sections both wanting Hacker News is not a reason to fetch it twice.
315func gatherNews(ctx context.Context, d *Deps, topic string, since, until time.Time) ([]NewsSection, int, int) {
316 sections := newsSections[topic]
317 if topic == "ai" {
318 sections = newsSections["tech"]
319 }
320 if len(sections) == 0 {
321 sections = newsSections["general"]
322 }
323
324 var (
325 mu sync.Mutex
326 tried int
327 failed int
328 wg sync.WaitGroup
329 byFeed = map[string][]NewsItem{}
330 scored []NewsItem
331 )
332 note := func(err error) {
333 tried++
334 if err != nil {
335 failed++
336 }
337 }
338
339 wantAgg := false
340 seenFeed := map[string]bool{}
341 for _, sec := range sections {
342 if sec.aggregators {
343 wantAgg = true
344 }
345 for _, f := range sec.feeds {
346 if seenFeed[f.url] {
347 continue
348 }
349 seenFeed[f.url] = true
350 wg.Add(1)
351 go func(f feed) {
352 defer wg.Done()
353 got, err := readFeed(ctx, d, f, since, until)
354 mu.Lock()
355 defer mu.Unlock()
356 note(err)
357 byFeed[f.url] = got
358 }(f)
359 }
360 }
361 if wantAgg {
362 wg.Add(2)
363 go func() {
364 defer wg.Done()
365 got, err := readHackerNews(ctx, d, since, until)
366 mu.Lock()
367 defer mu.Unlock()
368 note(err)
369 scored = append(scored, got...)
370 }()
371 go func() {
372 defer wg.Done()
373 got, err := readLobsters(ctx, d, since, until)
374 mu.Lock()
375 defer mu.Unlock()
376 note(err)
377 scored = append(scored, got...)
378 }()
379 }
380 wg.Wait()
381
382 // One story reaching two sections reads as the tool repeating itself, so a
383 // headline is placed in the first section that wanted it and nowhere else.
384 placed := map[string]bool{}
385 // The headlines already taken, for the case two publishers write one story
386 // up under different words. An exact key cannot see that, and on
387 // 2026-09-08 the same Navier-Stokes paper led the tech section twice.
388 var placedWords []map[string]bool
389 take := func(items []NewsItem, slots int, scoredFirst bool) []NewsItem {
390 items = dedupeNews(items)
391 sortNews(items, scoredFirst)
392 out := make([]NewsItem, 0, slots)
393 for _, it := range items {
394 if len(out) >= slots {
395 break
396 }
397 key := strings.ToLower(strings.TrimSpace(it.Headline))
398 if u := canonicalNewsURL(it.URL); u != "" {
399 key = u
400 }
401 if placed[key] {
402 continue
403 }
404 words := headlineWords(it.Headline)
405 if anySameStory(placedWords, words) {
406 continue
407 }
408 placed[key] = true
409 placedWords = append(placedWords, words)
410 out = append(out, it)
411 }
412 return out
413 }
414
415 out := make([]NewsSection, 0, len(sections))
416 for _, sec := range sections {
417 var desks []NewsItem
418 for _, f := range sec.feeds {
419 desks = append(desks, byFeed[f.url]...)
420 }
421 if got := fillSection(sec, desks, scored, take); len(got) > 0 {
422 out = append(out, NewsSection{Name: sec.name, Items: got})
423 }
424 }
425 return out, tried, failed
426}
427
428// fillSection shares one section's slots out between the newsrooms and the
429// aggregators. A section holding both has to split them, or the desks sort
430// first and eat every one, which is the same starvation the per section slots
431// were added to stop. Whatever one side does not use, the other takes.
432func fillSection(sec section, desks, scored []NewsItem, take func([]NewsItem, int, bool) []NewsItem) []NewsItem {
433 switch {
434 case !sec.aggregators:
435 return take(desks, sec.slots, false)
436 case len(sec.feeds) == 0:
437 return take(scored, sec.slots, true)
438 }
439 got := take(scored, sec.slots/2, true)
440 got = append(got, take(desks, sec.slots-len(got), false)...)
441 if short := sec.slots - len(got); short > 0 {
442 got = append(got, take(scored, short, true)...)
443 }
444 return got
445}
446
447// rssFeed covers RSS 2.0 and Atom in one shape, since BBC and NPR publish the
448// first and plenty of others publish the second, and the difference is not
449// worth two parsers.
450type rssFeed struct {
451 Items []struct {
452 Title string `xml:"title"`
453 Link string `xml:"link"`
454 Description string `xml:"description"`
455 PubDate string `xml:"pubDate"`
456 Date string `xml:"date"`
457 } `xml:"channel>item"`
458 Entries []struct {
459 Title string `xml:"title"`
460 Summary string `xml:"summary"`
461 Updated string `xml:"updated"`
462 Link struct {
463 Href string `xml:"href,attr"`
464 } `xml:"link"`
465 } `xml:"entry"`
466}
467
468func readFeed(ctx context.Context, d *Deps, f feed, since, until time.Time) ([]NewsItem, error) {
469 body, err := get(ctx, d, f.url, "application/rss+xml, application/xml, text/xml")
470 if err != nil {
471 return nil, err
472 }
473 var parsed rssFeed
474 if err := xml.Unmarshal(body, &parsed); err != nil {
475 return nil, fmt.Errorf("%s: %w", f.name, err)
476 }
477
478 var out []NewsItem
479 add := func(title, link, summary, when string, pos int) {
480 title = strings.TrimSpace(title)
481 if title == "" || pos >= deskDepth {
482 return
483 }
484 at, ok := parseFeedTime(when)
485 if !ok || at.Before(since) || !at.Before(until) {
486 return
487 }
488 out = append(out, NewsItem{
489 Source: f.name, Headline: title, URL: strings.TrimSpace(link),
490 Published: at.In(since.Location()).Format("Mon 2 Jan 15:04"),
491 Summary: trimSummary(summary), at: at, rank: pos,
492 })
493 }
494 for i, it := range parsed.Items {
495 when := it.PubDate
496 if when == "" {
497 when = it.Date
498 }
499 add(it.Title, it.Link, it.Description, when, i)
500 }
501 for i, e := range parsed.Entries {
502 add(e.Title, e.Link.Href, e.Summary, e.Updated, i)
503 }
504 return out, nil
505}
506
507// parseFeedTime covers what publishers actually send. RSS is meant to be
508// RFC1123 with a numeric zone and several send a named one or leave it off.
509func parseFeedTime(s string) (time.Time, bool) {
510 s = strings.TrimSpace(s)
511 if s == "" {
512 return time.Time{}, false
513 }
514 for _, layout := range []string{
515 time.RFC1123Z, time.RFC1123, time.RFC3339, time.RFC822Z, time.RFC822,
516 "Mon, 2 Jan 2006 15:04:05 -0700", "Mon, 2 Jan 2006 15:04:05 MST",
517 "2006-01-02T15:04:05Z07:00", "2006-01-02 15:04:05",
518 } {
519 if t, err := time.Parse(layout, s); err == nil {
520 return t, true
521 }
522 }
523 return time.Time{}, false
524}
525
526func trimSummary(s string) string {
527 s = stripTags(s)
528 s = strings.Join(strings.Fields(s), " ")
529 if len(s) > 280 {
530 s = strings.TrimSpace(s[:280]) + "…"
531 }
532 return s
533}
534
535// A feed description carries markup often enough that leaving it in wastes the
536// model's window on span tags.
537func stripTags(s string) string {
538 var b strings.Builder
539 depth := 0
540 for _, r := range s {
541 switch {
542 case r == '<':
543 depth++
544 case r == '>':
545 if depth > 0 {
546 depth--
547 }
548 case depth == 0:
549 b.WriteRune(r)
550 }
551 }
552 return b.String()
553}
554
555func readHackerNews(ctx context.Context, d *Deps, since, until time.Time) ([]NewsItem, error) {
556 // search_by_date with a timestamp filter rather than the front page, since
557 // the front page is whatever is on it now and says nothing about a window
558 // that closed on Sunday.
559 // The comparison operators have to be percent encoded. Sent raw, Algolia's
560 // front door answers 400 with an HTML body, which parses as no stories
561 // rather than as an error.
562 filters := fmt.Sprintf("created_at_i>%d,created_at_i<%d,points>%d",
563 since.Unix(), until.Unix(), hnMinPoints)
564 url := "https://hn.algolia.com/api/v1/search?tags=story&hitsPerPage=40&numericFilters=" +
565 neturl.QueryEscape(filters)
566 var payload struct {
567 Hits []struct {
568 Title string `json:"title"`
569 URL string `json:"url"`
570 Points int `json:"points"`
571 Comments int `json:"num_comments"`
572 ObjectID string `json:"objectID"`
573 CreatedAt int64 `json:"created_at_i"`
574 } `json:"hits"`
575 }
576 if err := getJSON(ctx, d, url, &payload); err != nil {
577 return nil, err
578 }
579 out := make([]NewsItem, 0, len(payload.Hits))
580 for _, h := range payload.Hits {
581 if strings.TrimSpace(h.Title) == "" {
582 continue
583 }
584 at := time.Unix(h.CreatedAt, 0)
585 link := h.URL
586 discuss := "https://news.ycombinator.com/item?id=" + h.ObjectID
587 if link == "" {
588 link = discuss
589 }
590 out = append(out, NewsItem{
591 Source: "Hacker News", Headline: h.Title, URL: link,
592 Published: at.In(since.Location()).Format("Mon 2 Jan 15:04"),
593 Points: h.Points, Comments: h.Comments, Discuss: discuss, at: at,
594 })
595 }
596 return out, nil
597}
598
599func readLobsters(ctx context.Context, d *Deps, since, until time.Time) ([]NewsItem, error) {
600 var payload []struct {
601 Title string `json:"title"`
602 URL string `json:"url"`
603 Score int `json:"score"`
604 Comments int `json:"comment_count"`
605 ShortID string `json:"short_id_url"`
606 CreatedAt string `json:"created_at"`
607 CommentsURL string `json:"comments_url"`
608 }
609 if err := getJSON(ctx, d, "https://lobste.rs/hottest.json", &payload); err != nil {
610 return nil, err
611 }
612 var out []NewsItem
613 for _, s := range payload {
614 if s.Score < lobstersMinPoints || strings.TrimSpace(s.Title) == "" {
615 continue
616 }
617 at, ok := parseFeedTime(s.CreatedAt)
618 if !ok || at.Before(since) || !at.Before(until) {
619 continue
620 }
621 link := s.URL
622 if link == "" {
623 link = s.CommentsURL
624 }
625 out = append(out, NewsItem{
626 Source: "Lobsters", Headline: s.Title, URL: link,
627 Published: at.In(since.Location()).Format("Mon 2 Jan 15:04"),
628 Points: s.Score, Comments: s.Comments, Discuss: s.CommentsURL, at: at,
629 })
630 }
631 return out, nil
632}
633
634// Two publishers carrying one story is worth knowing and two copies of one
635// publisher's own item is not, so this drops by url and by headline and leaves
636// the rest alone.
637//
638// Both keys are checked rather than one or the other. Keying on the url alone
639// let the same NPR story through twice on 2026-09-08, because the two copies
640// came off different feeds with different tracking parameters on them, and the
641// rundown carried it as two stories in the same section.
642func dedupeNews(in []NewsItem) []NewsItem {
643 seenURL := make(map[string]bool, len(in))
644 seenHead := make(map[string]bool, len(in))
645 out := in[:0]
646 for _, it := range in {
647 u := canonicalNewsURL(it.URL)
648 h := it.Source + "\x00" + strings.ToLower(strings.TrimSpace(it.Headline))
649 if (u != "" && seenURL[u]) || seenHead[h] {
650 continue
651 }
652 if u != "" {
653 seenURL[u] = true
654 }
655 seenHead[h] = true
656 out = append(out, it)
657 }
658 return out
659}
660
661// canonicalNewsURL is the address without the parts that vary between two feeds
662// carrying the same page: the scheme, a www, the query, the fragment and a
663// trailing slash.
664func canonicalNewsURL(raw string) string {
665 if raw == "" {
666 return ""
667 }
668 u, err := neturl.Parse(raw)
669 if err != nil || u.Host == "" {
670 return strings.ToLower(strings.TrimSpace(raw))
671 }
672 host := strings.TrimPrefix(strings.ToLower(u.Host), "www.")
673 return host + strings.TrimSuffix(u.EscapedPath(), "/")
674}
675
676// headlineWords is the distinctive half of a headline, for deciding whether two
677// items are the same story written up twice. Not a real tokeniser and does not
678// need to be: it only ever compares two headlines from the same rundown.
679func headlineWords(h string) map[string]bool {
680 out := map[string]bool{}
681 for _, w := range strings.FieldsFunc(strings.ToLower(h), func(r rune) bool {
682 return !unicode.IsLetter(r) && !unicode.IsDigit(r)
683 }) {
684 if len(w) < 4 || newsStopwords[w] {
685 continue
686 }
687 out[w] = true
688 }
689 return out
690}
691
692var newsStopwords = map[string]bool{
693 "after": true, "against": true, "amid": true, "about": true, "been": true,
694 "could": true, "from": true, "have": true, "into": true, "more": true,
695 "most": true, "over": true, "said": true, "says": true, "than": true,
696 "that": true, "them": true, "they": true, "this": true, "will": true,
697 "with": true, "what": true, "when": true, "were": true, "your": true,
698}
699
700func anySameStory(seen []map[string]bool, words map[string]bool) bool {
701 for _, prev := range seen {
702 if sameStory(prev, words) {
703 return true
704 }
705 }
706 return false
707}
708
709// sameStory is true for two headlines that are one story. The threshold is
710// containment of the shorter in the longer, since a follow up headline is
711// usually the first one with more on the end.
712func sameStory(a, b map[string]bool) bool {
713 if len(a) < 3 || len(b) < 3 {
714 return false
715 }
716 shared := 0
717 for w := range a {
718 if b[w] {
719 shared++
720 }
721 }
722 if shared < 3 {
723 return false
724 }
725 small := len(a)
726 if len(b) < small {
727 small = len(b)
728 }
729 return float64(shared)/float64(small) >= 0.75
730}
731
732// What leads depends on the topic. On tech the aggregators are the story and a
733// score is the best measure of that. On general news they are not: a post about
734// converting timestamps outranking a fatal plane crash is the wrong answer to
735// "any big news today", so there the desks lead and the scored items follow.
736func sortNews(items []NewsItem, scoredFirst bool) {
737 sort.SliceStable(items, func(i, j int) bool {
738 a, b := items[i], items[j]
739 if (a.Points > 0) != (b.Points > 0) {
740 return (a.Points > 0) == scoredFirst
741 }
742 if a.Points != b.Points {
743 return a.Points > b.Points
744 }
745 if a.rank != b.rank {
746 return a.rank < b.rank
747 }
748 return a.at.After(b.at)
749 })
750}