orchard

  1package main
  2
  3import (
  4	"context"
  5	"encoding/xml"
  6	"fmt"
  7	"io"
  8	"net/http"
  9	"slices"
 10	"sort"
 11	"strings"
 12	"time"
 13)
 14
 15// The day's major headlines from NPR and BBC, newest first, and nothing else.
 16// It was a market news panel before and then a four bucket balance across six
 17// outlets, and Isaac's read on both was that they were confusing to look at,
 18// so this reads like a wire and is sorted the way a wire is.
 19//
 20// These three feeds are the editor picked ones rather than a topic list, since
 21// what lands on a front page is the closest thing an RSS feed has to a signal
 22// that a story is big. BBC's own top stories feed is the UK edition and leads
 23// on the Budget and Reform UK, so the two regional editions are the American
 24// reader's version of it.
 25//
 26// The endpoint is the guard bucket, so one outlet going down or answering 429
 27// costs its own headlines and leaves the other its budget.
 28var wireFeeds = []struct{ name, endpoint, url string }{
 29	{"NPR", "npr", "https://feeds.npr.org/1001/rss.xml"},
 30	{"BBC", "bbc", "https://feeds.bbci.co.uk/news/world/us_and_canada/rss.xml"},
 31	{"BBC", "bbc", "https://feeds.bbci.co.uk/news/world/rss.xml"},
 32}
 33
 34const (
 35	wireEvery = 10 * time.Minute
 36	wireShown = 10
 37
 38	// At most this many rows from one outlet. BBC supplies two of the three
 39	// feeds and posts about twice as often as NPR does, so without this a busy
 40	// afternoon is a column of BBC.
 41	wirePerSource = 6
 42
 43	// Both outlets post enough that ten rows never reach back this far, so
 44	// this is the floor that keeps a dead feed off the panel rather than a
 45	// window anything is chosen inside.
 46	wireMaxAge = 2 * 24 * time.Hour
 47)
 48
 49// Headline is one story on the wire panel.
 50type Headline struct {
 51	Title  string `json:"title"`
 52	URL    string `json:"url"`
 53	Source string `json:"source"`
 54	Age    string `json:"age"`
 55}
 56
 57type rssFeed struct {
 58	Channel struct {
 59		Items []struct {
 60			Title   string `xml:"title"`
 61			Link    string `xml:"link"`
 62			PubDate string `xml:"pubDate"`
 63			GUID    string `xml:"guid"`
 64		} `xml:"item"`
 65	} `xml:"channel"`
 66}
 67
 68// dated is a headline with the timestamp the sort needs, which the panel shows
 69// only as an age.
 70type dated struct {
 71	Headline
 72	at time.Time
 73}
 74
 75func fetchWire(ctx context.Context, g *Guard, now time.Time) ([]Headline, error) {
 76	seen := map[string]bool{}
 77	var all []dated
 78
 79	for _, feed := range wireFeeds {
 80		items, err := fetchRSS(ctx, g, feed.endpoint, feed.url)
 81		if err != nil {
 82			// One dead feed costs its own headlines and not the panel.
 83			continue
 84		}
 85
 86		for _, it := range items.Channel.Items {
 87			title := strings.TrimSpace(it.Title)
 88			if title == "" || it.Link == "" {
 89				continue
 90			}
 91			if promotional(title) || sidebar(title) || clip(it.Link) {
 92				continue
 93			}
 94
 95			key := it.GUID
 96			if key == "" {
 97				key = it.Link
 98			}
 99			// BBC's world and US feeds overlap by about half, and the same
100			// story reaches two outlets under headlines that differ by a word,
101			// so the title is deduped as well as the identifier.
102			if seen[key] || seen[titleKey(title)] {
103				continue
104			}
105			seen[key], seen[titleKey(title)] = true, true
106
107			at, err := parseRSSTime(it.PubDate)
108			if err != nil || now.Sub(at) > wireMaxAge {
109				continue
110			}
111
112			all = append(all, dated{
113				Headline: Headline{
114					Title:  title,
115					URL:    it.Link,
116					Source: feed.name,
117					Age:    humanAge(at, now),
118				},
119				at: at,
120			})
121		}
122	}
123
124	if len(all) == 0 {
125		return nil, fmt.Errorf("wire: nothing fresh in %d feeds", len(wireFeeds))
126	}
127
128	sort.SliceStable(all, func(i, j int) bool { return all[i].at.After(all[j].at) })
129	return pick(dedupe(all)), nil
130}
131
132// dedupe drops the second telling of a story two outlets worded differently,
133// which the title key cannot catch. It runs over the sorted list so the copy
134// that survives is the newer one.
135func dedupe(all []dated) []dated {
136	kept := make([]dated, 0, len(all))
137	keys := make([][]string, 0, len(all))
138
139	for _, d := range all {
140		w := significant(d.Title)
141		if slices.ContainsFunc(keys, func(k []string) bool { return sameStory(k, w) }) {
142			continue
143		}
144		kept = append(kept, d)
145		keys = append(keys, w)
146	}
147	return kept
148}
149
150// pick takes the newest rows in order, holding one outlet to its cap, and then
151// gives the leftover slots back to whatever the cap passed over, since a short
152// panel is worse than a lopsided one. Both passes mark rows rather than emit
153// them, because the panel still has to read newest first afterwards.
154func pick(all []dated) []Headline {
155	taken := make([]bool, len(all))
156	perSource := map[string]int{}
157	count := 0
158
159	for i, d := range all {
160		if count == wireShown {
161			break
162		}
163		if perSource[d.Source] == wirePerSource {
164			continue
165		}
166		taken[i], perSource[d.Source], count = true, perSource[d.Source]+1, count+1
167	}
168
169	for i := range all {
170		if count == wireShown {
171			break
172		}
173		if !taken[i] {
174			taken[i], count = true, count+1
175		}
176	}
177
178	out := make([]Headline, 0, count)
179	for i, d := range all {
180		if taken[i] {
181			out = append(out, d.Headline)
182		}
183	}
184	return out
185}
186
187// Four words in common is enough to be one story, and rare enough that two
188// stories about the same person on the same day do not collide.
189const sameStoryWords = 4
190
191// The words long enough to carry a story and common enough to say nothing.
192var filler = []string{"about", "after", "against", "amid", "been", "before", "could", "does", "during", "from", "have", "into", "more", "over", "said", "says", "than", "that", "their", "them", "then", "there", "these", "they", "this", "were", "what", "when", "which", "will", "with", "would", "your"}
193
194// significant reduces a headline to the words worth comparing across outlets,
195// which is the long ones with the filler taken out. They are cut to a stem
196// because one outlet writes charged where the other writes charges.
197func significant(title string) []string {
198	var out []string
199	for _, w := range strings.FieldsFunc(strings.ToLower(title), func(r rune) bool {
200		return !(r >= 'a' && r <= 'z' || r >= '0' && r <= '9')
201	}) {
202		if len(w) < 4 || slices.Contains(filler, w) {
203			continue
204		}
205		if len(w) > 5 {
206			w = w[:5]
207		}
208		if !slices.Contains(out, w) {
209			out = append(out, w)
210		}
211	}
212	return out
213}
214
215func sameStory(a, b []string) bool {
216	shared := 0
217	for _, w := range a {
218		if slices.Contains(b, w) {
219			shared++
220		}
221	}
222	return shared >= sameStoryWords
223}
224
225// promoted is the genre Isaac asked to keep off the panel: advertising that
226// reads as news in a feed. Matching on the headline is the only lever
227// available, since none of these feeds mark it.
228var promoted = []string{
229	"promo code",
230	"sponsored",
231	"deal of the day",
232	"prime day",
233	"black friday",
234	"sign up for",
235	"subscribe to",
236	"newsletter",
237}
238
239// sidebars are the formats that are a page rather than a story, a minute of
240// footage or a rolling feed, and ten rows cannot spare a slot for one.
241// A question mark anywhere in a headline is the explainer beside the story.
242var sidebars = []string{
243	"watch:",
244	"watch live",
245	"listen:",
246	"in pictures",
247	"in photos",
248	"video:",
249	"live updates",
250	"your questions answered",
251}
252
253// clip drops BBC's video and live pages, which its news feeds mix in with the
254// stories.
255func clip(link string) bool {
256	return strings.Contains(link, "/news/videos/") || strings.Contains(link, "/news/live/")
257}
258
259func promotional(title string) bool {
260	lower := strings.ToLower(title)
261	return slices.ContainsFunc(promoted, func(p string) bool { return strings.Contains(lower, p) })
262}
263
264func sidebar(title string) bool {
265	lower := strings.ToLower(title)
266	if strings.Contains(lower, "?") {
267		return true
268	}
269	return slices.ContainsFunc(sidebars, func(p string) bool { return strings.Contains(lower, p) })
270}
271
272// titleKey reduces a headline to what two outlets covering one story share, so
273// punctuation and a trailing outlet name do not defeat the dedupe.
274func titleKey(title string) string {
275	var b strings.Builder
276	for _, r := range strings.ToLower(title) {
277		if r >= 'a' && r <= 'z' || r >= '0' && r <= '9' {
278			b.WriteRune(r)
279		}
280	}
281	key := b.String()
282	// Long enough to be the story and short enough that a differing tail does
283	// not make two copies of it look like two stories.
284	if len(key) > 60 {
285		key = key[:60]
286	}
287	return key
288}
289
290// fetchRSS is the XML twin of getJSON, guarded the same way. It is separate
291// rather than a flag on that function because the decode differs and nothing
292// else about the two is worth sharing.
293func fetchRSS(ctx context.Context, g *Guard, endpoint, url string) (*rssFeed, error) {
294	// Feeds are fetched back to back, so this has to wait out the pace between
295	// two on the same endpoint rather than lose the second one.
296	if err := g.Reserve(ctx, endpoint); err != nil {
297		return nil, err
298	}
299
300	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
301	if err != nil {
302		return nil, err
303	}
304	req.Header.Set("User-Agent", feedAgent)
305	req.Header.Set("Accept", "application/rss+xml, application/xml, text/xml")
306
307	resp, err := client.Do(req)
308	if err != nil {
309		g.Fail(endpoint, 0, 0)
310		return nil, err
311	}
312	defer resp.Body.Close()
313
314	if resp.StatusCode != http.StatusOK {
315		g.Fail(endpoint, resp.StatusCode, parseRetryAfter(resp.Header.Get("Retry-After")))
316		return nil, fmt.Errorf("%s: http %d", endpoint, resp.StatusCode)
317	}
318
319	var feed rssFeed
320	if err := xml.NewDecoder(io.LimitReader(resp.Body, maxBody)).Decode(&feed); err != nil {
321		g.Fail(endpoint, resp.StatusCode, 0)
322		return nil, fmt.Errorf("%s: %w", endpoint, err)
323	}
324
325	g.Succeed(endpoint)
326	return &feed, nil
327}
328
329// parseRSSTime reads the several date formats RSS feeds actually ship, rather
330// than the one RFC 822 says they should.
331func parseRSSTime(v string) (time.Time, error) {
332	v = strings.TrimSpace(v)
333	layouts := []string{
334		time.RFC1123Z,
335		time.RFC1123,
336		time.RFC822Z,
337		time.RFC822,
338		time.RFC3339,
339		"Mon, 2 Jan 2006 15:04:05 MST",
340		"Mon, 2 Jan 2006 15:04:05 -0700",
341	}
342	for _, l := range layouts {
343		if t, err := time.Parse(l, v); err == nil {
344			return t, nil
345		}
346	}
347	return time.Time{}, fmt.Errorf("unrecognised date %q", v)
348}