orchard

  1package tools
  2
  3// Searching X posts without an X account, an API key, or a mirror.
  4//
  5// Isaac has asked for this twice. The two obvious routes are both closed:
  6// xcancel has no API of its own and was served a cease and desist by X Corp on
  7// 24 August 2026, and Nitter is the thing xcancel is built on, needs Redis and
  8// a server of your own, and is under the same letter. The official Posts Search
  9// API is real and works and is a paid tier with a key.
 10//
 11// What is left is the search engines, which index public post pages like any
 12// other page. That is what this does, and it is worth being plain about the
 13// limits rather than presenting it as X search: only public posts are indexed,
 14// only some of those, and a search engine's copy can be older than the post.
 15// The description says so, so an answer built on it can say so too.
 16//
 17// It costs a DuckDuckGo call out of the same pool everything else uses, which
 18// is the point of building it here rather than reaching for another host.
 19
 20import (
 21	"context"
 22	"fmt"
 23	"net/url"
 24	"strings"
 25)
 26
 27var XSearch = Tool{
 28	Name: "x_search",
 29	Description: "Find public posts on X, formerly Twitter, through a web search of x.com. " +
 30		"Use it when Isaac asks what someone posted or what is being said on X. " +
 31		"It reads what a search engine has indexed rather than X itself, so it covers public " +
 32		"posts only, misses plenty of them, and can hand back a copy older than the post. " +
 33		"Say that when the answer rests on it. There is no free X API and this is the way in.",
 34	Schema: obj(map[string]any{
 35		"query":   str("what to look for, as a person would type it"),
 36		"account": str("one account to search within, with or without the @, optional"),
 37		"n":       integer("how many posts to return, default 6"),
 38	}, "query"),
 39	Run: func(ctx context.Context, d *Deps, a map[string]any) (any, error) {
 40		q := strings.TrimSpace(argStr(a, "query"))
 41		if q == "" {
 42			return nil, fmt.Errorf("query is required")
 43		}
 44		n := int(argNum(a, "n", 6))
 45		if n < 1 || n > 12 {
 46			n = 6
 47		}
 48
 49		// Both hostnames, since old posts are indexed under twitter.com and new
 50		// ones under x.com, and a search for one alone misses half of them.
 51		scope := "(site:x.com OR site:twitter.com)"
 52		if acct := strings.TrimSpace(strings.TrimPrefix(argStr(a, "account"), "@")); acct != "" {
 53			scope = "(site:x.com/" + acct + " OR site:twitter.com/" + acct + ")"
 54		}
 55
 56		body, err := get(ctx, d, "https://html.duckduckgo.com/html/?q="+
 57			url.QueryEscape(scope+" "+q), "text/html")
 58		if err != nil {
 59			return nil, err
 60		}
 61		hits := parseDDG(string(body), 0)
 62
 63		posts := make([]SearchHit, 0, n)
 64		for _, h := range hits {
 65			if !isXPost(h.URL) {
 66				continue
 67			}
 68			h.URL = asXCom(h.URL)
 69			posts = append(posts, h)
 70			if len(posts) >= n {
 71				break
 72			}
 73		}
 74		if len(posts) == 0 {
 75			return nil, fmt.Errorf("no indexed posts matched that. Search engines carry only part " +
 76				"of X and there is no free API, so say that rather than that nothing was posted")
 77		}
 78		return map[string]any{
 79			"query": q, "posts": posts, "count": len(posts),
 80			"note": "These come from a search engine's index of x.com, not from X. It holds public " +
 81				"posts only and not all of them, and a snippet can be older than the post. Do not " +
 82				"present this as a complete or current picture of what is on X.",
 83		}, nil
 84	},
 85}
 86
 87// isXPost keeps the status pages and drops the profile, hashtag and help pages
 88// a site search also returns.
 89func isXPost(raw string) bool {
 90	u, err := url.Parse(raw)
 91	if err != nil {
 92		return false
 93	}
 94	host := strings.TrimPrefix(strings.ToLower(u.Host), "www.")
 95	if host != "x.com" && host != "twitter.com" && host != "mobile.twitter.com" {
 96		return false
 97	}
 98	return strings.Contains(u.Path, "/status/")
 99}
100
101// asXCom rewrites a twitter.com address to the hostname that still resolves, so
102// every link in one answer goes to the same place.
103func asXCom(raw string) string {
104	u, err := url.Parse(raw)
105	if err != nil {
106		return raw
107	}
108	switch strings.TrimPrefix(strings.ToLower(u.Host), "www.") {
109	case "twitter.com", "mobile.twitter.com":
110		u.Host = "x.com"
111	}
112	return u.String()
113}