orchard

  1package main
  2
  3// The mirror lane. Pushed repositories are the working set and mirrored ones are
  4// the backup, with one browse UI over both. Every call to GitHub is
  5// unauthenticated, so there is no access token anywhere in this site.
  6
  7import (
  8	"bytes"
  9	"context"
 10	"encoding/json"
 11	"fmt"
 12	"log/slog"
 13	"net/http"
 14	"os"
 15	"os/exec"
 16	"path/filepath"
 17	"strings"
 18	"sync"
 19	"sync/atomic"
 20	"time"
 21)
 22
 23// syncTimeout bounds one repository's fetch, whose first run is a full clone.
 24const syncTimeout = 10 * time.Minute
 25
 26// GitHubRepo is the subset of the API response this uses.
 27type GitHubRepo struct {
 28	Name     string   `json:"name"`
 29	FullName string   `json:"full_name"`
 30	CloneURL string   `json:"clone_url"`
 31	Desc     string   `json:"description"`
 32	Homepage string   `json:"homepage"`
 33	Topics   []string `json:"topics"`
 34	Archived bool     `json:"archived"`
 35	Fork     bool     `json:"fork"`
 36	Private  bool     `json:"private"`
 37	Size     int      `json:"size"`
 38	Owner    struct {
 39		Login string `json:"login"`
 40	} `json:"owner"`
 41
 42	// explicit is true when a source named this repository outright. An account
 43	// sweep skips forks; a repository asked for by name is not skipped.
 44	explicit bool
 45}
 46
 47// Mirror keeps the local copies in step with the account.
 48type Mirror struct {
 49	store *Store
 50	db    *DB
 51
 52	// Set while a manual sync is in flight, so a second click is declined rather
 53	// than parked on the mutex below.
 54	running atomic.Bool
 55
 56	// One sync at a time: two `git remote update` runs against one repository
 57	// fight over the lock file.
 58	mu sync.Mutex
 59}
 60
 61func NewMirror(store *Store, db *DB) *Mirror {
 62	return &Mirror{store: store, db: db}
 63}
 64
 65// Run syncs on a ticker until the context is cancelled.
 66func (m *Mirror) Run(ctx context.Context, every time.Duration) {
 67	// A sync at startup too, or a container that restarts daily never syncs.
 68	if err := m.Sync(ctx); err != nil {
 69		slog.Error("initial mirror sync failed", slog.Any("err", err))
 70	}
 71
 72	t := time.NewTicker(every)
 73	defer t.Stop()
 74
 75	for {
 76		select {
 77		case <-ctx.Done():
 78			return
 79		case <-t.C:
 80			if err := m.Sync(ctx); err != nil {
 81				slog.Error("mirror sync failed", slog.Any("err", err))
 82			}
 83		}
 84	}
 85}
 86
 87// Sync lists the account and brings every mirrored repository up to date.
 88func (m *Mirror) Sync(ctx context.Context) error {
 89	m.mu.Lock()
 90	defer m.mu.Unlock()
 91
 92	sources, err := m.db.MirrorSources()
 93	if err != nil {
 94		return err
 95	}
 96	if len(sources) == 0 {
 97		slog.Info("mirror sync skipped: no sources configured")
 98		return nil
 99	}
100
101	remote, err := m.list(ctx, sources)
102	if err != nil {
103		return err
104	}
105	slog.Info("mirror sync starting",
106		slog.Int("sources", len(sources)), slog.Int("upstream_repos", len(remote)))
107
108	seen := make(map[string]bool, len(remote))
109	for _, gh := range remote {
110		// A private repository needs a token this site does not have, and an
111		// account sweep skips forks unless one was named outright.
112		if gh.Private || (gh.Fork && !gh.explicit) {
113			continue
114		}
115		// Ownership of a name is decided by what is on disk, not by a database
116		// row: a bare repository with no mirror row is a pushed one, offered to
117		// adopt and skipped if it declines.
118		if repo, onDisk := m.store.Open(gh.Name); onDisk {
119			if meta, err := m.db.Repo(gh.Name); err != nil || !meta.Mirror {
120				adopted, err := m.adopt(ctx, gh, repo)
121				if err != nil {
122					slog.Error("adopt failed",
123						slog.String("repo", gh.Name), slog.Any("err", err))
124				}
125				if !adopted {
126					seen[gh.Name] = true
127					continue
128				}
129			}
130		}
131		// Directories are named for the repository alone, so two accounts can
132		// collide on one name. First claim wins and the loser is refused.
133		if meta, err := m.db.Repo(gh.Name); err == nil && meta.Mirror &&
134			meta.Upstream != "" && meta.Upstream != gh.CloneURL {
135			slog.Warn("name is already mirrored from a different upstream",
136				slog.String("repo", gh.Name),
137				slog.String("mirroring", meta.Upstream),
138				slog.String("refused", gh.CloneURL))
139			continue
140		}
141		seen[gh.Name] = true
142
143		if err := m.syncOne(ctx, gh); err != nil {
144			slog.Error("mirror repo failed",
145				slog.String("repo", gh.Name), slog.Any("err", err))
146			_ = m.db.RecordSync(gh.Name, err)
147			continue
148		}
149		_ = m.db.RecordSync(gh.Name, nil)
150	}
151
152	// Anything mirrored that upstream no longer lists is gone from GitHub. The
153	// local copy is never deleted.
154	all, err := m.db.AllRepos()
155	if err != nil {
156		return err
157	}
158	for name, meta := range all {
159		if !meta.Mirror {
160			continue
161		}
162		// A repository whose source was removed is not gone from GitHub, only
163		// no longer watched.
164		if !coveredBySource(meta.Upstream, sources) {
165			continue
166		}
167		gone := !seen[name]
168		if gone != meta.UpstreamGone {
169			if gone {
170				slog.Warn("upstream repository is gone; local mirror is now the only copy",
171					slog.String("repo", name))
172			}
173			_ = m.db.MarkUpstreamGone(name, gone)
174		}
175	}
176	return nil
177}
178
179// list reads every configured source. One source failing is logged and the rest
180// still run, so a typo does not stop the other sources being backed up.
181func (m *Mirror) list(ctx context.Context, sources []MirrorSource) ([]GitHubRepo, error) {
182	client := &http.Client{Timeout: 30 * time.Second}
183
184	var all []GitHubRepo
185	// Keyed by full_name, since a sweep and a named repository can reach the same
186	// one. An explicit entry replaces a swept one, overriding the fork skip.
187	seen := make(map[string]int)
188	add := func(gh GitHubRepo) {
189		if i, ok := seen[gh.FullName]; ok {
190			if gh.explicit {
191				all[i] = gh
192			}
193			return
194		}
195		seen[gh.FullName] = len(all)
196		all = append(all, gh)
197	}
198
199	var failed int
200	for _, src := range sources {
201		var err error
202		switch src.Kind {
203		case sourceRepo:
204			var gh GitHubRepo
205			if err = getJSON(ctx, client,
206				fmt.Sprintf("https://api.github.com/repos/%s/%s", src.Owner, src.Name),
207				&gh); err == nil {
208				gh.explicit = true
209				add(gh)
210			}
211		default:
212			var repos []GitHubRepo
213			if repos, err = m.listAccount(ctx, client, src.Owner); err == nil {
214				for _, gh := range repos {
215					add(gh)
216				}
217			}
218		}
219		if err != nil {
220			failed++
221			slog.Error("mirror source failed",
222				slog.String("source", src.Label()), slog.Any("err", err))
223		}
224	}
225
226	if len(all) == 0 && failed > 0 {
227		return nil, fmt.Errorf("every mirror source failed (%d)", failed)
228	}
229	return all, nil
230}
231
232// listAccount pages through one account's public repositories.
233func (m *Mirror) listAccount(ctx context.Context, client *http.Client, owner string) ([]GitHubRepo, error) {
234	var all []GitHubRepo
235	for page := 1; page <= 10; page++ {
236		url := fmt.Sprintf(
237			"https://api.github.com/users/%s/repos?per_page=100&type=owner&page=%d",
238			owner, page)
239
240		var got []GitHubRepo
241		if err := getJSON(ctx, client, url, &got); err != nil {
242			return nil, err
243		}
244		all = append(all, got...)
245		if len(got) < 100 {
246			break
247		}
248	}
249	return all, nil
250}
251
252// getJSON is one unauthenticated GitHub call, rate limited by IP at 60 an hour.
253// The API requires a User-Agent and answers a missing one with a 403.
254func getJSON(ctx context.Context, client *http.Client, url string, out any) error {
255	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
256	if err != nil {
257		return err
258	}
259	req.Header.Set("Accept", "application/vnd.github+json")
260	req.Header.Set("User-Agent", "repos.bythewood.me")
261
262	resp, err := client.Do(req)
263	if err != nil {
264		return err
265	}
266	defer resp.Body.Close()
267
268	if resp.StatusCode != http.StatusOK {
269		return fmt.Errorf("github returned %d for %s", resp.StatusCode, url)
270	}
271	if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
272		return fmt.Errorf("decode %s: %w", url, err)
273	}
274	return nil
275}
276
277// coveredBySource reports whether a mirrored repository still belongs to a
278// configured source, which scopes the upstream_gone check.
279func coveredBySource(upstream string, sources []MirrorSource) bool {
280	path := strings.TrimSuffix(strings.TrimPrefix(upstream, "https://github.com/"), ".git")
281	owner, name, ok := strings.Cut(path, "/")
282	if !ok {
283		return false
284	}
285	for _, src := range sources {
286		if !strings.EqualFold(src.Owner, owner) {
287			continue
288		}
289		if src.Kind == sourceAccount || strings.EqualFold(src.Name, name) {
290			return true
291		}
292	}
293	return false
294}
295
296// adopt converts a pushed repository into a mirror in place, but only if upstream
297// already contains every local branch and matches every tag exactly. The probe
298// fetch writes to refs/adopt/*, so a repository that fails is left untouched.
299func (m *Mirror) adopt(ctx context.Context, gh GitHubRepo, repo Repo) (bool, error) {
300	ctx, cancel := context.WithTimeout(ctx, syncTimeout)
301	defer cancel()
302
303	// Removing origin first makes this safe to re-run after a probe that failed
304	// partway.
305	_, _ = run(ctx, repo, "remote", "remove", "origin")
306	if _, err := run(ctx, repo, "remote", "add", "origin", gh.CloneURL); err != nil {
307		return false, err
308	}
309
310	// Without --no-tags the fetch follows tags into refs/tags/*, which are real
311	// refs, and the probe stops being read-only.
312	cmd := gitCmd(ctx, repo, "fetch", "--no-tags", "--prune", "origin",
313		"+refs/heads/*:refs/adopt/heads/*", "+refs/tags/*:refs/adopt/tags/*")
314	var stderr bytes.Buffer
315	cmd.Stderr = &stderr
316	if err := cmd.Run(); err != nil {
317		_, _ = run(ctx, repo, "remote", "remove", "origin")
318		return false, fmt.Errorf("probe fetch %s: %w: %s",
319			gh.Name, err, strings.TrimSpace(stderr.String()))
320	}
321
322	upstream, err := refMap(ctx, repo, "refs/adopt")
323	if err != nil {
324		return false, err
325	}
326	local, err := refMap(ctx, repo, "refs/heads", "refs/tags")
327	if err != nil {
328		return false, err
329	}
330
331	ok := true
332	for name, sha := range local {
333		// refs/heads/main is probed as refs/adopt/heads/main.
334		up, found := upstream["refs/adopt/"+strings.TrimPrefix(name, "refs/")]
335		switch {
336		case !found:
337			slog.Warn("not adopting: ref is not on upstream",
338				slog.String("repo", gh.Name), slog.String("ref", name))
339			ok = false
340		case up == sha:
341		case !strings.HasPrefix(name, "refs/heads/"):
342			slog.Warn("not adopting: tag differs from upstream",
343				slog.String("repo", gh.Name), slog.String("ref", name))
344			ok = false
345		default:
346			// --is-ancestor is a predicate: the exit status is the answer.
347			if err := gitCmd(ctx, repo,
348				"merge-base", "--is-ancestor", sha, up).Run(); err != nil {
349				slog.Warn("not adopting: local commits are not on upstream",
350					slog.String("repo", gh.Name), slog.String("ref", name))
351				ok = false
352			}
353		}
354	}
355
356	// The probe refs come out either way.
357	for name := range upstream {
358		if _, err := run(ctx, repo, "update-ref", "-d", name); err != nil {
359			return false, err
360		}
361	}
362	if !ok {
363		_, _ = run(ctx, repo, "remote", "remove", "origin")
364		return false, nil
365	}
366
367	// Only now does origin become a mirror remote, which is what makes the
368	// `remote update --prune` in syncOne rewrite refs rather than add to them.
369	for _, kv := range [][2]string{
370		{"remote.origin.fetch", "+refs/*:refs/*"},
371		{"remote.origin.mirror", "true"},
372		// The same backup settings clone gives a fresh mirror.
373		{"core.logAllRefUpdates", "true"},
374		{"gc.reflogExpire", "never"},
375		{"gc.reflogExpireUnreachable", "never"},
376		{"receive.autogc", "false"},
377	} {
378		if _, err := run(ctx, repo, "config", kv[0], kv[1]); err != nil {
379			return false, err
380		}
381	}
382
383	slog.Info("adopted pushed repository as a mirror",
384		slog.String("repo", gh.Name), slog.String("upstream", gh.CloneURL))
385	return true, nil
386}
387
388// refMap reads refs under the given prefixes as full refname to object id.
389func refMap(ctx context.Context, repo Repo, prefixes ...string) (map[string]string, error) {
390	args := append([]string{"for-each-ref", "--format=%(objectname)%00%(refname)"}, prefixes...)
391	out, err := run(ctx, repo, args...)
392	if err != nil {
393		return nil, err
394	}
395	refs := make(map[string]string)
396	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
397		if line == "" {
398			continue
399		}
400		sha, name, found := strings.Cut(line, "\x00")
401		if !found {
402			continue
403		}
404		refs[name] = sha
405	}
406	return refs, nil
407}
408
409// syncOne clones or updates one mirror.
410func (m *Mirror) syncOne(ctx context.Context, gh GitHubRepo) error {
411	if !validName(gh.Name) {
412		return fmt.Errorf("upstream name is not usable here: %q", gh.Name)
413	}
414
415	ctx, cancel := context.WithTimeout(ctx, syncTimeout)
416	defer cancel()
417
418	path := filepath.Join(m.store.Root, gh.Name+".git")
419
420	if _, err := os.Stat(path); os.IsNotExist(err) {
421		if err := m.clone(ctx, gh, path); err != nil {
422			return err
423		}
424	} else {
425		repo := Repo{Name: gh.Name, Path: path}
426		// --prune propagates an upstream deletion, which is why every mirror is
427		// created with a never-expiring reflog. gitCmd rather than run, because
428		// run imposes gitTimeout, which is far too short for a fetch.
429		cmd := gitCmd(ctx, repo, "remote", "update", "--prune")
430		var stderr bytes.Buffer
431		cmd.Stderr = &stderr
432		if err := cmd.Run(); err != nil {
433			return fmt.Errorf("remote update %s: %w: %s",
434				gh.Name, err, strings.TrimSpace(stderr.String()))
435		}
436	}
437
438	m.store.InvalidateOverview(gh.Name)
439
440	if err := m.db.MarkMirror(gh.Name, gh.CloneURL, gh.Archived); err != nil {
441		return err
442	}
443	// Upstream owns a mirror's description and topics, and the UI will not edit
444	// them, so overwriting on each sync is correct.
445	return m.db.SetDescription(gh.Name, gh.Desc, gh.Topics, gh.Homepage)
446}
447
448func (m *Mirror) clone(ctx context.Context, gh GitHubRepo, path string) error {
449	slog.Info("cloning mirror",
450		slog.String("repo", gh.Name), slog.Int("upstream_kb", gh.Size))
451
452	cmd := exec.CommandContext(ctx, "git", "clone", "--mirror", gh.CloneURL, path)
453	cmd.Env = append(os.Environ(),
454		"GIT_TERMINAL_PROMPT=0", "GIT_CONFIG_NOSYSTEM=1", "HOME="+os.TempDir())
455
456	if out, err := cmd.CombinedOutput(); err != nil {
457		return fmt.Errorf("clone %s: %w: %s", gh.Name, err, strings.TrimSpace(string(out)))
458	}
459
460	// core.logAllRefUpdates is off by default in a bare repository, and it is what
461	// leaves an upstream force push's old tip recoverable in the reflog.
462	repo := Repo{Name: gh.Name, Path: path}
463	for _, kv := range [][2]string{
464		{"core.logAllRefUpdates", "true"},
465		{"gc.reflogExpire", "never"},
466		{"gc.reflogExpireUnreachable", "never"},
467		{"receive.autogc", "false"},
468	} {
469		if _, err := run(ctx, repo, "config", kv[0], kv[1]); err != nil {
470			return err
471		}
472	}
473	return nil
474}
475
476// RunGC repacks every repository on a ticker, which is what makes
477// receive.autogc=false safe to set.
478func RunGC(ctx context.Context, store *Store, every time.Duration) {
479	t := time.NewTicker(every)
480	defer t.Stop()
481
482	for {
483		select {
484		case <-ctx.Done():
485			return
486		case <-t.C:
487			repos, err := store.Discover()
488			if err != nil {
489				slog.Error("gc: discover failed", slog.Any("err", err))
490				continue
491			}
492			for _, repo := range repos {
493				if err := store.GC(ctx, repo); err != nil {
494					slog.Error("gc failed",
495						slog.String("repo", repo.Name), slog.Any("err", err))
496				}
497			}
498		}
499	}
500}
501
502// TrySync runs a sync in the background and reports whether it started one. It
503// declines rather than queues, so two clicks do not mean two syncs.
504func (m *Mirror) TrySync(ctx context.Context) bool {
505	if !m.running.CompareAndSwap(false, true) {
506		return false
507	}
508	go func() {
509		defer m.running.Store(false)
510		if err := m.Sync(ctx); err != nil {
511			slog.Error("manual mirror sync failed", slog.Any("err", err))
512		}
513	}()
514	return true
515}