mirror of
https://github.com/lukaszraczylo/ghtracker.git
synced 2026-10-17 01:17:48 +00:00
feat: add GitHub repository dashboard with Prometheus metrics
ghtracker authenticates as a GitHub App and scans a configured list of repositories on a schedule. It serves a dashboard, a JSON state endpoint, a health probe and Prometheus metrics. The dashboard lists only items that need attention: failing workflows, pull requests that wait too long or fail their checks, stale issues and projects that failed to scan. It has search, dynamic filters and a button that rescans on demand. The Vue app is embedded in the Go binary. The client protects the GitHub rate limit with conditional requests, a request reserve and Retry-After handling.
This commit is contained in:
@@ -0,0 +1,220 @@
|
||||
// Package collector refreshes repository data on a schedule and evaluates health from it.
|
||||
package collector
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"slices"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/ghtracker/internal/config"
|
||||
"github.com/lukaszraczylo/ghtracker/internal/model"
|
||||
)
|
||||
|
||||
const repoFetchTimeout = 5 * time.Minute
|
||||
|
||||
// Fetcher loads the data for one "owner/name" repository.
|
||||
type Fetcher interface {
|
||||
FetchRepo(ctx context.Context, full string) (model.Repo, error)
|
||||
}
|
||||
|
||||
// View is a consistent, health-evaluated read of the collector state.
|
||||
type View struct {
|
||||
Now time.Time
|
||||
LastRefresh time.Time
|
||||
Repos []model.Repo
|
||||
Alerts []model.Alert
|
||||
Thresholds config.Thresholds
|
||||
Interval time.Duration
|
||||
Crit int
|
||||
Warn int
|
||||
Loaded bool
|
||||
Refreshing bool
|
||||
}
|
||||
|
||||
// Stats are monotonic counters and timings of the refresh loop.
|
||||
type Stats struct {
|
||||
LastRefresh time.Time
|
||||
LastDuration time.Duration
|
||||
RefreshTotal uint64
|
||||
RepoErrorsTotal uint64
|
||||
Loaded bool
|
||||
RefreshInFlight bool
|
||||
}
|
||||
|
||||
type Collector struct {
|
||||
lastRefresh time.Time
|
||||
fetch Fetcher
|
||||
repos map[string]model.Repo
|
||||
now func() time.Time
|
||||
log *slog.Logger
|
||||
kick chan struct{}
|
||||
names []string
|
||||
th config.Thresholds
|
||||
interval time.Duration
|
||||
refreshes atomic.Uint64
|
||||
repoErrs atomic.Uint64
|
||||
lastDur time.Duration
|
||||
concurrency int
|
||||
mu sync.RWMutex
|
||||
limited atomic.Bool
|
||||
refreshing atomic.Bool
|
||||
loaded bool
|
||||
}
|
||||
|
||||
func New(f Fetcher, cfg *config.Config, log *slog.Logger) *Collector {
|
||||
return &Collector{
|
||||
fetch: f,
|
||||
names: cfg.Repos,
|
||||
th: cfg.Thresholds,
|
||||
interval: cfg.RefreshInterval,
|
||||
concurrency: cfg.Concurrency,
|
||||
now: time.Now,
|
||||
log: log,
|
||||
kick: make(chan struct{}, 1),
|
||||
repos: make(map[string]model.Repo, len(cfg.Repos)),
|
||||
}
|
||||
}
|
||||
|
||||
// Run refreshes immediately, then on every interval or kick, until ctx ends.
|
||||
func (c *Collector) Run(ctx context.Context) {
|
||||
t := time.NewTicker(c.interval)
|
||||
defer t.Stop()
|
||||
c.Refresh(ctx)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
c.Refresh(ctx)
|
||||
case <-c.kick:
|
||||
c.Refresh(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// RequestRefresh asks Run for an extra refresh; it reports false when one is already running or queued.
|
||||
func (c *Collector) RequestRefresh() bool {
|
||||
if c.refreshing.Load() {
|
||||
return false
|
||||
}
|
||||
select {
|
||||
case c.kick <- struct{}{}:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Refresh fetches every repo once; a failing repo keeps its last good data and is marked down.
|
||||
func (c *Collector) Refresh(ctx context.Context) {
|
||||
if !c.refreshing.CompareAndSwap(false, true) {
|
||||
return
|
||||
}
|
||||
defer c.refreshing.Store(false)
|
||||
|
||||
start := c.now()
|
||||
c.limited.Store(false)
|
||||
results := make([]model.Repo, len(c.names))
|
||||
sem := make(chan struct{}, c.concurrency)
|
||||
var wg sync.WaitGroup
|
||||
for i, name := range c.names {
|
||||
wg.Add(1)
|
||||
sem <- struct{}{}
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
results[i] = c.refreshOne(ctx, name, start)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
c.mu.Lock()
|
||||
for _, r := range results {
|
||||
c.repos[r.FullName] = r
|
||||
}
|
||||
c.lastRefresh = start
|
||||
c.lastDur = c.now().Sub(start)
|
||||
c.loaded = true
|
||||
c.mu.Unlock()
|
||||
c.refreshes.Add(1)
|
||||
c.log.Info("refresh complete", "repos", len(c.names), "duration", c.lastDur.Round(time.Millisecond))
|
||||
}
|
||||
|
||||
func (c *Collector) refreshOne(ctx context.Context, name string, at time.Time) model.Repo {
|
||||
ctx, cancel := context.WithTimeout(ctx, repoFetchTimeout)
|
||||
defer cancel()
|
||||
var r model.Repo
|
||||
err := model.ErrRateLimited
|
||||
// Once any repo hits the limit, remaining repos are skipped so no further requests go out.
|
||||
if !c.limited.Load() {
|
||||
r, err = c.fetch.FetchRepo(ctx, name)
|
||||
}
|
||||
if errors.Is(err, model.ErrRateLimited) {
|
||||
c.limited.Store(true)
|
||||
}
|
||||
if err == nil {
|
||||
r.FullName, r.Up, r.RefreshedAt = name, true, at
|
||||
return r
|
||||
}
|
||||
c.repoErrs.Add(1)
|
||||
c.log.Error("repo refresh failed", "repo", name, "err", err)
|
||||
c.mu.RLock()
|
||||
prev, ok := c.repos[name]
|
||||
c.mu.RUnlock()
|
||||
if !ok {
|
||||
prev = model.Repo{FullName: name, URL: "https://github.com/" + name}
|
||||
}
|
||||
prev.Up, prev.Err = false, err.Error()
|
||||
return prev
|
||||
}
|
||||
|
||||
// Stats returns refresh-loop counters.
|
||||
func (c *Collector) Stats() Stats {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
return Stats{
|
||||
LastRefresh: c.lastRefresh,
|
||||
LastDuration: c.lastDur,
|
||||
RefreshTotal: c.refreshes.Load(),
|
||||
RepoErrorsTotal: c.repoErrs.Load(),
|
||||
Loaded: c.loaded,
|
||||
RefreshInFlight: c.refreshing.Load(),
|
||||
}
|
||||
}
|
||||
|
||||
// View evaluates all repos at now. Repos not yet fetched appear with no data and are not alerted on.
|
||||
func (c *Collector) View() View {
|
||||
now := c.now()
|
||||
c.mu.RLock()
|
||||
v := View{
|
||||
Now: now, LastRefresh: c.lastRefresh, Interval: c.interval, Thresholds: c.th,
|
||||
Loaded: c.loaded, Refreshing: c.refreshing.Load(),
|
||||
}
|
||||
v.Repos = make([]model.Repo, 0, len(c.names))
|
||||
for _, n := range c.names {
|
||||
if r, ok := c.repos[n]; ok {
|
||||
v.Repos = append(v.Repos, r)
|
||||
}
|
||||
}
|
||||
c.mu.RUnlock()
|
||||
|
||||
for i := range v.Repos {
|
||||
v.Alerts = append(v.Alerts, Evaluate(&v.Repos[i], now, c.th)...)
|
||||
}
|
||||
sortAlerts(v.Alerts)
|
||||
for _, a := range v.Alerts {
|
||||
switch a.Severity {
|
||||
case model.Crit:
|
||||
v.Crit++
|
||||
case model.Warn:
|
||||
v.Warn++
|
||||
}
|
||||
}
|
||||
// Worst repos first so problems surface at the top of the dashboard.
|
||||
slices.SortStableFunc(v.Repos, func(a, b model.Repo) int { return int(b.Health) - int(a.Health) })
|
||||
return v
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
package collector
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/ghtracker/internal/config"
|
||||
"github.com/lukaszraczylo/ghtracker/internal/model"
|
||||
)
|
||||
|
||||
var t0 = time.Date(2026, 10, 2, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
func th() config.Thresholds {
|
||||
return config.Thresholds{
|
||||
PRWarnAfter: 48 * time.Hour, PRCritAfter: 7 * 24 * time.Hour,
|
||||
IssueStaleAfter: 30 * 24 * time.Hour, WorkflowStuck: 6 * time.Hour, WorkflowDormant: 30 * 24 * time.Hour, DataStaleAfter: 13 * time.Hour,
|
||||
}
|
||||
}
|
||||
|
||||
func kinds(a []model.Alert) map[string]model.Severity {
|
||||
m := map[string]model.Severity{}
|
||||
for _, x := range a {
|
||||
m[x.Kind] = max(m[x.Kind], x.Severity)
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func TestEvaluate(t *testing.T) {
|
||||
fresh := func() model.Repo {
|
||||
return model.Repo{FullName: "o/r", URL: "https://github.com/o/r", Up: true, RefreshedAt: t0}
|
||||
}
|
||||
failedRun := &model.Run{Status: "completed", Conclusion: "failure", URL: "run"}
|
||||
tests := map[string]struct {
|
||||
mutate func(*model.Repo)
|
||||
want map[string]model.Severity
|
||||
health model.Severity
|
||||
}{
|
||||
"healthy": {func(*model.Repo) {}, map[string]model.Severity{}, model.OK},
|
||||
"refresh failed": {func(r *model.Repo) { r.Err = "boom"; r.Up = false }, map[string]model.Severity{KindRefreshFailed: model.Crit}, model.Crit},
|
||||
"data stale": {func(r *model.Repo) { r.RefreshedAt = t0.Add(-14 * time.Hour) }, map[string]model.Severity{KindDataStale: model.Warn}, model.Warn},
|
||||
"partial data": {func(r *model.Repo) { r.Warnings = []string{"workflows: 403"} }, map[string]model.Severity{KindPartialData: model.Warn}, model.Warn},
|
||||
"workflow failed": {func(r *model.Repo) {
|
||||
r.Workflows = []model.Workflow{{Name: "CI", Latest: *failedRun, LastDone: failedRun}}
|
||||
}, map[string]model.Severity{KindWorkflowFail: model.Crit}, model.Crit},
|
||||
"workflow rerun in progress after failure still critical": {func(r *model.Repo) {
|
||||
r.Workflows = []model.Workflow{{Name: "CI", Latest: model.Run{Status: "in_progress", CreatedAt: t0}, LastDone: failedRun}}
|
||||
}, map[string]model.Severity{KindWorkflowFail: model.Crit}, model.Crit},
|
||||
"dormant workflow failure muted": {func(r *model.Repo) {
|
||||
old := &model.Run{Status: "completed", Conclusion: "failure", CreatedAt: t0.Add(-40 * 24 * time.Hour)}
|
||||
r.Workflows = []model.Workflow{{Name: "Old", Latest: *old, LastDone: old}}
|
||||
}, map[string]model.Severity{}, model.OK},
|
||||
"workflow success": {func(r *model.Repo) {
|
||||
ok := &model.Run{Status: "completed", Conclusion: "success"}
|
||||
r.Workflows = []model.Workflow{{Name: "CI", Latest: *ok, LastDone: ok}}
|
||||
}, map[string]model.Severity{}, model.OK},
|
||||
"cancelled is not failure": {func(r *model.Repo) {
|
||||
c := &model.Run{Status: "completed", Conclusion: "cancelled"}
|
||||
r.Workflows = []model.Workflow{{Name: "CI", Latest: *c, LastDone: c}}
|
||||
}, map[string]model.Severity{}, model.OK},
|
||||
"workflow stuck": {func(r *model.Repo) {
|
||||
r.Workflows = []model.Workflow{{Name: "CI", Latest: model.Run{Status: "queued", CreatedAt: t0.Add(-7 * time.Hour)}}}
|
||||
}, map[string]model.Severity{KindWorkflowStuck: model.Warn}, model.Warn},
|
||||
"pr fresh": {func(r *model.Repo) {
|
||||
r.PRs = []model.PullRequest{{Number: 1, CreatedAt: t0.Add(-47 * time.Hour)}}
|
||||
}, map[string]model.Severity{}, model.OK},
|
||||
"pr waiting warn": {func(r *model.Repo) {
|
||||
r.PRs = []model.PullRequest{{Number: 1, CreatedAt: t0.Add(-49 * time.Hour)}}
|
||||
}, map[string]model.Severity{KindPRWaiting: model.Warn}, model.Warn},
|
||||
"pr waiting crit": {func(r *model.Repo) {
|
||||
r.PRs = []model.PullRequest{{Number: 1, CreatedAt: t0.Add(-8 * 24 * time.Hour)}}
|
||||
}, map[string]model.Severity{KindPRWaiting: model.Crit}, model.Crit},
|
||||
"draft pr ignored": {func(r *model.Repo) {
|
||||
r.PRs = []model.PullRequest{{Number: 1, Draft: true, CreatedAt: t0.Add(-30 * 24 * time.Hour), Checks: model.CheckSummary{State: model.ChecksFail}}}
|
||||
}, map[string]model.Severity{}, model.OK},
|
||||
"pr checks failing": {func(r *model.Repo) {
|
||||
r.PRs = []model.PullRequest{{Number: 1, CreatedAt: t0, Checks: model.CheckSummary{State: model.ChecksFail, Failing: []string{"build"}}}}
|
||||
}, map[string]model.Severity{KindPRChecks: model.Warn}, model.Warn},
|
||||
"issues stale aggregated": {func(r *model.Repo) {
|
||||
old := t0.Add(-40 * 24 * time.Hour)
|
||||
r.Issues = []model.Issue{{Number: 1, UpdatedAt: old}, {Number: 2, UpdatedAt: old}, {Number: 3, UpdatedAt: t0}}
|
||||
}, map[string]model.Severity{KindIssuesStale: model.Warn}, model.Warn},
|
||||
"archived repo skips work alerts": {func(r *model.Repo) {
|
||||
r.Archived = true
|
||||
r.PRs = []model.PullRequest{{Number: 1, CreatedAt: t0.Add(-30 * 24 * time.Hour)}}
|
||||
}, map[string]model.Severity{}, model.OK},
|
||||
}
|
||||
for name, tt := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
r := fresh()
|
||||
tt.mutate(&r)
|
||||
alerts := Evaluate(&r, t0, th())
|
||||
got := kinds(alerts)
|
||||
if len(got) != len(tt.want) {
|
||||
t.Fatalf("alerts = %+v, want kinds %v", alerts, tt.want)
|
||||
}
|
||||
for k, sev := range tt.want {
|
||||
if got[k] != sev {
|
||||
t.Fatalf("kind %s severity = %v, want %v", k, got[k], sev)
|
||||
}
|
||||
}
|
||||
if r.Health != tt.health {
|
||||
t.Fatalf("health = %v, want %v", r.Health, tt.health)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateDisabledThresholds(t *testing.T) {
|
||||
r := model.Repo{FullName: "o/r", RefreshedAt: t0, PRs: []model.PullRequest{{Number: 1, CreatedAt: t0.Add(-100 * 24 * time.Hour)}},
|
||||
Issues: []model.Issue{{UpdatedAt: t0.Add(-100 * 24 * time.Hour)}}}
|
||||
off := config.Thresholds{PRWarnAfter: -1, PRCritAfter: -1, IssueStaleAfter: -1, WorkflowStuck: -1, DataStaleAfter: -1}
|
||||
if a := Evaluate(&r, t0, off); len(a) != 0 {
|
||||
t.Fatalf("disabled checks produced %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgeAdvancesWithoutRefresh(t *testing.T) {
|
||||
r := model.Repo{FullName: "o/r", RefreshedAt: t0, PRs: []model.PullRequest{{Number: 1, CreatedAt: t0.Add(-40 * time.Hour)}}}
|
||||
cfg := th()
|
||||
cfg.DataStaleAfter = -1
|
||||
if a := Evaluate(&r, t0, cfg); len(a) != 0 {
|
||||
t.Fatalf("unexpected %+v", a)
|
||||
}
|
||||
if a := Evaluate(&r, t0.Add(10*time.Hour), cfg); len(a) != 1 || a[0].Severity != model.Warn {
|
||||
t.Fatalf("PR must cross the warn threshold as time passes: %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAge(t *testing.T) {
|
||||
cases := map[time.Duration]string{-time.Minute: "0m", 5 * time.Minute: "5m", 90 * time.Minute: "1h", 47 * time.Hour: "47h", 49 * time.Hour: "2d", 10 * 24 * time.Hour: "10d"}
|
||||
for d, want := range cases {
|
||||
if got := Age(d); got != want {
|
||||
t.Errorf("Age(%s) = %s, want %s", d, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type fakeFetcher struct {
|
||||
fn func(name string) (model.Repo, error)
|
||||
calls []string
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func (f *fakeFetcher) FetchRepo(_ context.Context, name string) (model.Repo, error) {
|
||||
f.mu.Lock()
|
||||
f.calls = append(f.calls, name)
|
||||
f.mu.Unlock()
|
||||
return f.fn(name)
|
||||
}
|
||||
|
||||
func newCollector(f Fetcher, repos ...string) *Collector {
|
||||
cfg := &config.Config{Repos: repos, RefreshInterval: time.Hour, Concurrency: 1, Thresholds: th()}
|
||||
c := New(f, cfg, slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||
c.now = func() time.Time { return t0 }
|
||||
return c
|
||||
}
|
||||
|
||||
func TestRefreshKeepsLastGoodDataOnFailure(t *testing.T) {
|
||||
fail := false
|
||||
f := &fakeFetcher{fn: func(name string) (model.Repo, error) {
|
||||
if fail {
|
||||
return model.Repo{}, errors.New("boom")
|
||||
}
|
||||
return model.Repo{FullName: name, URL: "u", Issues: []model.Issue{{Number: 1, UpdatedAt: t0}}}, nil
|
||||
}}
|
||||
c := newCollector(f, "o/a")
|
||||
c.Refresh(context.Background())
|
||||
if v := c.View(); !v.Loaded || len(v.Repos) != 1 || !v.Repos[0].Up || len(v.Repos[0].Issues) != 1 {
|
||||
t.Fatalf("first view: %+v", v)
|
||||
}
|
||||
fail = true
|
||||
c.Refresh(context.Background())
|
||||
v := c.View()
|
||||
r := v.Repos[0]
|
||||
if r.Up || r.Err != "boom" || len(r.Issues) != 1 {
|
||||
t.Fatalf("must keep old data and flag error: %+v", r)
|
||||
}
|
||||
if v.Crit != 1 || r.Health != model.Crit {
|
||||
t.Fatalf("crit=%d health=%v", v.Crit, r.Health)
|
||||
}
|
||||
if s := c.Stats(); s.RepoErrorsTotal != 1 || s.RefreshTotal != 2 {
|
||||
t.Fatalf("stats %+v", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFirstRefreshFailureStillListsRepo(t *testing.T) {
|
||||
c := newCollector(&fakeFetcher{fn: func(string) (model.Repo, error) { return model.Repo{}, errors.New("nope") }}, "o/a")
|
||||
c.Refresh(context.Background())
|
||||
v := c.View()
|
||||
if len(v.Repos) != 1 || v.Repos[0].Up || v.Repos[0].URL != "https://github.com/o/a" {
|
||||
t.Fatalf("%+v", v.Repos)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRateLimitStopsRemainingRepos(t *testing.T) {
|
||||
f := &fakeFetcher{fn: func(name string) (model.Repo, error) {
|
||||
if name == "o/b" {
|
||||
return model.Repo{}, model.ErrRateLimited
|
||||
}
|
||||
return model.Repo{FullName: name}, nil
|
||||
}}
|
||||
c := newCollector(f, "o/a", "o/b", "o/c", "o/d")
|
||||
c.Refresh(context.Background())
|
||||
if len(f.calls) != 2 {
|
||||
t.Fatalf("calls after limit = %v, want only a and b", f.calls)
|
||||
}
|
||||
v := c.View()
|
||||
if len(v.Repos) != 4 {
|
||||
t.Fatalf("all repos must stay listed: %d", len(v.Repos))
|
||||
}
|
||||
// Next cycle starts with a clean limit flag.
|
||||
f.fn = func(name string) (model.Repo, error) { return model.Repo{FullName: name}, nil }
|
||||
f.calls = nil
|
||||
c.Refresh(context.Background())
|
||||
if len(f.calls) != 4 {
|
||||
t.Fatalf("limit flag must reset, calls=%v", f.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestViewSortsWorstFirstAndAlertsBySeverity(t *testing.T) {
|
||||
f := &fakeFetcher{fn: func(name string) (model.Repo, error) {
|
||||
r := model.Repo{FullName: name}
|
||||
switch name {
|
||||
case "o/crit":
|
||||
c := &model.Run{Status: "completed", Conclusion: "failure"}
|
||||
r.Workflows = []model.Workflow{{Name: "CI", Latest: *c, LastDone: c}}
|
||||
case "o/warn":
|
||||
r.PRs = []model.PullRequest{{Number: 1, CreatedAt: t0.Add(-72 * time.Hour)}}
|
||||
}
|
||||
return r, nil
|
||||
}}
|
||||
c := newCollector(f, "o/ok", "o/warn", "o/crit")
|
||||
c.Refresh(context.Background())
|
||||
v := c.View()
|
||||
if v.Repos[0].FullName != "o/crit" || v.Repos[1].FullName != "o/warn" || v.Repos[2].FullName != "o/ok" {
|
||||
t.Fatalf("order: %s %s %s", v.Repos[0].FullName, v.Repos[1].FullName, v.Repos[2].FullName)
|
||||
}
|
||||
if len(v.Alerts) != 2 || v.Alerts[0].Severity != model.Crit || v.Crit != 1 || v.Warn != 1 {
|
||||
t.Fatalf("alerts: %+v", v.Alerts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestRefreshCoalesces(t *testing.T) {
|
||||
c := newCollector(&fakeFetcher{fn: func(n string) (model.Repo, error) { return model.Repo{FullName: n}, nil }}, "o/a")
|
||||
if !c.RequestRefresh() {
|
||||
t.Fatal("first request must queue")
|
||||
}
|
||||
if c.RequestRefresh() {
|
||||
t.Fatal("second request must coalesce while one is queued")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRefreshesOnStartAndStopsOnCancel(t *testing.T) {
|
||||
c := newCollector(&fakeFetcher{fn: func(n string) (model.Repo, error) { return model.Repo{FullName: n}, nil }}, "o/a")
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() { c.Run(ctx); close(done) }()
|
||||
deadline := time.After(2 * time.Second)
|
||||
for !c.Stats().Loaded {
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatal("no initial refresh")
|
||||
case <-time.After(5 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("Run did not stop")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSummarizeChecks(t *testing.T) {
|
||||
tests := map[string]struct {
|
||||
want string
|
||||
in model.CheckSummary
|
||||
}{
|
||||
"few distinct": {in: model.CheckSummary{Failed: 2, Failing: []string{"build", "lint"}}, want: "2 checks failing: build, lint"},
|
||||
"duplicates collapse and extras count": {
|
||||
in: model.CheckSummary{Failed: 6, Failing: []string{"pr-checks / Tests", "pr-checks / Tests", "pr-checks / Lint", "pr-checks / Scan", "pr-checks / Gosec"}},
|
||||
want: "6 checks failing: Gosec, Lint +2 more",
|
||||
},
|
||||
"mixed prefixes stay intact": {in: model.CheckSummary{Failed: 2, Failing: []string{"a / x", "b / y"}}, want: "2 checks failing: a / x, b / y"},
|
||||
"no names": {in: model.CheckSummary{Failed: 1}, want: "1 checks failing"},
|
||||
}
|
||||
for name, tt := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if got := summarizeChecks(tt.in); got != tt.want {
|
||||
t.Fatalf("got %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSummarizeWarningsMergesSharedReason(t *testing.T) {
|
||||
got := summarizeWarnings([]string{
|
||||
"issues: github: HTTP 403: Resource not accessible by integration",
|
||||
"workflows: github: HTTP 403: Resource not accessible by integration",
|
||||
"releases: github: HTTP 500: boom",
|
||||
})
|
||||
want := []string{
|
||||
"issues, workflows: github: HTTP 403: Resource not accessible by integration",
|
||||
"releases: github: HTTP 500: boom",
|
||||
}
|
||||
if len(got) != 2 || got[0] != want[0] || got[1] != want[1] {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
package collector
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/lukaszraczylo/ghtracker/internal/config"
|
||||
"github.com/lukaszraczylo/ghtracker/internal/model"
|
||||
)
|
||||
|
||||
// Alert kinds; they double as stable identifiers for dashboards.
|
||||
const (
|
||||
KindRefreshFailed = "refresh_failed"
|
||||
KindDataStale = "data_stale"
|
||||
KindPartialData = "partial_data"
|
||||
KindWorkflowFail = "workflow_failed"
|
||||
KindWorkflowStuck = "workflow_stuck"
|
||||
KindPRWaiting = "pr_waiting"
|
||||
KindPRChecks = "pr_checks_failing"
|
||||
KindIssuesStale = "issues_stale"
|
||||
)
|
||||
|
||||
// Evaluate derives alerts for one repo at time now and stores the worst severity in r.Health.
|
||||
// It runs on every read so age-based alerts advance between refreshes.
|
||||
func Evaluate(r *model.Repo, now time.Time, th config.Thresholds) []model.Alert {
|
||||
var alerts []model.Alert
|
||||
add := func(sev model.Severity, kind, subject, detail, url string, since time.Time) {
|
||||
alerts = append(alerts, model.Alert{Severity: sev, Repo: r.FullName, Kind: kind, Subject: subject, Detail: detail, URL: url, Since: since})
|
||||
r.Health = max(r.Health, sev)
|
||||
}
|
||||
r.Health = model.OK
|
||||
|
||||
if r.Err != "" {
|
||||
add(model.Crit, KindRefreshFailed, "Refresh failed", r.Err, r.URL, r.RefreshedAt)
|
||||
}
|
||||
if th.DataStaleAfter > 0 && !r.RefreshedAt.IsZero() && now.Sub(r.RefreshedAt) > th.DataStaleAfter {
|
||||
add(model.Warn, KindDataStale, "Data is stale", "last refreshed "+Age(now.Sub(r.RefreshedAt))+" ago", r.URL, r.RefreshedAt)
|
||||
}
|
||||
for _, detail := range summarizeWarnings(r.Warnings) {
|
||||
add(model.Warn, KindPartialData, "Incomplete data", detail, r.URL, r.RefreshedAt)
|
||||
}
|
||||
if r.Archived {
|
||||
return alerts
|
||||
}
|
||||
|
||||
for _, wf := range r.Workflows {
|
||||
if wf.LastDone != nil && wf.LastDone.Failed() && !Dormant(wf, now, th) {
|
||||
add(model.Crit, KindWorkflowFail, workflowSubject(wf), fmt.Sprintf("%s on %s (%s)", strings.ReplaceAll(wf.LastDone.Conclusion, "_", " "), wf.LastDone.Branch, wf.LastDone.Event), wf.LastDone.URL, wf.LastDone.CreatedAt)
|
||||
}
|
||||
if wf.Latest.Running() && th.WorkflowStuck > 0 && now.Sub(wf.Latest.CreatedAt) > th.WorkflowStuck {
|
||||
add(model.Warn, KindWorkflowStuck, workflowSubject(wf),
|
||||
fmt.Sprintf("%s for %s", strings.ReplaceAll(wf.Latest.Status, "_", " "), Age(now.Sub(wf.Latest.CreatedAt))),
|
||||
wf.Latest.URL, wf.Latest.CreatedAt)
|
||||
}
|
||||
}
|
||||
|
||||
for _, pr := range r.PRs {
|
||||
if pr.Draft {
|
||||
continue
|
||||
}
|
||||
age := now.Sub(pr.CreatedAt)
|
||||
subject := fmt.Sprintf("#%d %s", pr.Number, pr.Title)
|
||||
switch {
|
||||
case th.PRCritAfter > 0 && age >= th.PRCritAfter:
|
||||
add(model.Crit, KindPRWaiting, subject, "open "+Age(age), pr.URL, pr.CreatedAt)
|
||||
case th.PRWarnAfter > 0 && age >= th.PRWarnAfter:
|
||||
add(model.Warn, KindPRWaiting, subject, "open "+Age(age), pr.URL, pr.CreatedAt)
|
||||
}
|
||||
if pr.Checks.State == model.ChecksFail {
|
||||
add(model.Warn, KindPRChecks, subject, summarizeChecks(pr.Checks), pr.URL, pr.UpdatedAt)
|
||||
}
|
||||
}
|
||||
|
||||
if th.IssueStaleAfter > 0 {
|
||||
var stale int
|
||||
var oldest time.Time
|
||||
for _, is := range r.Issues {
|
||||
if now.Sub(is.UpdatedAt) >= th.IssueStaleAfter {
|
||||
stale++
|
||||
if oldest.IsZero() || is.UpdatedAt.Before(oldest) {
|
||||
oldest = is.UpdatedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
if stale > 0 {
|
||||
add(model.Warn, KindIssuesStale, fmt.Sprintf("%d stale issues", stale),
|
||||
"no activity for "+Age(th.IssueStaleAfter)+" or more",
|
||||
r.URL+"/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-asc", oldest)
|
||||
}
|
||||
}
|
||||
return alerts
|
||||
}
|
||||
|
||||
func workflowSubject(wf model.Workflow) string {
|
||||
if wf.Background {
|
||||
return wf.Name + " (background)"
|
||||
}
|
||||
return wf.Name
|
||||
}
|
||||
|
||||
func sortAlerts(a []model.Alert) {
|
||||
slices.SortStableFunc(a, func(x, y model.Alert) int {
|
||||
if x.Severity != y.Severity {
|
||||
return int(y.Severity) - int(x.Severity)
|
||||
}
|
||||
if c := x.Since.Compare(y.Since); c != 0 {
|
||||
return c
|
||||
}
|
||||
return strings.Compare(x.Repo+x.Subject, y.Repo+y.Subject)
|
||||
})
|
||||
}
|
||||
|
||||
// Age renders a duration at a glance, e.g. "3d", "5h", "12m".
|
||||
func Age(d time.Duration) string {
|
||||
switch {
|
||||
case d < 0:
|
||||
return "0m"
|
||||
case d >= 48*time.Hour:
|
||||
return fmt.Sprintf("%dd", int(d.Hours()/24))
|
||||
case d >= time.Hour:
|
||||
return fmt.Sprintf("%dh", int(d.Hours()))
|
||||
default:
|
||||
return fmt.Sprintf("%dm", int(d.Minutes()))
|
||||
}
|
||||
}
|
||||
|
||||
// Dormant reports whether the workflow has not run for longer than the dormant threshold,
|
||||
// so an old failure no longer signals a current problem.
|
||||
func Dormant(wf model.Workflow, now time.Time, th config.Thresholds) bool {
|
||||
return th.WorkflowDormant > 0 && !wf.Latest.CreatedAt.IsZero() && now.Sub(wf.Latest.CreatedAt) > th.WorkflowDormant
|
||||
}
|
||||
|
||||
const maxNamedChecks = 2
|
||||
|
||||
// summarizeChecks renders failing checks as a count plus the first few distinct names.
|
||||
func summarizeChecks(c model.CheckSummary) string {
|
||||
names := slices.Compact(slices.Sorted(slices.Values(c.Failing)))
|
||||
if prefix, ok := sharedPrefix(names); ok {
|
||||
for i, n := range names {
|
||||
names[i] = strings.TrimPrefix(n, prefix)
|
||||
}
|
||||
}
|
||||
shown := names[:min(len(names), maxNamedChecks)]
|
||||
out := fmt.Sprintf("%d checks failing", c.Failed)
|
||||
if len(shown) > 0 {
|
||||
out += ": " + strings.Join(shown, ", ")
|
||||
}
|
||||
if extra := len(names) - len(shown); extra > 0 {
|
||||
out += fmt.Sprintf(" +%d more", extra)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sharedPrefix returns the common "group / " prefix of check names such as "pr-checks / Tests".
|
||||
func sharedPrefix(names []string) (string, bool) {
|
||||
if len(names) < 2 {
|
||||
return "", false
|
||||
}
|
||||
group, _, ok := strings.Cut(names[0], " / ")
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
prefix := group + " / "
|
||||
for _, n := range names {
|
||||
if !strings.HasPrefix(n, prefix) {
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
return prefix, true
|
||||
}
|
||||
|
||||
// summarizeWarnings merges "section: reason" warnings that share a reason into one line.
|
||||
func summarizeWarnings(warnings []string) []string {
|
||||
var reasons []string
|
||||
sections := map[string][]string{}
|
||||
for _, w := range warnings {
|
||||
section, reason, ok := strings.Cut(w, ": ")
|
||||
if !ok {
|
||||
section, reason = "data", w
|
||||
}
|
||||
if _, seen := sections[reason]; !seen {
|
||||
reasons = append(reasons, reason)
|
||||
}
|
||||
sections[reason] = append(sections[reason], section)
|
||||
}
|
||||
out := make([]string, 0, len(reasons))
|
||||
for _, reason := range reasons {
|
||||
out = append(out, strings.Join(sections[reason], ", ")+": "+reason)
|
||||
}
|
||||
return out
|
||||
}
|
||||
Reference in New Issue
Block a user