package github import ( "context" "fmt" "time" "github.com/google/go-github/v68/github" ) // DateFilterResult represents the result of date filtering type DateFilterResult int const ( // DateInclude means the item is within the date range DateInclude DateFilterResult = iota // DateTooNew means the item is newer than the 'until' date DateTooNew // DateTooOld means the item is older than the 'since' date DateTooOld ) // FilterByDate checks if a time falls within the specified date range func FilterByDate(t time.Time, since, until *time.Time) DateFilterResult { if until != nil && t.After(*until) { return DateTooNew } if since != nil && t.Before(*since) { return DateTooOld } return DateInclude } // PageFetcher is a generic interface for fetching paginated resources type PageFetcher[T any, R any] interface { // Fetch retrieves a page of items Fetch(ctx context.Context, page int) (items []T, resp *github.Response, err error) // Convert transforms a raw item into the result type Convert(item T) R // Filter determines if an item should be included based on date range // Returns DateInclude to include, DateTooNew/DateTooOld to exclude Filter(item T) DateFilterResult // ShouldSkip returns true if the item should be skipped entirely (e.g., PRs in issues list) ShouldSkip(item T) bool } // FetchConfig holds configuration for paginated fetching type FetchConfig struct { // ResourceName is used for progress messages (e.g., "issues", "pull requests") ResourceName string // EarlyTermination enables stopping when all items on a page are too old EarlyTermination bool // EarlyTerminationThreshold is the number of consecutive old pages before stopping EarlyTerminationThreshold int // Quiet suppresses per-page progress messages (useful for sub-fetches like reviews) Quiet bool } // DefaultFetchConfig returns sensible defaults func DefaultFetchConfig(resourceName string) FetchConfig { return FetchConfig{ ResourceName: resourceName, EarlyTermination: true, EarlyTerminationThreshold: 2, } } // FetchAllPages fetches all pages of a resource with caching, filtering, and early termination func FetchAllPages[T any, R any]( ctx context.Context, c *Client, cacheKey string, config FetchConfig, fetcher PageFetcher[T, R], ) ([]R, error) { // Check cache first (skip if no cache key provided) if cacheKey != "" { if cached, ok := c.cache.Get(cacheKey); ok { if results, ok := cached.([]R); ok { c.progress(fmt.Sprintf(" Using cached %s data", config.ResourceName)) return results, nil } } } var allResults []R page := 1 consecutiveOldPages := 0 for { items, resp, err := fetcher.Fetch(ctx, page) if err != nil { return nil, fmt.Errorf("failed to fetch %s: %w", config.ResourceName, err) } // Safety check for nil response if resp == nil { break } if !config.Quiet { c.progress(fmt.Sprintf(" Fetching %s page %d (%d %s so far)...", config.ResourceName, page, len(allResults), config.ResourceName)) } oldInPage := 0 totalEligible := 0 for _, item := range items { // Skip items that should be filtered out entirely (e.g., PRs in issues API) if fetcher.ShouldSkip(item) { continue } totalEligible++ // Apply date filtering switch fetcher.Filter(item) { case DateTooNew: continue case DateTooOld: oldInPage++ continue case DateInclude: allResults = append(allResults, fetcher.Convert(item)) } } // Early termination logic if config.EarlyTermination && totalEligible > 0 && oldInPage == totalEligible { consecutiveOldPages++ if consecutiveOldPages >= config.EarlyTerminationThreshold { if !config.Quiet { c.progress(fmt.Sprintf(" Reached %s older than date range, stopping early (page %d)", config.ResourceName, page)) } break } } else { consecutiveOldPages = 0 } if resp.NextPage == 0 { break } page = resp.NextPage } // Cache results (skip if no cache key provided) if cacheKey != "" { c.cache.Set(cacheKey, allResults) } return allResults, nil } // SimpleFetcher is a helper for creating simple fetchers without date filtering type SimpleFetcher[T any, R any] struct { FetchFn func(ctx context.Context, page int) ([]T, *github.Response, error) ConvertFn func(item T) R } func (f *SimpleFetcher[T, R]) Fetch(ctx context.Context, page int) ([]T, *github.Response, error) { return f.FetchFn(ctx, page) } func (f *SimpleFetcher[T, R]) Convert(item T) R { return f.ConvertFn(item) } func (f *SimpleFetcher[T, R]) Filter(item T) DateFilterResult { return DateInclude // No filtering } func (f *SimpleFetcher[T, R]) ShouldSkip(item T) bool { return false } // DateFilteredFetcher extends SimpleFetcher with date filtering type DateFilteredFetcher[T any, R any] struct { FetchFn func(ctx context.Context, page int) ([]T, *github.Response, error) ConvertFn func(item T) R GetDateFn func(item T) time.Time SkipFn func(item T) bool Since *time.Time Until *time.Time } func (f *DateFilteredFetcher[T, R]) Fetch(ctx context.Context, page int) ([]T, *github.Response, error) { return f.FetchFn(ctx, page) } func (f *DateFilteredFetcher[T, R]) Convert(item T) R { return f.ConvertFn(item) } func (f *DateFilteredFetcher[T, R]) Filter(item T) DateFilterResult { return FilterByDate(f.GetDateFn(item), f.Since, f.Until) } func (f *DateFilteredFetcher[T, R]) ShouldSkip(item T) bool { if f.SkipFn != nil { return f.SkipFn(item) } return false }