mirror of
https://github.com/lukaszraczylo/git-velocity.git
synced 2026-06-05 22:43:56 +00:00
Abstract git operations into generics.
This commit is contained in:
@@ -0,0 +1,306 @@
|
||||
package github
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/google/go-github/v68/github"
|
||||
)
|
||||
|
||||
// DateFilterResult represents the result of date filtering
|
||||
type DateFilterResult int
|
||||
|
||||
const (
|
||||
// DateInclude means the item is within the date range
|
||||
DateInclude DateFilterResult = iota
|
||||
// DateTooNew means the item is newer than the 'until' date
|
||||
DateTooNew
|
||||
// DateTooOld means the item is older than the 'since' date
|
||||
DateTooOld
|
||||
)
|
||||
|
||||
// FilterByDate checks if a time falls within the specified date range
|
||||
func FilterByDate(t time.Time, since, until *time.Time) DateFilterResult {
|
||||
if until != nil && t.After(*until) {
|
||||
return DateTooNew
|
||||
}
|
||||
if since != nil && t.Before(*since) {
|
||||
return DateTooOld
|
||||
}
|
||||
return DateInclude
|
||||
}
|
||||
|
||||
// PageFetcher is a generic interface for fetching paginated resources
|
||||
type PageFetcher[T any, R any] interface {
|
||||
// Fetch retrieves a page of items
|
||||
Fetch(ctx context.Context, page int) (items []T, resp *github.Response, err error)
|
||||
// Convert transforms a raw item into the result type
|
||||
Convert(item T) R
|
||||
// Filter determines if an item should be included based on date range
|
||||
// Returns DateInclude to include, DateTooNew/DateTooOld to exclude
|
||||
Filter(item T) DateFilterResult
|
||||
// ShouldSkip returns true if the item should be skipped entirely (e.g., PRs in issues list)
|
||||
ShouldSkip(item T) bool
|
||||
}
|
||||
|
||||
// FetchConfig holds configuration for paginated fetching
|
||||
type FetchConfig struct {
|
||||
// ResourceName is used for progress messages (e.g., "issues", "pull requests")
|
||||
ResourceName string
|
||||
// EarlyTermination enables stopping when all items on a page are too old
|
||||
EarlyTermination bool
|
||||
// EarlyTerminationThreshold is the number of consecutive old pages before stopping
|
||||
EarlyTerminationThreshold int
|
||||
}
|
||||
|
||||
// DefaultFetchConfig returns sensible defaults
|
||||
func DefaultFetchConfig(resourceName string) FetchConfig {
|
||||
return FetchConfig{
|
||||
ResourceName: resourceName,
|
||||
EarlyTermination: true,
|
||||
EarlyTerminationThreshold: 2,
|
||||
}
|
||||
}
|
||||
|
||||
// FetchAllPages fetches all pages of a resource with caching, filtering, and early termination
|
||||
func FetchAllPages[T any, R any](
|
||||
ctx context.Context,
|
||||
c *Client,
|
||||
cacheKey string,
|
||||
config FetchConfig,
|
||||
fetcher PageFetcher[T, R],
|
||||
) ([]R, error) {
|
||||
// Check cache first (skip if no cache key provided)
|
||||
if cacheKey != "" {
|
||||
if cached, ok := c.cache.Get(cacheKey); ok {
|
||||
if results, ok := cached.([]R); ok {
|
||||
c.progress(fmt.Sprintf(" Using cached %s data", config.ResourceName))
|
||||
return results, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var allResults []R
|
||||
page := 1
|
||||
consecutiveOldPages := 0
|
||||
|
||||
for {
|
||||
items, resp, err := fetcher.Fetch(ctx, page)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch %s: %w", config.ResourceName, err)
|
||||
}
|
||||
|
||||
c.progress(fmt.Sprintf(" Fetching %s page %d (%d %s so far)...",
|
||||
config.ResourceName, page, len(allResults), config.ResourceName))
|
||||
|
||||
oldInPage := 0
|
||||
totalEligible := 0
|
||||
|
||||
for _, item := range items {
|
||||
// Skip items that should be filtered out entirely (e.g., PRs in issues API)
|
||||
if fetcher.ShouldSkip(item) {
|
||||
continue
|
||||
}
|
||||
|
||||
totalEligible++
|
||||
|
||||
// Apply date filtering
|
||||
switch fetcher.Filter(item) {
|
||||
case DateTooNew:
|
||||
continue
|
||||
case DateTooOld:
|
||||
oldInPage++
|
||||
continue
|
||||
case DateInclude:
|
||||
allResults = append(allResults, fetcher.Convert(item))
|
||||
}
|
||||
}
|
||||
|
||||
// Early termination logic
|
||||
if config.EarlyTermination && totalEligible > 0 && oldInPage == totalEligible {
|
||||
consecutiveOldPages++
|
||||
if consecutiveOldPages >= config.EarlyTerminationThreshold {
|
||||
c.progress(fmt.Sprintf(" Reached %s older than date range, stopping early (page %d)",
|
||||
config.ResourceName, page))
|
||||
break
|
||||
}
|
||||
} else {
|
||||
consecutiveOldPages = 0
|
||||
}
|
||||
|
||||
if resp.NextPage == 0 {
|
||||
break
|
||||
}
|
||||
page = resp.NextPage
|
||||
}
|
||||
|
||||
// Cache results (skip if no cache key provided)
|
||||
if cacheKey != "" {
|
||||
c.cache.Set(cacheKey, allResults)
|
||||
}
|
||||
|
||||
return allResults, nil
|
||||
}
|
||||
|
||||
// SimpleFetcher is a helper for creating simple fetchers without date filtering
|
||||
type SimpleFetcher[T any, R any] struct {
|
||||
FetchFn func(ctx context.Context, page int) ([]T, *github.Response, error)
|
||||
ConvertFn func(item T) R
|
||||
}
|
||||
|
||||
func (f *SimpleFetcher[T, R]) Fetch(ctx context.Context, page int) ([]T, *github.Response, error) {
|
||||
return f.FetchFn(ctx, page)
|
||||
}
|
||||
|
||||
func (f *SimpleFetcher[T, R]) Convert(item T) R {
|
||||
return f.ConvertFn(item)
|
||||
}
|
||||
|
||||
func (f *SimpleFetcher[T, R]) Filter(item T) DateFilterResult {
|
||||
return DateInclude // No filtering
|
||||
}
|
||||
|
||||
func (f *SimpleFetcher[T, R]) ShouldSkip(item T) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// DateFilteredFetcher extends SimpleFetcher with date filtering
|
||||
type DateFilteredFetcher[T any, R any] struct {
|
||||
FetchFn func(ctx context.Context, page int) ([]T, *github.Response, error)
|
||||
ConvertFn func(item T) R
|
||||
GetDateFn func(item T) time.Time
|
||||
SkipFn func(item T) bool
|
||||
Since *time.Time
|
||||
Until *time.Time
|
||||
}
|
||||
|
||||
func (f *DateFilteredFetcher[T, R]) Fetch(ctx context.Context, page int) ([]T, *github.Response, error) {
|
||||
return f.FetchFn(ctx, page)
|
||||
}
|
||||
|
||||
func (f *DateFilteredFetcher[T, R]) Convert(item T) R {
|
||||
return f.ConvertFn(item)
|
||||
}
|
||||
|
||||
func (f *DateFilteredFetcher[T, R]) Filter(item T) DateFilterResult {
|
||||
return FilterByDate(f.GetDateFn(item), f.Since, f.Until)
|
||||
}
|
||||
|
||||
func (f *DateFilteredFetcher[T, R]) ShouldSkip(item T) bool {
|
||||
if f.SkipFn != nil {
|
||||
return f.SkipFn(item)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// WithRetry wraps a fetch function with retry logic
|
||||
func (c *Client) WithRetry(ctx context.Context, operation string, fn func() error) error {
|
||||
return c.retryWithBackoff(ctx, operation, fn)
|
||||
}
|
||||
|
||||
// EnrichingFetcher extends DateFilteredFetcher with per-item enrichment
|
||||
// This is useful when you need to fetch additional details for each item (e.g., commit details)
|
||||
type EnrichingFetcher[T any, R any] struct {
|
||||
FetchFn func(ctx context.Context, page int) ([]T, *github.Response, error)
|
||||
EnrichFn func(ctx context.Context, item T) (R, error) // Enriches and converts in one step
|
||||
GetDateFn func(item T) time.Time
|
||||
SkipFn func(item T) bool
|
||||
Since *time.Time
|
||||
Until *time.Time
|
||||
}
|
||||
|
||||
func (f *EnrichingFetcher[T, R]) Fetch(ctx context.Context, page int) ([]T, *github.Response, error) {
|
||||
return f.FetchFn(ctx, page)
|
||||
}
|
||||
|
||||
func (f *EnrichingFetcher[T, R]) Convert(item T) R {
|
||||
// This won't be used - FetchAllPagesWithEnrichment handles enrichment
|
||||
var zero R
|
||||
return zero
|
||||
}
|
||||
|
||||
func (f *EnrichingFetcher[T, R]) Filter(item T) DateFilterResult {
|
||||
return FilterByDate(f.GetDateFn(item), f.Since, f.Until)
|
||||
}
|
||||
|
||||
func (f *EnrichingFetcher[T, R]) ShouldSkip(item T) bool {
|
||||
if f.SkipFn != nil {
|
||||
return f.SkipFn(item)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// FetchAllPagesWithEnrichment is like FetchAllPages but calls EnrichFn for each item
|
||||
// This is useful when you need to make additional API calls per item (e.g., fetching commit details)
|
||||
func FetchAllPagesWithEnrichment[T any, R any](
|
||||
ctx context.Context,
|
||||
c *Client,
|
||||
cacheKey string,
|
||||
config FetchConfig,
|
||||
fetcher *EnrichingFetcher[T, R],
|
||||
progressEvery int, // Report progress every N items (0 = disabled)
|
||||
) ([]R, error) {
|
||||
// Check cache first
|
||||
if cacheKey != "" {
|
||||
if cached, ok := c.cache.Get(cacheKey); ok {
|
||||
if results, ok := cached.([]R); ok {
|
||||
c.progress(fmt.Sprintf(" Using cached %s data", config.ResourceName))
|
||||
return results, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var allResults []R
|
||||
page := 1
|
||||
|
||||
for {
|
||||
items, resp, err := fetcher.Fetch(ctx, page)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch %s: %w", config.ResourceName, err)
|
||||
}
|
||||
|
||||
c.progress(fmt.Sprintf(" Fetching %s page %d (%d %s so far)...",
|
||||
config.ResourceName, page, len(allResults), config.ResourceName))
|
||||
|
||||
itemsInPage := 0
|
||||
for i, item := range items {
|
||||
// Skip items that should be filtered out entirely
|
||||
if fetcher.ShouldSkip(item) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Apply date filtering
|
||||
if fetcher.Filter(item) != DateInclude {
|
||||
continue
|
||||
}
|
||||
|
||||
// Enrich the item (this may make additional API calls)
|
||||
enriched, err := fetcher.EnrichFn(ctx, item)
|
||||
if err != nil {
|
||||
c.progress(fmt.Sprintf(" Warning: failed to enrich item: %v", err))
|
||||
continue
|
||||
}
|
||||
|
||||
allResults = append(allResults, enriched)
|
||||
itemsInPage++
|
||||
|
||||
// Progress reporting
|
||||
if progressEvery > 0 && (i+1)%progressEvery == 0 {
|
||||
c.progress(fmt.Sprintf(" Processing item %d/%d on page %d...", i+1, len(items), page))
|
||||
}
|
||||
}
|
||||
|
||||
if resp.NextPage == 0 {
|
||||
break
|
||||
}
|
||||
page = resp.NextPage
|
||||
}
|
||||
|
||||
// Cache results
|
||||
if cacheKey != "" {
|
||||
c.cache.Set(cacheKey, allResults)
|
||||
}
|
||||
|
||||
return allResults, nil
|
||||
}
|
||||
Reference in New Issue
Block a user