这是indexloc提供的服务，不要输入任何密码
Skip to content

feat(feed): use exponential backoff for broken feeds #3809

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

feat(feed): use exponential backoff for broken feeds #3809

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions internal/model/feed.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ package model // import "miniflux.app/v2/internal/model"
import (
"fmt"
"io"
"math"
"time"

"miniflux.app/v2/internal/config"
Expand All @@ -20,6 +21,9 @@ const (
DefaultFeedSortingDirection = "desc"
)

// A feed with parsing errors will be checked at least every week.
var backoffMax = time.Duration(time.Hour * 24 * 7)

// Feed represents a feed in the application.
type Feed struct {
ID int64 `json:"id"`
Expand Down Expand Up @@ -143,10 +147,24 @@ func (f *Feed) ScheduleNextCheck(weeklyCount int, refreshDelay time.Duration) ti
interval = min(interval, config.Opts.SchedulerEntryFrequencyMaxInterval())
}

interval += backoff(f.ParsingErrorCount)

f.NextCheckAt = time.Now().Add(interval)
return interval
}

func backoff(count int) time.Duration {
if count == 0 {
return 0
}
// https://en.wikipedia.org/wiki/Exponential_backoff
backoff := time.Duration(math.Pow(2, float64(count))) * time.Hour
if backoff > backoffMax {
return backoffMax
}
return backoff
}

// FeedCreationRequest represents the request to create a feed.
type FeedCreationRequest struct {
FeedURL string `json:"feed_url"`
Expand Down
44 changes: 44 additions & 0 deletions internal/model/feed_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -376,3 +376,47 @@ func TestFeedScheduleNextCheckEntryFrequencyLargeNewTTL(t *testing.T) {
t.Error(`The next_check_at should be after timeBefore + entry frequency min interval`)
}
}

func TestFeedScheduleNextCheckParsingErrorsBackoff(t *testing.T) {
for count := range 10 {
f1 := &Feed{ParsingErrorCount: 0}
f2 := &Feed{ParsingErrorCount: count}
newTTL := time.Duration(1) * time.Minute * 2
f1.ScheduleNextCheck(0, newTTL)
f2.ScheduleNextCheck(0, newTTL)

if f1.NextCheckAt.IsZero() {
t.Error(`The next_check_at for f1 must be set`)
}
if f2.NextCheckAt.IsZero() {
t.Error(`The next_check_at for f2 must be set`)
}

if f1.NextCheckAt.Add(backoff(count)).Sub(f2.NextCheckAt).Minutes() > 10 {
t.Error("The next_check_at should have been using the exponential backoff.")
}
}
}

func TestFeedScheduleNextCheckParsingErrorsBackoffMax(t *testing.T) {
f1 := &Feed{ParsingErrorCount: 0}
newTTL := time.Duration(1) * time.Minute * 2
f1.ScheduleNextCheck(0, newTTL)
if f1.NextCheckAt.IsZero() {
t.Error(`The next_check_at for f1 must be set`)
}

for count := range 128 {
f2 := &Feed{ParsingErrorCount: count}
f2.ScheduleNextCheck(0, newTTL)

if f2.NextCheckAt.IsZero() {
t.Error(`The next_check_at for f1 must be set`)
}

offset := f2.NextCheckAt.Sub(f1.NextCheckAt)
if offset > backoffMax+time.Minute*1 {
t.Errorf("The next_check_at's offset for errors (%q) for %d errors should never be bigger than %q.", offset, count, backoffMax)
}
}
}
Loading