diff --git a/apps/server/internal/platform/account_content_test.go b/apps/server/internal/platform/account_content_test.go index 7046bf59e..441dbca5c 100644 --- a/apps/server/internal/platform/account_content_test.go +++ b/apps/server/internal/platform/account_content_test.go @@ -1,6 +1,9 @@ package platform import ( + "fmt" + "net/http" + "strconv" "strings" "testing" "time" @@ -8,6 +11,66 @@ import ( "github.com/stretchr/testify/require" ) +// The discovery job asks for pages no smaller than MinPageSize and rejects a +// page with more items than it asked for. Lemmy, PieFed, and PeerTube read +// fixed-size pages, so the smallest page they accept must be that size. +func TestFixedPageDiscoveryNeverReturnsMoreThanTheSmallestRequestedPage(t *testing.T) { + originalClient := httpClient + defer func() { httpClient = originalClient }() + + const instanceURL = "https://fed.example" + published := time.Now().UTC().Add(-time.Hour).Format(time.RFC3339) + full := func(limit string, item func(id int) string) string { + size, _ := strconv.Atoi(limit) + items := make([]string, 0, size) + for id := 1; id <= size; id++ { + items = append(items, item(id)) + } + return strings.Join(items, ",") + } + httpClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + query := req.URL.Query() + switch req.URL.Path { + case "/api/v3/user": + return jsonResponse(req, `{"posts":[`+full(query.Get("limit"), func(id int) string { + return fmt.Sprintf(`{"post":{"id":%d,"name":"Post","ap_id":"%s/post/%d","published":%q}}`, id, instanceURL, id, published) + })+`]}`), nil + case "/api/alpha/post/list": + return jsonResponse(req, `{"posts":[`+full(query.Get("limit"), func(id int) string { + return fmt.Sprintf(`{"post":{"id":%d,"title":"Post","ap_id":"%s/post/%d","published":%q}}`, id, instanceURL, id, published) + })+`],"next_page":"2"}`), nil + case "/api/v1/video-channels/demos/videos": + return jsonResponse(req, `{"total":1000,"data":[`+full(query.Get("count"), func(id int) string { + return fmt.Sprintf(`{"uuid":"uuid-%d","name":"Video","publishedAt":%q}`, id, published) + })+`]}`), nil + } + return jsonResponseWithStatus(req, http.StatusNotFound, `{}`), nil + })} + + tests := []struct { + name string + discoverer AccountContentDiscoverer + accountID string + }{ + {name: "lemmy", discoverer: NewLemmyAdapter(instanceURL), accountID: "5"}, + {name: "piefed", discoverer: NewPieFedAdapter(instanceURL), accountID: "5"}, + {name: "peertube", discoverer: NewPeerTubeAdapter(instanceURL), accountID: "demos"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + support := test.discoverer.AccountContentDiscoverySupport(AnalyticsAccountContext{AccountID: test.accountID}) + require.True(t, support.Supported) + smallest := max(1, support.MinPageSize) + page, err := test.discoverer.DiscoverAccountContent(t.Context(), "token", AccountContentDiscoveryRequest{ + AccountID: test.accountID, PageSize: smallest, PublishedAfter: time.Now().UTC().Add(-90 * 24 * time.Hour), + }) + require.NoError(t, err) + require.NotEmpty(t, page.Items) + require.LessOrEqual(t, len(page.Items), smallest, "a page must fit the smallest page size the adapter declares") + }) + } +} + func TestNormalizeAccountContentItemBoundsTextAndRejectsUnsafeProviderURLs(t *testing.T) { t.Parallel() diff --git a/apps/server/internal/platform/lemmy.go b/apps/server/internal/platform/lemmy.go index bb3708f5c..a5f2a5314 100644 --- a/apps/server/internal/platform/lemmy.go +++ b/apps/server/internal/platform/lemmy.go @@ -577,7 +577,8 @@ func (l *LemmyAdapter) AccountContentDiscoverySupport(input AnalyticsAccountCont if strings.TrimSpace(input.AccountID) == "" { return AccountContentDiscoverySupport{UnavailableReason: "Lemmy account content discovery requires a stable account identity."} } - return AccountContentDiscoverySupport{Supported: true, MaxPageSize: 20} + // Pages are numbered at a fixed size, so the job must not ask for less. + return AccountContentDiscoverySupport{Supported: true, MinPageSize: lemmyAccountContentPageSize, MaxPageSize: lemmyAccountContentPageSize} } func (l *LemmyAdapter) DiscoverAccountContent(ctx context.Context, accessToken string, input AccountContentDiscoveryRequest) (AccountContentPage, error) { diff --git a/apps/server/internal/platform/peertube.go b/apps/server/internal/platform/peertube.go index 573fab5ec..6f95b1546 100644 --- a/apps/server/internal/platform/peertube.go +++ b/apps/server/internal/platform/peertube.go @@ -827,7 +827,8 @@ func (p *PeerTubeAdapter) AccountContentDiscoverySupport(input AnalyticsAccountC if strings.TrimSpace(input.AccountID) == "" { return AccountContentDiscoverySupport{UnavailableReason: "PeerTube account content discovery requires a channel."} } - return AccountContentDiscoverySupport{Supported: true, MaxPageSize: 25} + // Every page is read at a fixed count, so the job must not ask for less. + return AccountContentDiscoverySupport{Supported: true, MinPageSize: 25, MaxPageSize: 25} } type peertubeChannelVideo struct { diff --git a/apps/server/internal/platform/piefed.go b/apps/server/internal/platform/piefed.go index 8197dab17..a3fcda44e 100644 --- a/apps/server/internal/platform/piefed.go +++ b/apps/server/internal/platform/piefed.go @@ -548,7 +548,8 @@ func (p *PieFedAdapter) AccountContentDiscoverySupport(input AnalyticsAccountCon if strings.TrimSpace(input.AccountID) == "" { return AccountContentDiscoverySupport{UnavailableReason: "PieFed account content discovery requires a stable account identity."} } - return AccountContentDiscoverySupport{Supported: true, MaxPageSize: 20} + // Pages are numbered at a fixed size, so the job must not ask for less. + return AccountContentDiscoverySupport{Supported: true, MinPageSize: 20, MaxPageSize: 20} } func (p *PieFedAdapter) DiscoverAccountContent(ctx context.Context, accessToken string, input AccountContentDiscoveryRequest) (AccountContentPage, error) { diff --git a/changes/discovery-fixed-page-size.md b/changes/discovery-fixed-page-size.md new file mode 100644 index 000000000..fe0c59d88 --- /dev/null +++ b/changes/discovery-fixed-page-size.md @@ -0,0 +1,3 @@ +### Fixed + +- Lemmy, PieFed, and PeerTube accounts with more than 250 posts or videos in the last 90 days finish their initial content discovery. The last page before the 250-item limit was rejected as oversized, so discovery failed on the same page every hour and never reached routine cycles.