From 76185f8275bfddbd954d1f140ffeb3e02df0c9c5 Mon Sep 17 00:00:00 2001 From: kevin9327 <5299031+kevin9327@users.noreply.github.com> Date: Sat, 19 Sep 2026 07:52:56 +0900 Subject: [PATCH] fix(community): declare the fixed discovery page size as the minimum Lemmy, PieFed, and PeerTube discovery read fixed pages of 20, 20, and 25 and ignore the requested page size, but declared no MinPageSize. Near the 250-item initial limit the discovery job asks for the smaller remainder and rejects a page with more items than it asked for, so an account with more than 250 items in the window failed on the same page every hour and never finished its initial cycle. Declaring the fixed size as the minimum lets the job stop at the limit instead, the way it already does for X. Co-Authored-By: Claude Opus 5 --- .../internal/platform/account_content_test.go | 63 +++++++++++++++++++ apps/server/internal/platform/lemmy.go | 3 +- apps/server/internal/platform/peertube.go | 3 +- apps/server/internal/platform/piefed.go | 3 +- changes/discovery-fixed-page-size.md | 3 + 5 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 changes/discovery-fixed-page-size.md diff --git a/apps/server/internal/platform/account_content_test.go b/apps/server/internal/platform/account_content_test.go index 7046bf59e..441dbca5c 100644 --- a/apps/server/internal/platform/account_content_test.go +++ b/apps/server/internal/platform/account_content_test.go @@ -1,6 +1,9 @@ package platform import ( + "fmt" + "net/http" + "strconv" "strings" "testing" "time" @@ -8,6 +11,66 @@ import ( "github.com/stretchr/testify/require" ) +// The discovery job asks for pages no smaller than MinPageSize and rejects a +// page with more items than it asked for. Lemmy, PieFed, and PeerTube read +// fixed-size pages, so the smallest page they accept must be that size. +func TestFixedPageDiscoveryNeverReturnsMoreThanTheSmallestRequestedPage(t *testing.T) { + originalClient := httpClient + defer func() { httpClient = originalClient }() + + const instanceURL = "https://fed.example" + published := time.Now().UTC().Add(-time.Hour).Format(time.RFC3339) + full := func(limit string, item func(id int) string) string { + size, _ := strconv.Atoi(limit) + items := make([]string, 0, size) + for id := 1; id <= size; id++ { + items = append(items, item(id)) + } + return strings.Join(items, ",") + } + httpClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + query := req.URL.Query() + switch req.URL.Path { + case "/api/v3/user": + return jsonResponse(req, `{"posts":[`+full(query.Get("limit"), func(id int) string { + return fmt.Sprintf(`{"post":{"id":%d,"name":"Post","ap_id":"%s/post/%d","published":%q}}`, id, instanceURL, id, published) + })+`]}`), nil + case "/api/alpha/post/list": + return jsonResponse(req, `{"posts":[`+full(query.Get("limit"), func(id int) string { + return fmt.Sprintf(`{"post":{"id":%d,"title":"Post","ap_id":"%s/post/%d","published":%q}}`, id, instanceURL, id, published) + })+`],"next_page":"2"}`), nil + case "/api/v1/video-channels/demos/videos": + return jsonResponse(req, `{"total":1000,"data":[`+full(query.Get("count"), func(id int) string { + return fmt.Sprintf(`{"uuid":"uuid-%d","name":"Video","publishedAt":%q}`, id, published) + })+`]}`), nil + } + return jsonResponseWithStatus(req, http.StatusNotFound, `{}`), nil + })} + + tests := []struct { + name string + discoverer AccountContentDiscoverer + accountID string + }{ + {name: "lemmy", discoverer: NewLemmyAdapter(instanceURL), accountID: "5"}, + {name: "piefed", discoverer: NewPieFedAdapter(instanceURL), accountID: "5"}, + {name: "peertube", discoverer: NewPeerTubeAdapter(instanceURL), accountID: "demos"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + support := test.discoverer.AccountContentDiscoverySupport(AnalyticsAccountContext{AccountID: test.accountID}) + require.True(t, support.Supported) + smallest := max(1, support.MinPageSize) + page, err := test.discoverer.DiscoverAccountContent(t.Context(), "token", AccountContentDiscoveryRequest{ + AccountID: test.accountID, PageSize: smallest, PublishedAfter: time.Now().UTC().Add(-90 * 24 * time.Hour), + }) + require.NoError(t, err) + require.NotEmpty(t, page.Items) + require.LessOrEqual(t, len(page.Items), smallest, "a page must fit the smallest page size the adapter declares") + }) + } +} + func TestNormalizeAccountContentItemBoundsTextAndRejectsUnsafeProviderURLs(t *testing.T) { t.Parallel() diff --git a/apps/server/internal/platform/lemmy.go b/apps/server/internal/platform/lemmy.go index bb3708f5c..a5f2a5314 100644 --- a/apps/server/internal/platform/lemmy.go +++ b/apps/server/internal/platform/lemmy.go @@ -577,7 +577,8 @@ func (l *LemmyAdapter) AccountContentDiscoverySupport(input AnalyticsAccountCont if strings.TrimSpace(input.AccountID) == "" { return AccountContentDiscoverySupport{UnavailableReason: "Lemmy account content discovery requires a stable account identity."} } - return AccountContentDiscoverySupport{Supported: true, MaxPageSize: 20} + // Pages are numbered at a fixed size, so the job must not ask for less. + return AccountContentDiscoverySupport{Supported: true, MinPageSize: lemmyAccountContentPageSize, MaxPageSize: lemmyAccountContentPageSize} } func (l *LemmyAdapter) DiscoverAccountContent(ctx context.Context, accessToken string, input AccountContentDiscoveryRequest) (AccountContentPage, error) { diff --git a/apps/server/internal/platform/peertube.go b/apps/server/internal/platform/peertube.go index 573fab5ec..6f95b1546 100644 --- a/apps/server/internal/platform/peertube.go +++ b/apps/server/internal/platform/peertube.go @@ -827,7 +827,8 @@ func (p *PeerTubeAdapter) AccountContentDiscoverySupport(input AnalyticsAccountC if strings.TrimSpace(input.AccountID) == "" { return AccountContentDiscoverySupport{UnavailableReason: "PeerTube account content discovery requires a channel."} } - return AccountContentDiscoverySupport{Supported: true, MaxPageSize: 25} + // Every page is read at a fixed count, so the job must not ask for less. + return AccountContentDiscoverySupport{Supported: true, MinPageSize: 25, MaxPageSize: 25} } type peertubeChannelVideo struct { diff --git a/apps/server/internal/platform/piefed.go b/apps/server/internal/platform/piefed.go index 8197dab17..a3fcda44e 100644 --- a/apps/server/internal/platform/piefed.go +++ b/apps/server/internal/platform/piefed.go @@ -548,7 +548,8 @@ func (p *PieFedAdapter) AccountContentDiscoverySupport(input AnalyticsAccountCon if strings.TrimSpace(input.AccountID) == "" { return AccountContentDiscoverySupport{UnavailableReason: "PieFed account content discovery requires a stable account identity."} } - return AccountContentDiscoverySupport{Supported: true, MaxPageSize: 20} + // Pages are numbered at a fixed size, so the job must not ask for less. + return AccountContentDiscoverySupport{Supported: true, MinPageSize: 20, MaxPageSize: 20} } func (p *PieFedAdapter) DiscoverAccountContent(ctx context.Context, accessToken string, input AccountContentDiscoveryRequest) (AccountContentPage, error) { diff --git a/changes/discovery-fixed-page-size.md b/changes/discovery-fixed-page-size.md new file mode 100644 index 000000000..fe0c59d88 --- /dev/null +++ b/changes/discovery-fixed-page-size.md @@ -0,0 +1,3 @@ +### Fixed + +- Lemmy, PieFed, and PeerTube accounts with more than 250 posts or videos in the last 90 days finish their initial content discovery. The last page before the 250-item limit was rejected as oversized, so discovery failed on the same page every hour and never reached routine cycles.