diff --git a/.changeset/curly-hounds-wait.md b/.changeset/curly-hounds-wait.md new file mode 100644 index 0000000000..58ebfe89a3 --- /dev/null +++ b/.changeset/curly-hounds-wait.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-stack-overflow-backend': patch +--- + +Now requests all questions available using pagination. Default max page is set to 100, with a configurable `maxPage` option on the collator. diff --git a/plugins/stack-overflow-backend/api-report.md b/plugins/stack-overflow-backend/api-report.md index 1a21ff34ac..11bfbaadc9 100644 --- a/plugins/stack-overflow-backend/api-report.md +++ b/plugins/stack-overflow-backend/api-report.md @@ -41,6 +41,7 @@ export class StackOverflowQuestionsCollatorFactory // @public export type StackOverflowQuestionsCollatorFactoryOptions = { baseUrl?: string; + maxPage?: number; apiKey?: string; requestParams: StackOverflowQuestionsRequestParams; logger: Logger; diff --git a/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts b/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts index 45261ce546..dc82f77a86 100644 --- a/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts +++ b/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts @@ -50,6 +50,7 @@ export type StackOverflowQuestionsRequestParams = { */ export type StackOverflowQuestionsCollatorFactoryOptions = { baseUrl?: string; + maxPage?: number; apiKey?: string; requestParams: StackOverflowQuestionsRequestParams; logger: Logger; @@ -66,12 +67,14 @@ export class StackOverflowQuestionsCollatorFactory protected requestParams: StackOverflowQuestionsRequestParams; private readonly baseUrl: string | undefined; private readonly apiKey: string | undefined; + private readonly maxPage: number | undefined; private readonly logger: Logger; public readonly type: string = 'stack-overflow'; private constructor(options: StackOverflowQuestionsCollatorFactoryOptions) { this.baseUrl = options.baseUrl; this.apiKey = options.apiKey; + this.maxPage = options.maxPage; this.requestParams = options.requestParams; this.logger = options.logger; } @@ -84,9 +87,11 @@ export class StackOverflowQuestionsCollatorFactory const baseUrl = config.getOptionalString('stackoverflow.baseUrl') || 'https://api.stackexchange.com/2.2'; + const maxPage = options.maxPage || 100; return new StackOverflowQuestionsCollatorFactory({ ...options, baseUrl, + maxPage, apiKey, }); } @@ -122,17 +127,31 @@ export class StackOverflowQuestionsCollatorFactory ? `${params ? '&' : '?'}key=${this.apiKey}` : ''; - const res = await fetch(`${this.baseUrl}/questions${params}${apiKeyParam}`); - const data = await res.json(); + let hasMorePages = true; + let page = 1; + while (hasMorePages) { + if (page === this.maxPage) { + this.logger.warn( + `Over ${this.maxPage} requests to the Stack Overflow API have been made, which may not have been intended. Either specify requestParams that limit the questions returned, or configure a higher maxPage if necessary.`, + ); + break; + } + const res = await fetch( + `${this.baseUrl}/questions${params}${apiKeyParam}&page=${page}`, + ); - for (const question of data.items) { - yield { - title: question.title, - location: question.link, - text: question.owner.display_name, - tags: question.tags, - answers: question.answer_count, - }; + const data = await res.json(); + for (const question of data.items) { + yield { + title: question.title, + location: question.link, + text: question.owner.display_name, + tags: question.tags, + answers: question.answer_count, + }; + } + hasMorePages = data.has_more; + page = page + 1; } } }