From 79040f73f7bb969b5e5233f72f52be76b55e0b08 Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Fri, 19 Aug 2022 15:05:14 +0200 Subject: [PATCH 1/6] add pagination to request for stack overflow questions Signed-off-by: Emma Indal --- .changeset/curly-hounds-wait.md | 5 +++ .../StackOverflowQuestionsCollatorFactory.ts | 41 ++++++++++++++----- 2 files changed, 35 insertions(+), 11 deletions(-) create mode 100644 .changeset/curly-hounds-wait.md diff --git a/.changeset/curly-hounds-wait.md b/.changeset/curly-hounds-wait.md new file mode 100644 index 0000000000..0d0a0865eb --- /dev/null +++ b/.changeset/curly-hounds-wait.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-stack-overflow-backend': patch +--- + +Now requests all questions available using pagination. diff --git a/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts b/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts index 45261ce546..1bce4fee46 100644 --- a/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts +++ b/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts @@ -95,6 +95,35 @@ export class StackOverflowQuestionsCollatorFactory return Readable.from(this.execute()); } + async getQuestions() { + let hasMorePages = true; + let page = 1; + const items = []; + + const params = qs.stringify(this.requestParams, { + arrayFormat: 'comma', + addQueryPrefix: true, + }); + + const apiKeyParam = this.apiKey + ? `${params ? '&' : '?'}key=${this.apiKey}` + : ''; + + while (hasMorePages) { + const res = await fetch( + `${this.baseUrl}/questions${params}${apiKeyParam}&page=${page}`, + ); + + const data = await res.json(); + items.push(...data.items); + hasMorePages = data.has_more; + page = page + 1; + } + return { + items, + }; + } + async *execute(): AsyncGenerator { if (!this.baseUrl) { this.logger.debug( @@ -113,17 +142,7 @@ export class StackOverflowQuestionsCollatorFactory this.logger.error(`Caught ${e}`); } - const params = qs.stringify(this.requestParams, { - arrayFormat: 'comma', - addQueryPrefix: true, - }); - - const apiKeyParam = this.apiKey - ? `${params ? '&' : '?'}key=${this.apiKey}` - : ''; - - const res = await fetch(`${this.baseUrl}/questions${params}${apiKeyParam}`); - const data = await res.json(); + const data = await this.getQuestions(); for (const question of data.items) { yield { From 0df9a2639fa262aa95dabcbe1cfb720fca79cb7e Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Fri, 19 Aug 2022 16:18:54 +0200 Subject: [PATCH 2/6] yield questions as we request them Signed-off-by: Emma Indal --- .../StackOverflowQuestionsCollatorFactory.ts | 65 ++++++++----------- 1 file changed, 27 insertions(+), 38 deletions(-) diff --git a/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts b/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts index 1bce4fee46..5803ef4b6b 100644 --- a/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts +++ b/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts @@ -95,35 +95,6 @@ export class StackOverflowQuestionsCollatorFactory return Readable.from(this.execute()); } - async getQuestions() { - let hasMorePages = true; - let page = 1; - const items = []; - - const params = qs.stringify(this.requestParams, { - arrayFormat: 'comma', - addQueryPrefix: true, - }); - - const apiKeyParam = this.apiKey - ? `${params ? '&' : '?'}key=${this.apiKey}` - : ''; - - while (hasMorePages) { - const res = await fetch( - `${this.baseUrl}/questions${params}${apiKeyParam}&page=${page}`, - ); - - const data = await res.json(); - items.push(...data.items); - hasMorePages = data.has_more; - page = page + 1; - } - return { - items, - }; - } - async *execute(): AsyncGenerator { if (!this.baseUrl) { this.logger.debug( @@ -142,16 +113,34 @@ export class StackOverflowQuestionsCollatorFactory this.logger.error(`Caught ${e}`); } - const data = await this.getQuestions(); + const params = qs.stringify(this.requestParams, { + arrayFormat: 'comma', + addQueryPrefix: true, + }); - for (const question of data.items) { - yield { - title: question.title, - location: question.link, - text: question.owner.display_name, - tags: question.tags, - answers: question.answer_count, - }; + const apiKeyParam = this.apiKey + ? `${params ? '&' : '?'}key=${this.apiKey}` + : ''; + + let hasMorePages = true; + let page = 1; + while (hasMorePages) { + const res = await fetch( + `${this.baseUrl}/questions${params}${apiKeyParam}&page=${page}`, + ); + + const data = await res.json(); + for (const question of data.items) { + yield { + title: question.title, + location: question.link, + text: question.owner.display_name, + tags: question.tags, + answers: question.answer_count, + }; + } + hasMorePages = data.has_more; + page = page + 1; } } } From 761d540ed2fa51b711c4eacb6c04be109764abf8 Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Fri, 19 Aug 2022 16:37:59 +0200 Subject: [PATCH 3/6] set max page to limit requests Signed-off-by: Emma Indal --- .../search/StackOverflowQuestionsCollatorFactory.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts b/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts index 5803ef4b6b..a98fc7a2bc 100644 --- a/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts +++ b/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts @@ -50,6 +50,7 @@ export type StackOverflowQuestionsRequestParams = { */ export type StackOverflowQuestionsCollatorFactoryOptions = { baseUrl?: string; + maxPage?: number; apiKey?: string; requestParams: StackOverflowQuestionsRequestParams; logger: Logger; @@ -66,12 +67,14 @@ export class StackOverflowQuestionsCollatorFactory protected requestParams: StackOverflowQuestionsRequestParams; private readonly baseUrl: string | undefined; private readonly apiKey: string | undefined; + private readonly maxPage: number | undefined; private readonly logger: Logger; public readonly type: string = 'stack-overflow'; private constructor(options: StackOverflowQuestionsCollatorFactoryOptions) { this.baseUrl = options.baseUrl; this.apiKey = options.apiKey; + this.maxPage = options.maxPage; this.requestParams = options.requestParams; this.logger = options.logger; } @@ -84,9 +87,11 @@ export class StackOverflowQuestionsCollatorFactory const baseUrl = config.getOptionalString('stackoverflow.baseUrl') || 'https://api.stackexchange.com/2.2'; + const maxPage = options.maxPage || 100; return new StackOverflowQuestionsCollatorFactory({ ...options, baseUrl, + maxPage, apiKey, }); } @@ -125,6 +130,12 @@ export class StackOverflowQuestionsCollatorFactory let hasMorePages = true; let page = 1; while (hasMorePages) { + if (page === this.maxPage) { + this.logger.warn( + 'You have made over 100 requests to the Stack Overflow API, we recommend you to specify requestParams to limit your search to not reach your API limit or configure your own maxPage.', + ); + break; + } const res = await fetch( `${this.baseUrl}/questions${params}${apiKeyParam}&page=${page}`, ); From da39f32f0d6163f92d5ed518c722f15bb6f4fd1e Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Fri, 19 Aug 2022 16:43:00 +0200 Subject: [PATCH 4/6] update changeset Signed-off-by: Emma Indal --- .changeset/curly-hounds-wait.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/curly-hounds-wait.md b/.changeset/curly-hounds-wait.md index 0d0a0865eb..58ebfe89a3 100644 --- a/.changeset/curly-hounds-wait.md +++ b/.changeset/curly-hounds-wait.md @@ -2,4 +2,4 @@ '@backstage/plugin-stack-overflow-backend': patch --- -Now requests all questions available using pagination. +Now requests all questions available using pagination. Default max page is set to 100, with a configurable `maxPage` option on the collator. From a7d4cc2583530c35dce99a37d76ad1e939da976f Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Mon, 22 Aug 2022 08:50:07 +0200 Subject: [PATCH 5/6] update api report Signed-off-by: Emma Indal --- plugins/stack-overflow-backend/api-report.md | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/stack-overflow-backend/api-report.md b/plugins/stack-overflow-backend/api-report.md index 1a21ff34ac..11bfbaadc9 100644 --- a/plugins/stack-overflow-backend/api-report.md +++ b/plugins/stack-overflow-backend/api-report.md @@ -41,6 +41,7 @@ export class StackOverflowQuestionsCollatorFactory // @public export type StackOverflowQuestionsCollatorFactoryOptions = { baseUrl?: string; + maxPage?: number; apiKey?: string; requestParams: StackOverflowQuestionsRequestParams; logger: Logger; From f75d4fb1c03addf682398f5ee780f39be24a924e Mon Sep 17 00:00:00 2001 From: Emma Indal Date: Mon, 22 Aug 2022 08:50:39 +0200 Subject: [PATCH 6/6] Update plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts Co-authored-by: Eric Peterson Signed-off-by: Emma Indal --- .../src/search/StackOverflowQuestionsCollatorFactory.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts b/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts index a98fc7a2bc..dc82f77a86 100644 --- a/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts +++ b/plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts @@ -132,7 +132,7 @@ export class StackOverflowQuestionsCollatorFactory while (hasMorePages) { if (page === this.maxPage) { this.logger.warn( - 'You have made over 100 requests to the Stack Overflow API, we recommend you to specify requestParams to limit your search to not reach your API limit or configure your own maxPage.', + `Over ${this.maxPage} requests to the Stack Overflow API have been made, which may not have been intended. Either specify requestParams that limit the questions returned, or configure a higher maxPage if necessary.`, ); break; }