Merge pull request #13253 from backstage/emmaindal/stack-overflow-collator-paginated-request

plugins/stack-overflow-backend/api-report.md

@@ -41,6 +41,7 @@ export class StackOverflowQuestionsCollatorFactory
 // @public
 export type StackOverflowQuestionsCollatorFactoryOptions = {
   baseUrl?: string;
+  maxPage?: number;
   apiKey?: string;
   requestParams: StackOverflowQuestionsRequestParams;
   logger: Logger;

plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts

@@ -50,6 +50,7 @@ export type StackOverflowQuestionsRequestParams = {
  */
 export type StackOverflowQuestionsCollatorFactoryOptions = {
   baseUrl?: string;
+  maxPage?: number;
   apiKey?: string;
   requestParams: StackOverflowQuestionsRequestParams;
   logger: Logger;
@@ -66,12 +67,14 @@ export class StackOverflowQuestionsCollatorFactory
   protected requestParams: StackOverflowQuestionsRequestParams;
   private readonly baseUrl: string | undefined;
   private readonly apiKey: string | undefined;
+  private readonly maxPage: number | undefined;
   private readonly logger: Logger;
   public readonly type: string = 'stack-overflow';
 
   private constructor(options: StackOverflowQuestionsCollatorFactoryOptions) {
     this.baseUrl = options.baseUrl;
     this.apiKey = options.apiKey;
+    this.maxPage = options.maxPage;
     this.requestParams = options.requestParams;
     this.logger = options.logger;
   }
@@ -84,9 +87,11 @@ export class StackOverflowQuestionsCollatorFactory
     const baseUrl =
       config.getOptionalString('stackoverflow.baseUrl') ||
       'https://api.stackexchange.com/2.2';
+    const maxPage = options.maxPage || 100;
     return new StackOverflowQuestionsCollatorFactory({
       ...options,
       baseUrl,
+      maxPage,
       apiKey,
     });
   }
@@ -122,17 +127,31 @@ export class StackOverflowQuestionsCollatorFactory
       ? `${params ? '&' : '?'}key=${this.apiKey}`
       : '';
 
-    const res = await fetch(`${this.baseUrl}/questions${params}${apiKeyParam}`);
-    const data = await res.json();
+    let hasMorePages = true;
+    let page = 1;
+    while (hasMorePages) {
+      if (page === this.maxPage) {
+        this.logger.warn(
+          `Over ${this.maxPage} requests to the Stack Overflow API have been made, which may not have been intended. Either specify requestParams that limit the questions returned, or configure a higher maxPage if necessary.`,
+        );
+        break;
+      }
+      const res = await fetch(
+        `${this.baseUrl}/questions${params}${apiKeyParam}&page=${page}`,
+      );
 
-    for (const question of data.items) {
-      yield {
-        title: question.title,
-        location: question.link,
-        text: question.owner.display_name,
-        tags: question.tags,
-        answers: question.answer_count,
-      };
+      const data = await res.json();
+      for (const question of data.items) {
+        yield {
+          title: question.title,
+          location: question.link,
+          text: question.owner.display_name,
+          tags: question.tags,
+          answers: question.answer_count,
+        };
+      }
+      hasMorePages = data.has_more;
+      page = page + 1;
     }
   }
 }