From 82f82225d171cea9718aae27416b6f576f6ba201 Mon Sep 17 00:00:00 2001 From: Raffaello De Pieri Date: Wed, 24 Jul 2024 15:54:58 +0100 Subject: [PATCH 1/6] search-backend-module-stack-overflow-collator remove site property from StackOverflowQuestionsCollatorFactory.requestParams Signed-off-by: Raffaello De Pieri --- .../src/collators/StackOverflowQuestionsCollatorFactory.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.ts b/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.ts index 2a81d7c749..6bab62d447 100644 --- a/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.ts +++ b/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.ts @@ -86,7 +86,6 @@ export class StackOverflowQuestionsCollatorFactory this.requestParams = options.requestParams ?? { order: 'desc', sort: 'activity', - site: 'stackoverflow', ...(options.requestParams ?? {}), }; this.logger = options.logger.child({ documentType: this.type }); From e010047c68bed3815de2a822c60dfb61c5653965 Mon Sep 17 00:00:00 2001 From: Raffaello De Pieri Date: Wed, 24 Jul 2024 16:48:24 +0100 Subject: [PATCH 2/6] search-backend-module-stack-overflow-collator remove site property from StackOverflowQuestionsCollatorFactory.requestParams, add coverage to test request search expectations Signed-off-by: Raffaello De Pieri --- ...ckOverflowQuestionsCollatorFactory.test.ts | 278 ++++++++++++++++-- .../StackOverflowQuestionsCollatorFactory.ts | 2 +- 2 files changed, 250 insertions(+), 30 deletions(-) diff --git a/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.test.ts b/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.test.ts index 90b324866f..170cf15c38 100644 --- a/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.test.ts +++ b/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.test.ts @@ -68,27 +68,36 @@ const mockOverrideQuestion = { has_more: false, }; -describe('StackOverflowQuestionsCollatorFactory', () => { +const testSearchQuery = (request, expectedSearch) => { + const executedSearch = {}; + request.url.searchParams.forEach((value, key) => { + executedSearch[key] = value; + }); + expect(executedSearch).toEqual(expectedSearch); +}; + +describe('StackOverflowQuestionsCollatorFactory using custom request params', () => { const config = new ConfigReader({ stackoverflow: { baseUrl: 'http://stack.overflow.local', }, }); - const defaultOptions: StackOverflowQuestionsCollatorFactoryOptions = { - logger, - requestParams: { - tagged: ['developer-portal'], - pagesize: 100, - order: 'desc', - sort: 'activity', - }, - }; + const optionsWithCustomRequestParams: StackOverflowQuestionsCollatorFactoryOptions = + { + logger, + requestParams: { + tagged: ['developer-portal'], + pagesize: 100, + order: 'desc', + sort: 'activity', + }, + }; it('has expected type', () => { const factory = StackOverflowQuestionsCollatorFactory.fromConfig( config, - defaultOptions, + optionsWithCustomRequestParams, ); expect(factory.type).toBe('stack-overflow'); }); @@ -100,89 +109,300 @@ describe('StackOverflowQuestionsCollatorFactory', () => { it('returns a readable stream', async () => { const factory = StackOverflowQuestionsCollatorFactory.fromConfig( config, - defaultOptions, + optionsWithCustomRequestParams, ); const collator = await factory.getCollator(); expect(collator).toBeInstanceOf(Readable); }); it('fetches from the configured endpoint', async () => { + let request; worker.use( - rest.get('http://stack.overflow.local/questions', (_, res, ctx) => - res(ctx.status(200), ctx.json(mockQuestion)), - ), + rest.get('http://stack.overflow.local/questions', (req, res, ctx) => { + request = req; + + return res(ctx.status(200), ctx.json(mockQuestion)); + }), ); const factory = StackOverflowQuestionsCollatorFactory.fromConfig( config, - defaultOptions, + optionsWithCustomRequestParams, ); const collator = await factory.getCollator(); const pipeline = TestPipeline.fromCollator(collator); const { documents } = await pipeline.execute(); + const expectedSearch = { + order: 'desc', + sort: 'activity', + tagged: 'developer-portal', + page: '1', + pagesize: '100', + }; + testSearchQuery(request, expectedSearch); expect(documents).toHaveLength(mockQuestion.items.length); }); it('fetches from the overridden endpoint', async () => { + let request; worker.use( - rest.get('http://stack.overflow.override/questions', (_, res, ctx) => - res(ctx.status(200), ctx.json(mockOverrideQuestion)), + rest.get( + 'http://stack.overflow.override/questions', + (req, res, ctx) => { + request = req; + return res(ctx.status(200), ctx.json(mockOverrideQuestion)); + }, ), ); const factory = StackOverflowQuestionsCollatorFactory.fromConfig(config, { logger, baseUrl: 'http://stack.overflow.override', - requestParams: defaultOptions.requestParams, + requestParams: optionsWithCustomRequestParams.requestParams, }); const collator = await factory.getCollator(); const pipeline = TestPipeline.fromCollator(collator); const { documents } = await pipeline.execute(); + const expectedSearch = { + order: 'desc', + sort: 'activity', + tagged: 'developer-portal', + page: '1', + pagesize: '100', + }; + testSearchQuery(request, expectedSearch); expect(documents).toHaveLength(mockOverrideQuestion.items.length); }); it('uses API key when provided', async () => { + let request; worker.use( - rest.get('http://stack.overflow.override/questions', (req, res, ctx) => - req.url.searchParams.get('key') === 'abcdefg' - ? res(ctx.status(200), ctx.json(mockOverrideQuestion)) - : res(ctx.status(401), ctx.json({})), + rest.get( + 'http://stack.overflow.override/questions', + (req, res, ctx) => { + request = req; + return req.url.searchParams.get('key') === 'abcdefg' + ? res(ctx.status(200), ctx.json(mockOverrideQuestion)) + : res(ctx.status(401), ctx.json({})); + }, ), ); const factory = StackOverflowQuestionsCollatorFactory.fromConfig(config, { logger, baseUrl: 'http://stack.overflow.override', apiKey: 'abcdefg', - requestParams: defaultOptions.requestParams, + requestParams: optionsWithCustomRequestParams.requestParams, }); const collator = await factory.getCollator(); const pipeline = TestPipeline.fromCollator(collator); const { documents } = await pipeline.execute(); + const expectedSearch = { + key: 'abcdefg', + order: 'desc', + sort: 'activity', + tagged: 'developer-portal', + page: '1', + pagesize: '100', + }; + testSearchQuery(request, expectedSearch); expect(documents).toHaveLength(mockOverrideQuestion.items.length); }); it('uses teamName when provided', async () => { + let request; worker.use( - rest.get('http://stack.overflow.override/questions', (req, res, ctx) => - req.url.searchParams.get('team') === 'abcdefg' - ? res(ctx.status(200), ctx.json(mockOverrideQuestion)) - : res(ctx.status(401), ctx.json({})), + rest.get( + 'http://stack.overflow.override/questions', + (req, res, ctx) => { + request = req; + return req.url.searchParams.get('team') === 'abcdefg' + ? res(ctx.status(200), ctx.json(mockOverrideQuestion)) + : res(ctx.status(401), ctx.json({})); + }, ), ); const factory = StackOverflowQuestionsCollatorFactory.fromConfig(config, { logger, baseUrl: 'http://stack.overflow.override', teamName: 'abcdefg', - requestParams: defaultOptions.requestParams, + requestParams: optionsWithCustomRequestParams.requestParams, }); const collator = await factory.getCollator(); const pipeline = TestPipeline.fromCollator(collator); const { documents } = await pipeline.execute(); + const expectedSearch = { + team: 'abcdefg', + order: 'desc', + sort: 'activity', + tagged: 'developer-portal', + page: '1', + pagesize: '100', + }; + testSearchQuery(request, expectedSearch); + expect(documents).toHaveLength(mockOverrideQuestion.items.length); + }); + }); +}); + +describe('StackOverflowQuestionsCollatorFactory using default request params', () => { + const config = new ConfigReader({ + stackoverflow: { + baseUrl: 'http://stack.overflow.local', + }, + }); + + const optionsWithCustomRequestParams: StackOverflowQuestionsCollatorFactoryOptions = + { + logger, + }; + + it('has expected type', () => { + const factory = StackOverflowQuestionsCollatorFactory.fromConfig( + config, + optionsWithCustomRequestParams, + ); + expect(factory.type).toBe('stack-overflow'); + }); + + describe('getCollator', () => { + const worker = setupServer(); + registerMswTestHooks(worker); + + it('returns a readable stream', async () => { + const factory = StackOverflowQuestionsCollatorFactory.fromConfig( + config, + optionsWithCustomRequestParams, + ); + const collator = await factory.getCollator(); + expect(collator).toBeInstanceOf(Readable); + }); + + it('fetches from the configured endpoint', async () => { + let request; + worker.use( + rest.get('http://stack.overflow.local/questions', (req, res, ctx) => { + request = req; + + return res(ctx.status(200), ctx.json(mockQuestion)); + }), + ); + const factory = StackOverflowQuestionsCollatorFactory.fromConfig( + config, + optionsWithCustomRequestParams, + ); + const collator = await factory.getCollator(); + const pipeline = TestPipeline.fromCollator(collator); + const { documents } = await pipeline.execute(); + + const expectedSearch = { + order: 'desc', + sort: 'activity', + page: '1', + }; + testSearchQuery(request, expectedSearch); + expect(documents).toHaveLength(mockQuestion.items.length); + }); + + it('fetches from the overridden endpoint', async () => { + let request; + worker.use( + rest.get( + 'http://stack.overflow.override/questions', + (req, res, ctx) => { + request = req; + return res(ctx.status(200), ctx.json(mockOverrideQuestion)); + }, + ), + ); + const factory = StackOverflowQuestionsCollatorFactory.fromConfig(config, { + logger, + baseUrl: 'http://stack.overflow.override', + requestParams: optionsWithCustomRequestParams.requestParams, + }); + const collator = await factory.getCollator(); + + const pipeline = TestPipeline.fromCollator(collator); + const { documents } = await pipeline.execute(); + + const expectedSearch = { + order: 'desc', + sort: 'activity', + page: '1', + }; + testSearchQuery(request, expectedSearch); + expect(documents).toHaveLength(mockOverrideQuestion.items.length); + }); + + it('uses API key when provided', async () => { + let request; + worker.use( + rest.get( + 'http://stack.overflow.override/questions', + (req, res, ctx) => { + request = req; + return req.url.searchParams.get('key') === 'abcdefg' + ? res(ctx.status(200), ctx.json(mockOverrideQuestion)) + : res(ctx.status(401), ctx.json({})); + }, + ), + ); + const factory = StackOverflowQuestionsCollatorFactory.fromConfig(config, { + logger, + baseUrl: 'http://stack.overflow.override', + apiKey: 'abcdefg', + requestParams: optionsWithCustomRequestParams.requestParams, + }); + const collator = await factory.getCollator(); + + const pipeline = TestPipeline.fromCollator(collator); + const { documents } = await pipeline.execute(); + + const expectedSearch = { + key: 'abcdefg', + order: 'desc', + sort: 'activity', + page: '1', + }; + testSearchQuery(request, expectedSearch); + expect(documents).toHaveLength(mockOverrideQuestion.items.length); + }); + + it('uses teamName when provided', async () => { + let request; + worker.use( + rest.get( + 'http://stack.overflow.override/questions', + (req, res, ctx) => { + request = req; + return req.url.searchParams.get('team') === 'abcdefg' + ? res(ctx.status(200), ctx.json(mockOverrideQuestion)) + : res(ctx.status(401), ctx.json({})); + }, + ), + ); + const factory = StackOverflowQuestionsCollatorFactory.fromConfig(config, { + logger, + baseUrl: 'http://stack.overflow.override', + teamName: 'abcdefg', + requestParams: optionsWithCustomRequestParams.requestParams, + }); + const collator = await factory.getCollator(); + + const pipeline = TestPipeline.fromCollator(collator); + const { documents } = await pipeline.execute(); + + const expectedSearch = { + team: 'abcdefg', + order: 'desc', + sort: 'activity', + page: '1', + }; + testSearchQuery(request, expectedSearch); expect(documents).toHaveLength(mockOverrideQuestion.items.length); }); }); diff --git a/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.ts b/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.ts index 6bab62d447..5365b56550 100644 --- a/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.ts +++ b/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.ts @@ -83,7 +83,7 @@ export class StackOverflowQuestionsCollatorFactory this.maxPage = options.maxPage; // Sets the same default request parameters as the official API documentation // See https://api.stackexchange.com/docs/questions - this.requestParams = options.requestParams ?? { + this.requestParams = { order: 'desc', sort: 'activity', ...(options.requestParams ?? {}), From 479808f7b8521d7cc087e94d2e5c96b3358784ad Mon Sep 17 00:00:00 2001 From: Raffaello De Pieri Date: Wed, 24 Jul 2024 16:57:49 +0100 Subject: [PATCH 3/6] run changeset Signed-off-by: Raffaello De Pieri --- .changeset/quiet-deers-visit.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/quiet-deers-visit.md diff --git a/.changeset/quiet-deers-visit.md b/.changeset/quiet-deers-visit.md new file mode 100644 index 0000000000..a0776af978 --- /dev/null +++ b/.changeset/quiet-deers-visit.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-search-backend-module-stack-overflow-collator': minor +--- + +Always set default request parameters for requests to stackoverflow while allow to overwrite them. Remove site parameter as causing the request to fail. From 5e0c969b19268b221d55fbffdff52cfea9023c56 Mon Sep 17 00:00:00 2001 From: Raffaello De Pieri Date: Wed, 24 Jul 2024 23:37:21 +0100 Subject: [PATCH 4/6] fix types Signed-off-by: Raffaello De Pieri --- ...StackOverflowQuestionsCollatorFactory.test.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.test.ts b/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.test.ts index 170cf15c38..75df446db3 100644 --- a/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.test.ts +++ b/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.test.ts @@ -25,7 +25,7 @@ import { TestPipeline } from '@backstage/plugin-search-backend-node'; import { ConfigReader } from '@backstage/config'; import { Readable } from 'stream'; import { setupServer } from 'msw/node'; -import { rest } from 'msw'; +import { rest, RestRequest } from 'msw'; const logger = mockServices.logger.mock(); @@ -68,9 +68,17 @@ const mockOverrideQuestion = { has_more: false, }; -const testSearchQuery = (request, expectedSearch) => { - const executedSearch = {}; - request.url.searchParams.forEach((value, key) => { +const testSearchQuery = ( + request: RestRequest | undefined, + expectedSearch: unknown, +) => { + if (!request) { + expect(request).not.toBeFalsy(); + return; + } + + const executedSearch: { [key: string]: string } = {}; + request.url.searchParams.forEach((value: string, key: string) => { executedSearch[key] = value; }); expect(executedSearch).toEqual(expectedSearch); From 48d3eea823f9d5cb65bbdfb5d0f510bbf715d7b1 Mon Sep 17 00:00:00 2001 From: Raffaello De Pieri Date: Thu, 1 Aug 2024 13:41:51 +0100 Subject: [PATCH 5/6] handle site in request parameter add site with value "stackoverflow" to requestParams when using the default base url: https://api.stackexchange.com/2.3 Signed-off-by: Raffaello De Pieri --- ...ckOverflowQuestionsCollatorFactory.test.ts | 149 ++++++++++++++---- .../StackOverflowQuestionsCollatorFactory.ts | 17 +- 2 files changed, 131 insertions(+), 35 deletions(-) diff --git a/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.test.ts b/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.test.ts index 75df446db3..6a040ad3e8 100644 --- a/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.test.ts +++ b/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.test.ts @@ -29,6 +29,8 @@ import { rest, RestRequest } from 'msw'; const logger = mockServices.logger.mock(); +const BASE_URL = 'https://api.stackexchange.com/2.3'; + const mockQuestion = { items: [ { @@ -110,6 +112,98 @@ describe('StackOverflowQuestionsCollatorFactory using custom request params', () expect(factory.type).toBe('stack-overflow'); }); + describe('Manage site query parameter', () => { + const worker = setupServer(); + registerMswTestHooks(worker); + + it('uses site query parameter when provided and baseUrl is not default', async () => { + let request; + worker.use( + rest.get('http://stack.overflow.local/questions', (req, res, ctx) => { + request = req; + + return res(ctx.status(200), ctx.json(mockQuestion)); + }), + ); + const factory = StackOverflowQuestionsCollatorFactory.fromConfig(config, { + logger, + baseUrl: 'http://stack.overflow.local', + requestParams: { + site: 'stackoverflow', + }, + }); + + const collator = await factory.getCollator(); + const pipeline = TestPipeline.fromCollator(collator); + const { documents } = await pipeline.execute(); + const expectedSearch = { + order: 'desc', + site: 'stackoverflow', + sort: 'activity', + page: '1', + }; + testSearchQuery(request, expectedSearch); + expect(documents).toHaveLength(mockQuestion.items.length); + }); + + it('uses site query parameter when provided and baseUrl is default', async () => { + let request; + worker.use( + rest.get(`${BASE_URL}/questions`, (req, res, ctx) => { + request = req; + + return res(ctx.status(200), ctx.json(mockQuestion)); + }), + ); + const factory = StackOverflowQuestionsCollatorFactory.fromConfig(config, { + logger, + baseUrl: BASE_URL, + requestParams: { + site: 'foo_bar_baz', + }, + }); + + const collator = await factory.getCollator(); + const pipeline = TestPipeline.fromCollator(collator); + const { documents } = await pipeline.execute(); + const expectedSearch = { + order: 'desc', + site: 'foo_bar_baz', + sort: 'activity', + page: '1', + }; + testSearchQuery(request, expectedSearch); + expect(documents).toHaveLength(mockQuestion.items.length); + }); + + it('uses default when site is not provided and baseUrl is default', async () => { + let request; + worker.use( + rest.get(`${BASE_URL}/questions`, (req, res, ctx) => { + request = req; + + return res(ctx.status(200), ctx.json(mockQuestion)); + }), + ); + const factory = StackOverflowQuestionsCollatorFactory.fromConfig(config, { + logger, + baseUrl: BASE_URL, + }); + + const collator = await factory.getCollator(); + const pipeline = TestPipeline.fromCollator(collator); + const { documents } = await pipeline.execute(); + const expectedSearch = { + order: 'desc', + site: 'stackoverflow', + sort: 'activity', + page: '1', + }; + testSearchQuery(request, expectedSearch); + expect(documents).toHaveLength(mockQuestion.items.length); + }); + }); + describe('getCollator', () => { const worker = setupServer(); registerMswTestHooks(worker); @@ -260,7 +354,7 @@ describe('StackOverflowQuestionsCollatorFactory using custom request params', () describe('StackOverflowQuestionsCollatorFactory using default request params', () => { const config = new ConfigReader({ stackoverflow: { - baseUrl: 'http://stack.overflow.local', + baseUrl: BASE_URL, }, }); @@ -293,7 +387,7 @@ describe('StackOverflowQuestionsCollatorFactory using default request params', ( it('fetches from the configured endpoint', async () => { let request; worker.use( - rest.get('http://stack.overflow.local/questions', (req, res, ctx) => { + rest.get(`${BASE_URL}/questions`, (req, res, ctx) => { request = req; return res(ctx.status(200), ctx.json(mockQuestion)); @@ -309,6 +403,7 @@ describe('StackOverflowQuestionsCollatorFactory using default request params', ( const expectedSearch = { order: 'desc', + site: 'stackoverflow', sort: 'activity', page: '1', }; @@ -319,17 +414,14 @@ describe('StackOverflowQuestionsCollatorFactory using default request params', ( it('fetches from the overridden endpoint', async () => { let request; worker.use( - rest.get( - 'http://stack.overflow.override/questions', - (req, res, ctx) => { - request = req; - return res(ctx.status(200), ctx.json(mockOverrideQuestion)); - }, - ), + rest.get(`${BASE_URL}/questions`, (req, res, ctx) => { + request = req; + return res(ctx.status(200), ctx.json(mockOverrideQuestion)); + }), ); const factory = StackOverflowQuestionsCollatorFactory.fromConfig(config, { logger, - baseUrl: 'http://stack.overflow.override', + baseUrl: BASE_URL, requestParams: optionsWithCustomRequestParams.requestParams, }); const collator = await factory.getCollator(); @@ -339,6 +431,7 @@ describe('StackOverflowQuestionsCollatorFactory using default request params', ( const expectedSearch = { order: 'desc', + site: 'stackoverflow', sort: 'activity', page: '1', }; @@ -349,19 +442,16 @@ describe('StackOverflowQuestionsCollatorFactory using default request params', ( it('uses API key when provided', async () => { let request; worker.use( - rest.get( - 'http://stack.overflow.override/questions', - (req, res, ctx) => { - request = req; - return req.url.searchParams.get('key') === 'abcdefg' - ? res(ctx.status(200), ctx.json(mockOverrideQuestion)) - : res(ctx.status(401), ctx.json({})); - }, - ), + rest.get(`${BASE_URL}/questions`, (req, res, ctx) => { + request = req; + return req.url.searchParams.get('key') === 'abcdefg' + ? res(ctx.status(200), ctx.json(mockOverrideQuestion)) + : res(ctx.status(401), ctx.json({})); + }), ); const factory = StackOverflowQuestionsCollatorFactory.fromConfig(config, { logger, - baseUrl: 'http://stack.overflow.override', + baseUrl: BASE_URL, apiKey: 'abcdefg', requestParams: optionsWithCustomRequestParams.requestParams, }); @@ -373,6 +463,7 @@ describe('StackOverflowQuestionsCollatorFactory using default request params', ( const expectedSearch = { key: 'abcdefg', order: 'desc', + site: 'stackoverflow', sort: 'activity', page: '1', }; @@ -383,19 +474,16 @@ describe('StackOverflowQuestionsCollatorFactory using default request params', ( it('uses teamName when provided', async () => { let request; worker.use( - rest.get( - 'http://stack.overflow.override/questions', - (req, res, ctx) => { - request = req; - return req.url.searchParams.get('team') === 'abcdefg' - ? res(ctx.status(200), ctx.json(mockOverrideQuestion)) - : res(ctx.status(401), ctx.json({})); - }, - ), + rest.get(`${BASE_URL}/questions`, (req, res, ctx) => { + request = req; + return req.url.searchParams.get('team') === 'abcdefg' + ? res(ctx.status(200), ctx.json(mockOverrideQuestion)) + : res(ctx.status(401), ctx.json({})); + }), ); const factory = StackOverflowQuestionsCollatorFactory.fromConfig(config, { logger, - baseUrl: 'http://stack.overflow.override', + baseUrl: BASE_URL, teamName: 'abcdefg', requestParams: optionsWithCustomRequestParams.requestParams, }); @@ -407,6 +495,7 @@ describe('StackOverflowQuestionsCollatorFactory using default request params', ( const expectedSearch = { team: 'abcdefg', order: 'desc', + site: 'stackoverflow', sort: 'activity', page: '1', }; diff --git a/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.ts b/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.ts index 5365b56550..e8c7b41781 100644 --- a/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.ts +++ b/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.ts @@ -40,7 +40,7 @@ export interface StackOverflowDocument extends IndexableDocument { * @public */ export type StackOverflowQuestionsRequestParams = { - [key: string]: string | string[] | number; + [key: string]: string | string[] | number | null; }; /** @@ -58,6 +58,8 @@ export type StackOverflowQuestionsCollatorFactoryOptions = { logger: LoggerService; }; +const DEFAULT_BASE_URL = 'https://api.stackexchange.com/2.3'; +const DEFAULT_MAX_PAGE = 100; /** * Search collator responsible for collecting stack overflow questions to index. * @@ -81,6 +83,8 @@ export class StackOverflowQuestionsCollatorFactory this.apiAccessToken = options.apiAccessToken; this.teamName = options.teamName; this.maxPage = options.maxPage; + this.logger = options.logger.child({ documentType: this.type }); + // Sets the same default request parameters as the official API documentation // See https://api.stackexchange.com/docs/questions this.requestParams = { @@ -88,7 +92,10 @@ export class StackOverflowQuestionsCollatorFactory sort: 'activity', ...(options.requestParams ?? {}), }; - this.logger = options.logger.child({ documentType: this.type }); + + if (!options.requestParams?.site && this.baseUrl === DEFAULT_BASE_URL) { + this.requestParams.site = 'stackoverflow'; + } } static fromConfig( @@ -101,12 +108,12 @@ export class StackOverflowQuestionsCollatorFactory ); const teamName = config.getOptionalString('stackoverflow.teamName'); const baseUrl = - config.getOptionalString('stackoverflow.baseUrl') || - 'https://api.stackexchange.com/2.3'; - const maxPage = options.maxPage || 100; + config.getOptionalString('stackoverflow.baseUrl') || DEFAULT_BASE_URL; + const maxPage = options.maxPage || DEFAULT_MAX_PAGE; const requestParams = config .getOptionalConfig('stackoverflow.requestParams') ?.get(); + return new StackOverflowQuestionsCollatorFactory({ baseUrl, maxPage, From 31addbbdda76b473cef3e4d649311f10b8428b24 Mon Sep 17 00:00:00 2001 From: Raffaello De Pieri Date: Thu, 1 Aug 2024 14:17:40 +0100 Subject: [PATCH 6/6] revert(StackOverflowQuestionsRequestParams): revert signature removing unecessary type in StackOverflowQuestionsRequestParams Signed-off-by: Raffaello De Pieri --- .../src/collators/StackOverflowQuestionsCollatorFactory.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.ts b/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.ts index e8c7b41781..33ceb4f870 100644 --- a/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.ts +++ b/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.ts @@ -40,7 +40,7 @@ export interface StackOverflowDocument extends IndexableDocument { * @public */ export type StackOverflowQuestionsRequestParams = { - [key: string]: string | string[] | number | null; + [key: string]: string | string[] | number; }; /**