diff --git a/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.test.ts b/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.test.ts index 75df446db3..6a040ad3e8 100644 --- a/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.test.ts +++ b/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.test.ts @@ -29,6 +29,8 @@ import { rest, RestRequest } from 'msw'; const logger = mockServices.logger.mock(); +const BASE_URL = 'https://api.stackexchange.com/2.3'; + const mockQuestion = { items: [ { @@ -110,6 +112,98 @@ describe('StackOverflowQuestionsCollatorFactory using custom request params', () expect(factory.type).toBe('stack-overflow'); }); + describe('Manage site query parameter', () => { + const worker = setupServer(); + registerMswTestHooks(worker); + + it('uses site query parameter when provided and baseUrl is not default', async () => { + let request; + worker.use( + rest.get('http://stack.overflow.local/questions', (req, res, ctx) => { + request = req; + + return res(ctx.status(200), ctx.json(mockQuestion)); + }), + ); + const factory = StackOverflowQuestionsCollatorFactory.fromConfig(config, { + logger, + baseUrl: 'http://stack.overflow.local', + requestParams: { + site: 'stackoverflow', + }, + }); + + const collator = await factory.getCollator(); + const pipeline = TestPipeline.fromCollator(collator); + const { documents } = await pipeline.execute(); + const expectedSearch = { + order: 'desc', + site: 'stackoverflow', + sort: 'activity', + page: '1', + }; + testSearchQuery(request, expectedSearch); + expect(documents).toHaveLength(mockQuestion.items.length); + }); + + it('uses site query parameter when provided and baseUrl is default', async () => { + let request; + worker.use( + rest.get(`${BASE_URL}/questions`, (req, res, ctx) => { + request = req; + + return res(ctx.status(200), ctx.json(mockQuestion)); + }), + ); + const factory = StackOverflowQuestionsCollatorFactory.fromConfig(config, { + logger, + baseUrl: BASE_URL, + requestParams: { + site: 'foo_bar_baz', + }, + }); + + const collator = await factory.getCollator(); + const pipeline = TestPipeline.fromCollator(collator); + const { documents } = await pipeline.execute(); + const expectedSearch = { + order: 'desc', + site: 'foo_bar_baz', + sort: 'activity', + page: '1', + }; + testSearchQuery(request, expectedSearch); + expect(documents).toHaveLength(mockQuestion.items.length); + }); + + it('uses default when site is not provided and baseUrl is default', async () => { + let request; + worker.use( + rest.get(`${BASE_URL}/questions`, (req, res, ctx) => { + request = req; + + return res(ctx.status(200), ctx.json(mockQuestion)); + }), + ); + const factory = StackOverflowQuestionsCollatorFactory.fromConfig(config, { + logger, + baseUrl: BASE_URL, + }); + + const collator = await factory.getCollator(); + const pipeline = TestPipeline.fromCollator(collator); + const { documents } = await pipeline.execute(); + const expectedSearch = { + order: 'desc', + site: 'stackoverflow', + sort: 'activity', + page: '1', + }; + testSearchQuery(request, expectedSearch); + expect(documents).toHaveLength(mockQuestion.items.length); + }); + }); + describe('getCollator', () => { const worker = setupServer(); registerMswTestHooks(worker); @@ -260,7 +354,7 @@ describe('StackOverflowQuestionsCollatorFactory using custom request params', () describe('StackOverflowQuestionsCollatorFactory using default request params', () => { const config = new ConfigReader({ stackoverflow: { - baseUrl: 'http://stack.overflow.local', + baseUrl: BASE_URL, }, }); @@ -293,7 +387,7 @@ describe('StackOverflowQuestionsCollatorFactory using default request params', ( it('fetches from the configured endpoint', async () => { let request; worker.use( - rest.get('http://stack.overflow.local/questions', (req, res, ctx) => { + rest.get(`${BASE_URL}/questions`, (req, res, ctx) => { request = req; return res(ctx.status(200), ctx.json(mockQuestion)); @@ -309,6 +403,7 @@ describe('StackOverflowQuestionsCollatorFactory using default request params', ( const expectedSearch = { order: 'desc', + site: 'stackoverflow', sort: 'activity', page: '1', }; @@ -319,17 +414,14 @@ describe('StackOverflowQuestionsCollatorFactory using default request params', ( it('fetches from the overridden endpoint', async () => { let request; worker.use( - rest.get( - 'http://stack.overflow.override/questions', - (req, res, ctx) => { - request = req; - return res(ctx.status(200), ctx.json(mockOverrideQuestion)); - }, - ), + rest.get(`${BASE_URL}/questions`, (req, res, ctx) => { + request = req; + return res(ctx.status(200), ctx.json(mockOverrideQuestion)); + }), ); const factory = StackOverflowQuestionsCollatorFactory.fromConfig(config, { logger, - baseUrl: 'http://stack.overflow.override', + baseUrl: BASE_URL, requestParams: optionsWithCustomRequestParams.requestParams, }); const collator = await factory.getCollator(); @@ -339,6 +431,7 @@ describe('StackOverflowQuestionsCollatorFactory using default request params', ( const expectedSearch = { order: 'desc', + site: 'stackoverflow', sort: 'activity', page: '1', }; @@ -349,19 +442,16 @@ describe('StackOverflowQuestionsCollatorFactory using default request params', ( it('uses API key when provided', async () => { let request; worker.use( - rest.get( - 'http://stack.overflow.override/questions', - (req, res, ctx) => { - request = req; - return req.url.searchParams.get('key') === 'abcdefg' - ? res(ctx.status(200), ctx.json(mockOverrideQuestion)) - : res(ctx.status(401), ctx.json({})); - }, - ), + rest.get(`${BASE_URL}/questions`, (req, res, ctx) => { + request = req; + return req.url.searchParams.get('key') === 'abcdefg' + ? res(ctx.status(200), ctx.json(mockOverrideQuestion)) + : res(ctx.status(401), ctx.json({})); + }), ); const factory = StackOverflowQuestionsCollatorFactory.fromConfig(config, { logger, - baseUrl: 'http://stack.overflow.override', + baseUrl: BASE_URL, apiKey: 'abcdefg', requestParams: optionsWithCustomRequestParams.requestParams, }); @@ -373,6 +463,7 @@ describe('StackOverflowQuestionsCollatorFactory using default request params', ( const expectedSearch = { key: 'abcdefg', order: 'desc', + site: 'stackoverflow', sort: 'activity', page: '1', }; @@ -383,19 +474,16 @@ describe('StackOverflowQuestionsCollatorFactory using default request params', ( it('uses teamName when provided', async () => { let request; worker.use( - rest.get( - 'http://stack.overflow.override/questions', - (req, res, ctx) => { - request = req; - return req.url.searchParams.get('team') === 'abcdefg' - ? res(ctx.status(200), ctx.json(mockOverrideQuestion)) - : res(ctx.status(401), ctx.json({})); - }, - ), + rest.get(`${BASE_URL}/questions`, (req, res, ctx) => { + request = req; + return req.url.searchParams.get('team') === 'abcdefg' + ? res(ctx.status(200), ctx.json(mockOverrideQuestion)) + : res(ctx.status(401), ctx.json({})); + }), ); const factory = StackOverflowQuestionsCollatorFactory.fromConfig(config, { logger, - baseUrl: 'http://stack.overflow.override', + baseUrl: BASE_URL, teamName: 'abcdefg', requestParams: optionsWithCustomRequestParams.requestParams, }); @@ -407,6 +495,7 @@ describe('StackOverflowQuestionsCollatorFactory using default request params', ( const expectedSearch = { team: 'abcdefg', order: 'desc', + site: 'stackoverflow', sort: 'activity', page: '1', }; diff --git a/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.ts b/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.ts index 5365b56550..e8c7b41781 100644 --- a/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.ts +++ b/plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.ts @@ -40,7 +40,7 @@ export interface StackOverflowDocument extends IndexableDocument { * @public */ export type StackOverflowQuestionsRequestParams = { - [key: string]: string | string[] | number; + [key: string]: string | string[] | number | null; }; /** @@ -58,6 +58,8 @@ export type StackOverflowQuestionsCollatorFactoryOptions = { logger: LoggerService; }; +const DEFAULT_BASE_URL = 'https://api.stackexchange.com/2.3'; +const DEFAULT_MAX_PAGE = 100; /** * Search collator responsible for collecting stack overflow questions to index. * @@ -81,6 +83,8 @@ export class StackOverflowQuestionsCollatorFactory this.apiAccessToken = options.apiAccessToken; this.teamName = options.teamName; this.maxPage = options.maxPage; + this.logger = options.logger.child({ documentType: this.type }); + // Sets the same default request parameters as the official API documentation // See https://api.stackexchange.com/docs/questions this.requestParams = { @@ -88,7 +92,10 @@ export class StackOverflowQuestionsCollatorFactory sort: 'activity', ...(options.requestParams ?? {}), }; - this.logger = options.logger.child({ documentType: this.type }); + + if (!options.requestParams?.site && this.baseUrl === DEFAULT_BASE_URL) { + this.requestParams.site = 'stackoverflow'; + } } static fromConfig( @@ -101,12 +108,12 @@ export class StackOverflowQuestionsCollatorFactory ); const teamName = config.getOptionalString('stackoverflow.teamName'); const baseUrl = - config.getOptionalString('stackoverflow.baseUrl') || - 'https://api.stackexchange.com/2.3'; - const maxPage = options.maxPage || 100; + config.getOptionalString('stackoverflow.baseUrl') || DEFAULT_BASE_URL; + const maxPage = options.maxPage || DEFAULT_MAX_PAGE; const requestParams = config .getOptionalConfig('stackoverflow.requestParams') ?.get(); + return new StackOverflowQuestionsCollatorFactory({ baseUrl, maxPage,