paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 21 Packages Projects Releases Wiki Activity

handle site in request parameter

Browse Source 
add site with value "stackoverflow" to requestParams when using the default base url: https://api.stackexchange.com/2.3

Signed-off-by: Raffaello De Pieri <raffaello.depieri@onefootball.com>
This commit is contained in:
Raffaello De Pieri
2024-08-01 13:41:51 +01:00
parent 5e0c969b19
commit 48d3eea823
2 changed files with 131 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.test.ts
+119 -30
View File
@@ -29,6 +29,8 @@ import { rest, RestRequest } from 'msw';
const logger = mockServices.logger.mock();
const BASE_URL = 'https://api.stackexchange.com/2.3';
const mockQuestion = {
items: [
{
@@ -110,6 +112,98 @@ describe('StackOverflowQuestionsCollatorFactory using custom request params', ()
expect(factory.type).toBe('stack-overflow');
});
describe('Manage site query parameter', () => {
const worker = setupServer();
registerMswTestHooks(worker);
it('uses site query parameter when provided and baseUrl is not default', async () => {
let request;
worker.use(
rest.get('http://stack.overflow.local/questions', (req, res, ctx) => {
request = req;
return res(ctx.status(200), ctx.json(mockQuestion));
}),
);
const factory = StackOverflowQuestionsCollatorFactory.fromConfig(config, {
logger,
baseUrl: 'http://stack.overflow.local',
requestParams: {
site: 'stackoverflow',
},
});
const collator = await factory.getCollator();
const pipeline = TestPipeline.fromCollator(collator);
const { documents } = await pipeline.execute();
const expectedSearch = {
order: 'desc',
site: 'stackoverflow',
sort: 'activity',
page: '1',
};
testSearchQuery(request, expectedSearch);
expect(documents).toHaveLength(mockQuestion.items.length);
});
it('uses site query parameter when provided and baseUrl is default', async () => {
let request;
worker.use(
rest.get(`${BASE_URL}/questions`, (req, res, ctx) => {
request = req;
return res(ctx.status(200), ctx.json(mockQuestion));
}),
);
const factory = StackOverflowQuestionsCollatorFactory.fromConfig(config, {
logger,
baseUrl: BASE_URL,
requestParams: {
site: 'foo_bar_baz',
},
});
const collator = await factory.getCollator();
const pipeline = TestPipeline.fromCollator(collator);
const { documents } = await pipeline.execute();
const expectedSearch = {
order: 'desc',
site: 'foo_bar_baz',
sort: 'activity',
page: '1',
};
testSearchQuery(request, expectedSearch);
expect(documents).toHaveLength(mockQuestion.items.length);
});
it('uses default when site is not provided and baseUrl is default', async () => {
let request;
worker.use(
rest.get(`${BASE_URL}/questions`, (req, res, ctx) => {
request = req;
return res(ctx.status(200), ctx.json(mockQuestion));
}),
);
const factory = StackOverflowQuestionsCollatorFactory.fromConfig(config, {
logger,
baseUrl: BASE_URL,
});
const collator = await factory.getCollator();
const pipeline = TestPipeline.fromCollator(collator);
const { documents } = await pipeline.execute();
const expectedSearch = {
order: 'desc',
site: 'stackoverflow',
sort: 'activity',
page: '1',
};
testSearchQuery(request, expectedSearch);
expect(documents).toHaveLength(mockQuestion.items.length);
});
});
describe('getCollator', () => {
const worker = setupServer();
registerMswTestHooks(worker);
@@ -260,7 +354,7 @@ describe('StackOverflowQuestionsCollatorFactory using custom request params', ()
describe('StackOverflowQuestionsCollatorFactory using default request params', () => {
const config = new ConfigReader({
stackoverflow: {
baseUrl: 'http://stack.overflow.local',
baseUrl: BASE_URL,
},
});
@@ -293,7 +387,7 @@ describe('StackOverflowQuestionsCollatorFactory using default request params', (
it('fetches from the configured endpoint', async () => {
let request;
worker.use(
rest.get('http://stack.overflow.local/questions', (req, res, ctx) => {
rest.get(`${BASE_URL}/questions`, (req, res, ctx) => {
request = req;
return res(ctx.status(200), ctx.json(mockQuestion));
@@ -309,6 +403,7 @@ describe('StackOverflowQuestionsCollatorFactory using default request params', (
const expectedSearch = {
order: 'desc',
site: 'stackoverflow',
sort: 'activity',
page: '1',
};
@@ -319,17 +414,14 @@ describe('StackOverflowQuestionsCollatorFactory using default request params', (
it('fetches from the overridden endpoint', async () => {
let request;
worker.use(
rest.get(
'http://stack.overflow.override/questions',
(req, res, ctx) => {
request = req;
return res(ctx.status(200), ctx.json(mockOverrideQuestion));
},
),
rest.get(`${BASE_URL}/questions`, (req, res, ctx) => {
request = req;
return res(ctx.status(200), ctx.json(mockOverrideQuestion));
}),
);
const factory = StackOverflowQuestionsCollatorFactory.fromConfig(config, {
logger,
baseUrl: 'http://stack.overflow.override',
baseUrl: BASE_URL,
requestParams: optionsWithCustomRequestParams.requestParams,
});
const collator = await factory.getCollator();
@@ -339,6 +431,7 @@ describe('StackOverflowQuestionsCollatorFactory using default request params', (
const expectedSearch = {
order: 'desc',
site: 'stackoverflow',
sort: 'activity',
page: '1',
};
@@ -349,19 +442,16 @@ describe('StackOverflowQuestionsCollatorFactory using default request params', (
it('uses API key when provided', async () => {
let request;
worker.use(
rest.get(
'http://stack.overflow.override/questions',
(req, res, ctx) => {
request = req;
return req.url.searchParams.get('key') === 'abcdefg'
? res(ctx.status(200), ctx.json(mockOverrideQuestion))
: res(ctx.status(401), ctx.json({}));
},
),
rest.get(`${BASE_URL}/questions`, (req, res, ctx) => {
request = req;
return req.url.searchParams.get('key') === 'abcdefg'
? res(ctx.status(200), ctx.json(mockOverrideQuestion))
: res(ctx.status(401), ctx.json({}));
}),
);
const factory = StackOverflowQuestionsCollatorFactory.fromConfig(config, {
logger,
baseUrl: 'http://stack.overflow.override',
baseUrl: BASE_URL,
apiKey: 'abcdefg',
requestParams: optionsWithCustomRequestParams.requestParams,
});
@@ -373,6 +463,7 @@ describe('StackOverflowQuestionsCollatorFactory using default request params', (
const expectedSearch = {
key: 'abcdefg',
order: 'desc',
site: 'stackoverflow',
sort: 'activity',
page: '1',
};
@@ -383,19 +474,16 @@ describe('StackOverflowQuestionsCollatorFactory using default request params', (
it('uses teamName when provided', async () => {
let request;
worker.use(
rest.get(
'http://stack.overflow.override/questions',
(req, res, ctx) => {
request = req;
return req.url.searchParams.get('team') === 'abcdefg'
? res(ctx.status(200), ctx.json(mockOverrideQuestion))
: res(ctx.status(401), ctx.json({}));
},
),
rest.get(`${BASE_URL}/questions`, (req, res, ctx) => {
request = req;
return req.url.searchParams.get('team') === 'abcdefg'
? res(ctx.status(200), ctx.json(mockOverrideQuestion))
: res(ctx.status(401), ctx.json({}));
}),
);
const factory = StackOverflowQuestionsCollatorFactory.fromConfig(config, {
logger,
baseUrl: 'http://stack.overflow.override',
baseUrl: BASE_URL,
teamName: 'abcdefg',
requestParams: optionsWithCustomRequestParams.requestParams,
});
@@ -407,6 +495,7 @@ describe('StackOverflowQuestionsCollatorFactory using default request params', (
const expectedSearch = {
team: 'abcdefg',
order: 'desc',
site: 'stackoverflow',
sort: 'activity',
page: '1',
};
plugins/search-backend-module-stack-overflow-collator/src/collators/StackOverflowQuestionsCollatorFactory.ts
+12 -5
View File
@@ -40,7 +40,7 @@ export interface StackOverflowDocument extends IndexableDocument {
* @public
*/
export type StackOverflowQuestionsRequestParams = {
[key: string]: string | string[] | number;
[key: string]: string | string[] | number | null;
};
/**
@@ -58,6 +58,8 @@ export type StackOverflowQuestionsCollatorFactoryOptions = {
logger: LoggerService;
};
const DEFAULT_BASE_URL = 'https://api.stackexchange.com/2.3';
const DEFAULT_MAX_PAGE = 100;
/**
* Search collator responsible for collecting stack overflow questions to index.
*
@@ -81,6 +83,8 @@ export class StackOverflowQuestionsCollatorFactory
this.apiAccessToken = options.apiAccessToken;
this.teamName = options.teamName;
this.maxPage = options.maxPage;
this.logger = options.logger.child({ documentType: this.type });
// Sets the same default request parameters as the official API documentation
// See https://api.stackexchange.com/docs/questions
this.requestParams = {
@@ -88,7 +92,10 @@ export class StackOverflowQuestionsCollatorFactory
sort: 'activity',
...(options.requestParams ?? {}),
};
this.logger = options.logger.child({ documentType: this.type });
if (!options.requestParams?.site && this.baseUrl === DEFAULT_BASE_URL) {
this.requestParams.site = 'stackoverflow';
}
}
static fromConfig(
@@ -101,12 +108,12 @@ export class StackOverflowQuestionsCollatorFactory
);
const teamName = config.getOptionalString('stackoverflow.teamName');
const baseUrl =
config.getOptionalString('stackoverflow.baseUrl') ||
'https://api.stackexchange.com/2.3';
const maxPage = options.maxPage || 100;
config.getOptionalString('stackoverflow.baseUrl') || DEFAULT_BASE_URL;
const maxPage = options.maxPage || DEFAULT_MAX_PAGE;
const requestParams = config
.getOptionalConfig('stackoverflow.requestParams')
?.get<StackOverflowQuestionsRequestParams>();
return new StackOverflowQuestionsCollatorFactory({
baseUrl,
maxPage,