paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 26 Packages Projects Releases Wiki Activity

feat(stackoverflow-backend): extract search stack overflow module

Browse Source 
Signed-off-by: Camila Belo <camilaibs@gmail.com>
This commit is contained in:
Camila Belo
2023-10-09 16:19:16 +02:00
parent a8a4911713
commit 27e22c6a21
17 changed files with 377 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/stack-overflow-backend/alpha-api-report.md
-11
View File
@@ -1,11 +0,0 @@
## API Report File for "@backstage/plugin-stack-overflow-backend"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
```ts
import { BackendFeature } from '@backstage/backend-plugin-api';
// @alpha
const _default: () => BackendFeature;
export default _default;
```
plugins/stack-overflow-backend/api-report.md
+13 -46
View File
@@ -3,54 +3,21 @@
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
```ts
/// <reference types="node" />
import { StackOverflowDocument as StackOverflowDocument_2 } from '@backstage/plugin-search-backend-module-stack-overflow';
import { StackOverflowQuestionsCollatorFactory as StackOverflowQuestionsCollatorFactory_2 } from '@backstage/plugin-search-backend-module-stack-overflow';
import { StackOverflowQuestionsRequestParams as StackOverflowQuestionsRequestParams_2 } from '@backstage/plugin-search-backend-module-stack-overflow';
import { Config } from '@backstage/config';
import { DocumentCollatorFactory } from '@backstage/plugin-search-common';
import { IndexableDocument } from '@backstage/plugin-search-common';
import { Logger } from 'winston';
import { Readable } from 'stream';
// @public @deprecated (undocumented)
export type StackOverflowDocument = StackOverflowDocument_2;
// @public
export interface StackOverflowDocument extends IndexableDocument {
// (undocumented)
answers: number;
// (undocumented)
tags: string[];
}
// @public @deprecated (undocumented)
export const StackOverflowQuestionsCollatorFactory: typeof StackOverflowQuestionsCollatorFactory_2;
// @public
export class StackOverflowQuestionsCollatorFactory
implements DocumentCollatorFactory
{
// (undocumented)
execute(): AsyncGenerator<StackOverflowDocument>;
// (undocumented)
static fromConfig(
config: Config,
options: StackOverflowQuestionsCollatorFactoryOptions,
): StackOverflowQuestionsCollatorFactory;
// (undocumented)
getCollator(): Promise<Readable>;
// (undocumented)
protected requestParams: StackOverflowQuestionsRequestParams;
// (undocumented)
readonly type: string;
}
// @public @deprecated (undocumented)
export type StackOverflowQuestionsCollatorFactoryOptions =
StackOverflowQuestionsCollatorFactory_2;
// @public
export type StackOverflowQuestionsCollatorFactoryOptions = {
baseUrl?: string;
maxPage?: number;
apiKey?: string;
apiAccessToken?: string;
teamName?: string;
requestParams?: StackOverflowQuestionsRequestParams;
logger: Logger;
};
// @public
export type StackOverflowQuestionsRequestParams = {
[key: string]: string | string[] | number;
};
// @public @deprecated (undocumented)
export type StackOverflowQuestionsRequestParams =
StackOverflowQuestionsRequestParams_2;
```
plugins/stack-overflow-backend/package.json
+5 -19
View File
@@ -5,22 +5,9 @@
"types": "src/index.ts",
"license": "Apache-2.0",
"publishConfig": {
"access": "public"
},
"exports": {
".": "./src/index.ts",
"./alpha": "./src/alpha.ts",
"./package.json": "./package.json"
},
"typesVersions": {
"*": {
"alpha": [
"src/alpha.ts"
],
"package.json": [
"package.json"
]
}
"access": "public",
"main": "dist/index.cjs.js",
"types": "dist/index.d.ts"
},
"backstage": {
"role": "backend-plugin"
@@ -46,10 +33,8 @@
},
"dependencies": {
"@backstage/backend-common": "workspace:^",
"@backstage/backend-plugin-api": "workspace:^",
"@backstage/backend-tasks": "workspace:^",
"@backstage/config": "workspace:^",
"@backstage/plugin-search-backend-node": "workspace:^",
"@backstage/plugin-search-backend-module-stack-overflow": "workspace:^",
"@backstage/plugin-search-common": "workspace:^",
"node-fetch": "^2.6.7",
"qs": "^6.9.4",
@@ -58,6 +43,7 @@
"devDependencies": {
"@backstage/backend-test-utils": "workspace:^",
"@backstage/cli": "workspace:^",
"@backstage/plugin-search-backend-node": "workspace:^",
"msw": "^1.0.0"
},
"files": [
plugins/stack-overflow-backend/src/alpha.ts
-70
View File
@@ -1,70 +0,0 @@
/*
* Copyright 2023 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { loggerToWinstonLogger } from '@backstage/backend-common';
import { readTaskScheduleDefinitionFromConfig } from '@backstage/backend-tasks';
import {
coreServices,
createBackendModule,
} from '@backstage/backend-plugin-api';
import { searchIndexRegistryExtensionPoint } from '@backstage/plugin-search-backend-node/alpha';
import { StackOverflowQuestionsCollatorFactory } from './search';
/**
* @packageDocumentation
* A module for the search backend that exports Stack Overflow modules.
*/
/**
* Search backend module for the Stack Overflow index.
*
* @alpha
*/
export default createBackendModule({
moduleId: 'stackoverflowCollator',
pluginId: 'search',
register(env) {
env.registerInit({
deps: {
config: coreServices.rootConfig,
logger: coreServices.logger,
discovery: coreServices.discovery,
scheduler: coreServices.scheduler,
indexRegistry: searchIndexRegistryExtensionPoint,
},
async init({ config, logger, scheduler, indexRegistry }) {
const defaultSchedule = {
frequency: { minutes: 10 },
timeout: { minutes: 15 },
initialDelay: { seconds: 3 },
};
const schedule = config.has('search.collators.stackoverflow.schedule')
? readTaskScheduleDefinitionFromConfig(
config.getConfig('search.collators.stackoverflow.schedule'),
)
: defaultSchedule;
indexRegistry.addCollator({
schedule: scheduler.createScheduledTaskRunner(schedule),
factory: StackOverflowQuestionsCollatorFactory.fromConfig(config, {
logger: loggerToWinstonLogger(logger),
}),
});
},
});
},
});
plugins/stack-overflow-backend/src/index.ts
+37 -1
View File
@@ -20,4 +20,40 @@
* @packageDocumentation
*/
export * from './search';
import {
StackOverflowDocument as _StackOverflowDocument,
StackOverflowQuestionsRequestParams as _StackOverflowQuestionsRequestParams,
StackOverflowQuestionsCollatorFactory as _StackOverflowQuestionsCollatorFactory,
StackOverflowQuestionsCollatorFactoryOptions as _StackOverflowQuestionsCollatorFactoryOptions,
} from '@backstage/plugin-search-backend-module-stack-overflow';
/**
* @public
* @deprecated
* Import from `@backstage/plugin-search-backend-module-stack-overflow` instead.
*/
export type StackOverflowDocument = _StackOverflowDocument;
/**
* @public
* @deprecated
* Import from `@backstage/plugin-search-backend-module-stack-overflow` instead.
*/
export type StackOverflowQuestionsRequestParams =
_StackOverflowQuestionsRequestParams;
/**
* @public
* @deprecated
* Import from `@backstage/plugin-search-backend-module-stack-overflow` instead.
*/
export type StackOverflowQuestionsCollatorFactoryOptions =
_StackOverflowQuestionsCollatorFactory;
/**
* @public
* @deprecated
* Import from `@backstage/plugin-search-backend-module-stack-overflow` instead.
*/
export const StackOverflowQuestionsCollatorFactory =
_StackOverflowQuestionsCollatorFactory;
plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.test.ts
-187
View File
@@ -1,187 +0,0 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { getVoidLogger } from '@backstage/backend-common';
import {
StackOverflowQuestionsCollatorFactory,
StackOverflowQuestionsCollatorFactoryOptions,
} from './StackOverflowQuestionsCollatorFactory';
import { setupRequestMockHandlers } from '@backstage/backend-test-utils';
import { TestPipeline } from '@backstage/plugin-search-backend-node';
import { ConfigReader } from '@backstage/config';
import { Readable } from 'stream';
import { setupServer } from 'msw/node';
import { rest } from 'msw';
const logger = getVoidLogger();
const mockQuestion = {
items: [
{
tags: ['backstage'],
owner: {
display_name: 'The Riddler',
},
answer_count: 1,
link: 'https://stack.overflow.local/questions/2911',
title: 'This is the first question',
},
],
has_more: false,
};
const mockOverrideQuestion = {
items: [
{
tags: ['backstage'],
owner: {
display_name: 'The Riddler',
},
answer_count: 1,
link: 'https://stack.overflow.local/questions/1',
title: 'This is the first question',
},
{
tags: ['backstage'],
owner: {
display_name: 'The Riddler',
},
answer_count: 1,
link: 'https://stack.overflow.local/questions/2',
title: 'this is another question',
},
],
has_more: false,
};
describe('StackOverflowQuestionsCollatorFactory', () => {
const config = new ConfigReader({
stackoverflow: {
baseUrl: 'http://stack.overflow.local',
},
});
const defaultOptions: StackOverflowQuestionsCollatorFactoryOptions = {
logger,
requestParams: {
tagged: ['developer-portal'],
pagesize: 100,
order: 'desc',
sort: 'activity',
},
};
it('has expected type', () => {
const factory = StackOverflowQuestionsCollatorFactory.fromConfig(
config,
defaultOptions,
);
expect(factory.type).toBe('stack-overflow');
});
describe('getCollator', () => {
const worker = setupServer();
setupRequestMockHandlers(worker);
it('returns a readable stream', async () => {
const factory = StackOverflowQuestionsCollatorFactory.fromConfig(
config,
defaultOptions,
);
const collator = await factory.getCollator();
expect(collator).toBeInstanceOf(Readable);
});
it('fetches from the configured endpoint', async () => {
worker.use(
rest.get('http://stack.overflow.local/questions', (_, res, ctx) =>
res(ctx.status(200), ctx.json(mockQuestion)),
),
);
const factory = StackOverflowQuestionsCollatorFactory.fromConfig(
config,
defaultOptions,
);
const collator = await factory.getCollator();
const pipeline = TestPipeline.fromCollator(collator);
const { documents } = await pipeline.execute();
expect(documents).toHaveLength(mockQuestion.items.length);
});
it('fetches from the overridden endpoint', async () => {
worker.use(
rest.get('http://stack.overflow.override/questions', (_, res, ctx) =>
res(ctx.status(200), ctx.json(mockOverrideQuestion)),
),
);
const factory = StackOverflowQuestionsCollatorFactory.fromConfig(config, {
logger,
baseUrl: 'http://stack.overflow.override',
requestParams: defaultOptions.requestParams,
});
const collator = await factory.getCollator();
const pipeline = TestPipeline.fromCollator(collator);
const { documents } = await pipeline.execute();
expect(documents).toHaveLength(mockOverrideQuestion.items.length);
});
it('uses API key when provided', async () => {
worker.use(
rest.get('http://stack.overflow.override/questions', (req, res, ctx) =>
req.url.searchParams.get('key') === 'abcdefg'
? res(ctx.status(200), ctx.json(mockOverrideQuestion))
: res(ctx.status(401), ctx.json({})),
),
);
const factory = StackOverflowQuestionsCollatorFactory.fromConfig(config, {
logger,
baseUrl: 'http://stack.overflow.override',
apiKey: 'abcdefg',
requestParams: defaultOptions.requestParams,
});
const collator = await factory.getCollator();
const pipeline = TestPipeline.fromCollator(collator);
const { documents } = await pipeline.execute();
expect(documents).toHaveLength(mockOverrideQuestion.items.length);
});
it('uses teamName when provided', async () => {
worker.use(
rest.get('http://stack.overflow.override/questions', (req, res, ctx) =>
req.url.searchParams.get('team') === 'abcdefg'
? res(ctx.status(200), ctx.json(mockOverrideQuestion))
: res(ctx.status(401), ctx.json({})),
),
);
const factory = StackOverflowQuestionsCollatorFactory.fromConfig(config, {
logger,
baseUrl: 'http://stack.overflow.override',
teamName: 'abcdefg',
requestParams: defaultOptions.requestParams,
});
const collator = await factory.getCollator();
const pipeline = TestPipeline.fromCollator(collator);
const { documents } = await pipeline.execute();
expect(documents).toHaveLength(mockOverrideQuestion.items.length);
});
});
});
plugins/stack-overflow-backend/src/search/StackOverflowQuestionsCollatorFactory.ts
-202
View File
@@ -1,202 +0,0 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
IndexableDocument,
DocumentCollatorFactory,
} from '@backstage/plugin-search-common';
import { Config } from '@backstage/config';
import { Readable } from 'stream';
import fetch from 'node-fetch';
import qs from 'qs';
import { Logger } from 'winston';
/**
* Extended IndexableDocument with stack overflow specific properties
*
* @public
*/
export interface StackOverflowDocument extends IndexableDocument {
answers: number;
tags: string[];
}
/**
* Type representing the request parameters accepted by the {@link StackOverflowQuestionsCollatorFactory}
*
* @public
*/
export type StackOverflowQuestionsRequestParams = {
[key: string]: string | string[] | number;
};
/**
* Options for {@link StackOverflowQuestionsCollatorFactory}
*
* @public
*/
export type StackOverflowQuestionsCollatorFactoryOptions = {
baseUrl?: string;
maxPage?: number;
apiKey?: string;
apiAccessToken?: string;
teamName?: string;
requestParams?: StackOverflowQuestionsRequestParams;
logger: Logger;
};
/**
* Search collator responsible for collecting stack overflow questions to index.
*
* @public
*/
export class StackOverflowQuestionsCollatorFactory
implements DocumentCollatorFactory
{
protected requestParams: StackOverflowQuestionsRequestParams;
private readonly baseUrl: string | undefined;
private readonly apiKey: string | undefined;
private readonly apiAccessToken: string | undefined;
private readonly teamName: string | undefined;
private readonly maxPage: number | undefined;
private readonly logger: Logger;
public readonly type: string = 'stack-overflow';
private constructor(options: StackOverflowQuestionsCollatorFactoryOptions) {
this.baseUrl = options.baseUrl;
this.apiKey = options.apiKey;
this.apiAccessToken = options.apiAccessToken;
this.teamName = options.teamName;
this.maxPage = options.maxPage;
// Sets the same default request parameters as the official API documentation
// See https://api.stackexchange.com/docs/questions
this.requestParams = options.requestParams ?? {
order: 'desc',
sort: 'activity',
site: 'stackoverflow',
...(options.requestParams ?? {}),
};
this.logger = options.logger.child({ documentType: this.type });
}
static fromConfig(
config: Config,
options: StackOverflowQuestionsCollatorFactoryOptions,
) {
const apiKey = config.getOptionalString('stackoverflow.apiKey');
const apiAccessToken = config.getOptionalString(
'stackoverflow.apiAccessToken',
);
const teamName = config.getOptionalString('stackoverflow.teamName');
const baseUrl =
config.getOptionalString('stackoverflow.baseUrl') ||
'https://api.stackexchange.com/2.3';
const maxPage = options.maxPage || 100;
const requestParams = config
.getOptionalConfig('stackoverflow.requestParams')
?.get<StackOverflowQuestionsRequestParams>();
return new StackOverflowQuestionsCollatorFactory({
baseUrl,
maxPage,
apiKey,
apiAccessToken,
teamName,
requestParams,
...options,
});
}
async getCollator() {
return Readable.from(this.execute());
}
async *execute(): AsyncGenerator<StackOverflowDocument> {
if (!this.baseUrl) {
this.logger.debug(
`No stackoverflow.baseUrl configured in your app-config.yaml`,
);
}
if (this.apiKey && this.teamName) {
this.logger.debug(
'Both stackoverflow.apiKey and stackoverflow.teamName configured in your app-config.yaml, apiKey must be removed before teamName will be used',
);
}
try {
if (Object.keys(this.requestParams).indexOf('key') >= 0) {
this.logger.warn(
'The API Key should be passed as a separate param to bypass encoding',
);
delete this.requestParams.key;
}
} catch (e) {
this.logger.error(`Caught ${e}`);
}
const params = qs.stringify(this.requestParams, {
arrayFormat: 'comma',
addQueryPrefix: true,
});
const apiKeyParam = this.apiKey
? `${params ? '&' : '?'}key=${this.apiKey}`
: '';
const teamParam = this.teamName
? `${params ? '&' : '?'}team=${this.teamName}`
: '';
// PAT change requires team name as a parameter
const requestUrl = this.apiKey
? `${this.baseUrl}/questions${params}${apiKeyParam}`
: `${this.baseUrl}/questions${params}${teamParam}`;
let hasMorePages = true;
let page = 1;
while (hasMorePages) {
if (page === this.maxPage) {
this.logger.warn(
`Over ${this.maxPage} requests to the Stack Overflow API have been made, which may not have been intended. Either specify requestParams that limit the questions returned, or configure a higher maxPage if necessary.`,
);
break;
}
const res = await fetch(
`${requestUrl}&page=${page}`,
this.apiAccessToken
? {
headers: {
'X-API-Access-Token': this.apiAccessToken,
},
}
: undefined,
);
const data = await res.json();
for (const question of data.items ?? []) {
yield {
title: question.title,
location: question.link,
text: question.owner.display_name,
tags: question.tags,
answers: question.answer_count,
};
}
hasMorePages = data.has_more;
page = page + 1;
}
}
}
plugins/stack-overflow-backend/src/search/index.ts
-17
View File
@@ -1,17 +0,0 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export * from './StackOverflowQuestionsCollatorFactory';