paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 49 Packages Projects Releases Wiki Activity

Refactor the Bitbucket Discovery processor

Browse Source 
Signed-off-by: Mathias Åhsberg <mathias.ahsberg@resurs.se>
This commit is contained in:
Mathias Åhsberg
2021-03-30 10:38:32 +00:00
parent 2d2b955ed1
commit 13abbc15da
6 changed files with 88 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts
+8 -7
View File
@@ -120,12 +120,13 @@ describe('BitbucketDiscoveryProcessor', () => {
);
const actual = await readBitbucketOrg(client, target);
expect(actual).toContainEqual({
expect(actual.scanned).toBe(2);
expect(actual.matches).toContainEqual({
type: 'url',
target:
'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse/catalog.yaml',
});
expect(actual).toContainEqual({
expect(actual.matches).toContainEqual({
type: 'url',
target:
'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse/catalog.yaml',
@@ -168,13 +169,13 @@ describe('BitbucketDiscoveryProcessor', () => {
);
const actual = await readBitbucketOrg(client, target);
expect(actual).toContainEqual({
expect(actual.scanned).toBe(3);
expect(actual.matches).toContainEqual({
type: 'url',
target:
'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse/catalog.yaml',
});
expect(actual).toContainEqual({
expect(actual.matches).toContainEqual({
type: 'url',
target:
'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse/catalog.yaml',
@@ -206,8 +207,8 @@ describe('BitbucketDiscoveryProcessor', () => {
);
const actual = await readBitbucketOrg(client, target);
expect(actual).toContainEqual({
expect(actual.scanned).toBe(3);
expect(actual.matches).toContainEqual({
type: 'url',
target:
'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml',
plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts
+40 -30
View File
@@ -16,14 +16,17 @@
import { Logger } from 'winston';
import { Config } from '@backstage/config';
import { ScmIntegrations } from '@backstage/integration';
import {
ScmIntegrationRegistry,
ScmIntegrations,
} from '@backstage/integration';
import { LocationSpec } from '@backstage/catalog-model';
import { BitbucketClient, pageIterator } from './bitbucket';
import { BitbucketClient, paginated } from './bitbucket';
import { CatalogProcessor, CatalogProcessorEmit } from './types';
import { results } from './index';
export class BitbucketDiscoveryProcessor implements CatalogProcessor {
private readonly integrations: ScmIntegrations;
private readonly integrations: ScmIntegrationRegistry;
private readonly logger: Logger;
static fromConfig(config: Config, options: { logger: Logger }) {
@@ -35,7 +38,10 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
});
}
constructor(options: { integrations: ScmIntegrations; logger: Logger }) {
constructor(options: {
integrations: ScmIntegrationRegistry;
logger: Logger;
}) {
this.integrations = options.integrations;
this.logger = options.logger;
}
@@ -67,9 +73,9 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
const startTimestamp = Date.now();
this.logger.info(`Reading Bitbucket repositories from ${location.target}`);
const repositories = await readBitbucketOrg(client, location.target);
const result = await readBitbucketOrg(client, location.target);
for (const repository of repositories) {
for (const repository of result.matches) {
emit(
results.location(
repository,
@@ -82,8 +88,8 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
}
const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
this.logger.info(
`Read ${repositories.length} Bitbucket repositories in ${duration} seconds`,
this.logger.debug(
`Read ${result.scanned} Bitbucket repositories (${result.matches.length} matching the pattern) in ${duration} seconds`,
);
return true;
@@ -93,30 +99,29 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
export async function readBitbucketOrg(
client: BitbucketClient,
target: string,
): Promise<LocationSpec[]> {
): Promise<Result> {
const { projectSearchPath, repoSearchPath, catalogPath } = parseUrl(target);
const projectIterator = pageIterator(options => client.listProjects(options));
let result: LocationSpec[] = [];
const projects = paginated(options => client.listProjects(options));
const result: Result = {
scanned: 0,
matches: [],
};
for await (const page of projectIterator) {
for (const project of page.values) {
if (!projectSearchPath.test(project.key)) {
continue;
}
const repoIterator = pageIterator(options =>
client.listRepositories(project.key, options),
);
for await (const repoPage of repoIterator) {
result = result.concat(
repoPage.values
.filter(v => repoSearchPath.test(v.slug))
.map(repo => {
return {
type: 'url',
target: `${repo.links.self[0].href}${catalogPath}`,
};
}),
);
for await (const project of projects) {
if (!projectSearchPath.test(project.key)) {
continue;
}
const repositories = paginated(options =>
client.listRepositories(project.key, options),
);
for await (const repository of repositories) {
result.scanned++;
if (repoSearchPath.test(repository.slug)) {
result.matches.push({
type: 'url',
target: `${repository.links.self[0].href}${catalogPath}`,
});
}
}
}
@@ -144,3 +149,8 @@ function parseUrl(
function escapeRegExp(str: string): RegExp {
return new RegExp(`^${str.replace(/\*/g, '.*')}$`);
}
type Result = {
scanned: number;
matches: LocationSpec[];
};
plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts
+18 -44
View File
@@ -46,14 +46,12 @@ export class BitbucketClient {
options?: ListOptions,
): Promise<PagedResponse<any>> {
const request = new URL(endpoint);
if (options) {
(Object.keys(options) as Array<keyof typeof options>).forEach(key => {
const value: any = options[key] as any;
if (value) {
request.searchParams.append(key, value);
}
});
for (const key in options) {
if (options[key]) {
request.searchParams.append(key, options[key]!.toString());
}
}
const response = await fetch(
request.toString(),
getBitbucketRequestOptions(this.config),
@@ -72,6 +70,7 @@ export class BitbucketClient {
}
export type ListOptions = {
[key: string]: number | undefined;
limit?: number | undefined;
start?: number | undefined;
};
@@ -85,42 +84,17 @@ export type PagedResponse<T> = {
nextPageStart: number;
};
export function pageIterator(
pagedRequest: (options: ListOptions) => Promise<PagedResponse<any>>,
export async function* paginated(
request: (options: ListOptions) => Promise<PagedResponse<any>>,
options?: ListOptions,
): AsyncIterable<PagedResponse<any>> {
return {
[Symbol.asyncIterator]: () => {
const opts = options || { start: 0 };
let finished = false;
return {
async next() {
if (!finished) {
try {
const response = await pagedRequest(opts);
finished = response.isLastPage;
opts.start = response.nextPageStart;
return Promise.resolve({
value: response,
done: false,
});
} catch (error) {
return Promise.reject({
value: undefined,
done: true,
error: error,
});
}
} else {
opts.start = 0;
finished = false;
return Promise.resolve({
value: undefined,
done: true,
});
}
},
};
},
};
) {
const opts = options || { start: 0 };
let res;
do {
res = await request(opts);
opts.start = res.nextPageStart;
for (const item of res.values) {
yield item;
}
} while (!res.isLastPage);
}
plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts
+1 -1
View File
@@ -13,5 +13,5 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { BitbucketClient, pageIterator } from './client';
export { BitbucketClient, paginated } from './client';
export type { PagedResponse } from './client';