paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 48 Packages Projects Releases Wiki Activity

Externalize repository processing for BitbucketDiscoveryProcessor

Browse Source 
Signed-off-by: Mathias Åhsberg <mathias.ahsberg@resurs.se>
This commit is contained in:
Mathias Åhsberg
2021-04-12 15:04:22 +00:00
parent d269bf83c8
commit c2306f898d
9 changed files with 441 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts
+223 -127
View File
@@ -14,13 +14,15 @@
* limitations under the License.
*/
import { getVoidLogger } from '@backstage/backend-common';
import {
BitbucketDiscoveryProcessor,
readBitbucketOrg,
} from './BitbucketDiscoveryProcessor';
import { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor';
import { ConfigReader } from '@backstage/config';
import { LocationSpec } from '@backstage/catalog-model';
import { BitbucketClient, PagedResponse } from './bitbucket';
import {
BitbucketClient,
BitbucketRepositoryParser,
PagedResponse,
} from './bitbucket';
import { results } from './index';
function pagedResponse(values: any): PagedResponse<any> {
return {
@@ -30,11 +32,6 @@ function pagedResponse(values: any): PagedResponse<any> {
}
describe('BitbucketDiscoveryProcessor', () => {
const client: jest.Mocked<BitbucketClient> = {
listProjects: jest.fn(),
listRepositories: jest.fn(),
} as any;
afterEach(() => jest.resetAllMocks());
describe('reject unrelated entries', () => {
@@ -81,137 +78,236 @@ describe('BitbucketDiscoveryProcessor', () => {
});
describe('handles repositories', () => {
const processor = BitbucketDiscoveryProcessor.fromConfig(
new ConfigReader({
integrations: {
bitbucket: [{ host: 'bitbucket.mycompany.com', token: 'blob' }],
},
}),
{ logger: getVoidLogger() },
);
it('output all repositories', async () => {
const target =
'https://bitbucket.mycompany.com/projects/*/repos/*/catalog.yaml';
client.listProjects.mockResolvedValue(
pagedResponse([{ key: 'backstage' }, { key: 'demo' }]),
);
client.listRepositories.mockResolvedValueOnce(
pagedResponse([
{
slug: 'backstage',
links: {
self: [
{
href:
'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse',
},
],
},
},
]),
);
client.listRepositories.mockResolvedValueOnce(
pagedResponse([
{
slug: 'demo',
links: {
self: [
{
href:
'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse',
},
],
},
},
]),
);
const actual = await readBitbucketOrg(client, target);
expect(actual.scanned).toBe(2);
expect(actual.matches).toContainEqual({
type: 'url',
const location: LocationSpec = {
type: 'bitbucket-discovery',
target:
'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse/catalog.yaml',
'https://bitbucket.mycompany.com/projects/*/repos/*/catalog.yaml',
};
jest
.spyOn(BitbucketClient.prototype, 'listProjects')
.mockResolvedValue(
pagedResponse([{ key: 'backstage' }, { key: 'demo' }]),
);
jest
.spyOn(BitbucketClient.prototype, 'listRepositories')
.mockResolvedValueOnce(
pagedResponse([
{
slug: 'backstage',
links: {
self: [
{
href:
'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse',
},
],
},
},
]),
);
jest
.spyOn(BitbucketClient.prototype, 'listRepositories')
.mockResolvedValueOnce(
pagedResponse([
{
slug: 'demo',
links: {
self: [
{
href:
'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse',
},
],
},
},
]),
);
const emitter = jest.fn();
await processor.readLocation(location, false, emitter);
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse/catalog.yaml',
},
optional: true,
});
expect(actual.matches).toContainEqual({
type: 'url',
target:
'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse/catalog.yaml',
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse/catalog.yaml',
},
optional: true,
});
});
it('output repositories with wildcards', async () => {
const target =
'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-*/catalog.yaml';
client.listProjects.mockResolvedValue(
pagedResponse([{ key: 'backstage' }]),
);
client.listRepositories.mockResolvedValueOnce(
pagedResponse([
{ slug: 'backstage' },
{
slug: 'techdocs-cli',
links: {
self: [
{
href:
'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse',
},
],
},
},
{
slug: 'techdocs-container',
links: {
self: [
{
href:
'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse',
},
],
},
},
]),
);
const actual = await readBitbucketOrg(client, target);
expect(actual.scanned).toBe(3);
expect(actual.matches).toContainEqual({
type: 'url',
const location: LocationSpec = {
type: 'bitbucket-discovery',
target:
'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse/catalog.yaml',
'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-*/catalog.yaml',
};
jest
.spyOn(BitbucketClient.prototype, 'listProjects')
.mockResolvedValue(pagedResponse([{ key: 'backstage' }]));
jest
.spyOn(BitbucketClient.prototype, 'listRepositories')
.mockResolvedValueOnce(
pagedResponse([
{ slug: 'backstage' },
{
slug: 'techdocs-cli',
links: {
self: [
{
href:
'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse',
},
],
},
},
{
slug: 'techdocs-container',
links: {
self: [
{
href:
'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse',
},
],
},
},
]),
);
const emitter = jest.fn();
await processor.readLocation(location, false, emitter);
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse/catalog.yaml',
},
optional: true,
});
expect(actual.matches).toContainEqual({
type: 'url',
target:
'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse/catalog.yaml',
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse/catalog.yaml',
},
optional: true,
});
});
it('filter unrelated repositories', async () => {
const target =
'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml';
client.listProjects.mockResolvedValue(
pagedResponse([{ key: 'backstage' }]),
);
client.listRepositories.mockResolvedValue(
pagedResponse([
{ slug: 'abstest' },
{ slug: 'testxyz' },
{
slug: 'test',
links: {
self: [
{
href:
'https://bitbucket.mycompany.com/projects/backstage/repos/test',
},
],
},
},
]),
);
const actual = await readBitbucketOrg(client, target);
expect(actual.scanned).toBe(3);
expect(actual.matches).toContainEqual({
type: 'url',
const location: LocationSpec = {
type: 'bitbucket-discovery',
target:
'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml',
};
jest
.spyOn(BitbucketClient.prototype, 'listProjects')
.mockResolvedValue(pagedResponse([{ key: 'backstage' }]));
jest
.spyOn(BitbucketClient.prototype, 'listRepositories')
.mockResolvedValue(
pagedResponse([
{ slug: 'abstest' },
{ slug: 'testxyz' },
{
slug: 'test',
links: {
self: [
{
href:
'https://bitbucket.mycompany.com/projects/backstage/repos/test',
},
],
},
},
]),
);
const emitter = jest.fn();
await processor.readLocation(location, false, emitter);
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml',
},
optional: true,
});
});
});
describe('Custom repository parser', () => {
const customRepositoryParser: BitbucketRepositoryParser = async function* customRepositoryParser({}) {
yield results.location(
{
type: 'custom-location-type',
target: 'custom-target',
},
true,
);
};
const processor = BitbucketDiscoveryProcessor.fromConfig(
new ConfigReader({
integrations: {
bitbucket: [{ host: 'bitbucket.mycompany.com', token: 'blob' }],
},
}),
{ parser: customRepositoryParser, logger: getVoidLogger() },
);
it('use custom repository parser', async () => {
const location: LocationSpec = {
type: 'bitbucket-discovery',
target:
'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml',
};
jest
.spyOn(BitbucketClient.prototype, 'listProjects')
.mockResolvedValue(pagedResponse([{ key: 'backstage' }]));
jest
.spyOn(BitbucketClient.prototype, 'listRepositories')
.mockResolvedValue(pagedResponse([{ slug: 'test' }]));
const emitter = jest.fn();
await processor.readLocation(location, false, emitter);
expect(emitter).toHaveBeenCalledTimes(1);
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'custom-location-type',
target: 'custom-target',
},
optional: true,
});
});
});
plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts
+26 -19
View File
@@ -21,15 +21,24 @@ import {
ScmIntegrations,
} from '@backstage/integration';
import { LocationSpec } from '@backstage/catalog-model';
import { BitbucketClient, paginated } from './bitbucket';
import {
Repository,
BitbucketRepositoryParser,
BitbucketClient,
defaultRepositoryParser,
paginated,
} from './bitbucket';
import { CatalogProcessor, CatalogProcessorEmit } from './types';
import { results } from './index';
export class BitbucketDiscoveryProcessor implements CatalogProcessor {
private readonly integrations: ScmIntegrationRegistry;
private readonly parser: BitbucketRepositoryParser;
private readonly logger: Logger;
static fromConfig(config: Config, options: { logger: Logger }) {
static fromConfig(
config: Config,
options: { parser?: BitbucketRepositoryParser; logger: Logger },
) {
const integrations = ScmIntegrations.fromConfig(config);
return new BitbucketDiscoveryProcessor({
@@ -40,9 +49,11 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
constructor(options: {
integrations: ScmIntegrationRegistry;
parser?: BitbucketRepositoryParser;
logger: Logger;
}) {
this.integrations = options.integrations;
this.parser = options.parser || defaultRepositoryParser;
this.logger = options.logger;
}
@@ -73,18 +84,18 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
const startTimestamp = Date.now();
this.logger.info(`Reading Bitbucket repositories from ${location.target}`);
const { catalogPath } = parseUrl(location.target);
const result = await readBitbucketOrg(client, location.target);
for (const repository of result.matches) {
emit(
results.location(
repository,
// Not all locations may actually exist, since the user defined them as a wildcard pattern.
// Thus, we emit them as optional and let the downstream processor find them while not outputting
// an error if it couldn't.
true,
),
);
for await (const entity of this.parser({
client: client,
repository: repository,
path: catalogPath,
})) {
emit(entity);
}
}
const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
@@ -100,7 +111,7 @@ export async function readBitbucketOrg(
client: BitbucketClient,
target: string,
): Promise<Result> {
const { projectSearchPath, repoSearchPath, catalogPath } = parseUrl(target);
const { projectSearchPath, repoSearchPath } = parseUrl(target);
const projects = paginated(options => client.listProjects(options));
const result: Result = {
scanned: 0,
@@ -116,12 +127,8 @@ export async function readBitbucketOrg(
);
for await (const repository of repositories) {
result.scanned++;
if (repoSearchPath.test(repository.slug)) {
result.matches.push({
type: 'url',
target: `${repository.links.self[0].href}${catalogPath}`,
});
result.matches.push(repository);
}
}
}
@@ -152,5 +159,5 @@ function escapeRegExp(str: string): RegExp {
type Result = {
scanned: number;
matches: LocationSpec[];
matches: Repository[];
};
plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts
+55
View File
@@ -0,0 +1,55 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { defaultRepositoryParser } from './BitbucketRepositoryParser';
import { Project, Repository } from './types';
import { BitbucketClient } from './client';
import { results } from '../index';
describe('BitbucketRepositoryParser', () => {
describe('defaultRepositoryParser', () => {
it('emits location', async () => {
const browseUrl =
'https://bitbucket.mycompany.com/projects/project-key/repos/repo-slug/browse';
const path = '/catalog-info.yaml';
const expected = [
results.location(
{
type: 'url',
target: `${browseUrl}${path}`,
},
true,
),
];
const actual = await defaultRepositoryParser({
client: {} as BitbucketClient,
repository: {
project: {} as Project,
slug: 'repo-slug',
links: {
self: [{ href: browseUrl }],
},
} as Repository,
path: path,
});
let i = 0;
for await (const entity of actual) {
expect(entity).toStrictEqual(expected[i]);
i++;
}
});
});
});
plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts
+41
View File
@@ -0,0 +1,41 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Repository } from './types';
import { CatalogProcessorResult } from '../types';
import { results } from '../index';
import { BitbucketClient } from './client';
export type BitbucketRepositoryParser = (options: {
client: BitbucketClient;
repository: Repository;
path: string;
}) => AsyncIterable<CatalogProcessorResult>;
export const defaultRepositoryParser: BitbucketRepositoryParser = async function* defaultRepositoryParser({
repository,
path,
}) {
yield results.location(
{
type: 'url',
target: `${repository.links.self[0].href}${path}`,
},
// Not all locations may actually exist, since the user defined them as a wildcard pattern.
// Thus, we emit them as optional and let the downstream processor find them while not outputting
// an error if it couldn't.
true,
);
};
plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts
+9
View File
@@ -41,6 +41,15 @@ export class BitbucketClient {
);
}
async getRaw(
projectKey: string,
repo: string,
path: string,
): Promise<Response> {
const request = `${this.config.apiBaseUrl}/projects/${projectKey}/repos/${repo}/raw/${path}`;
return fetch(request, getBitbucketRequestOptions(this.config));
}
private async pagedRequest(
endpoint: string,
options?: ListOptions,
plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts
+3
View File
@@ -15,3 +15,6 @@
*/
export { BitbucketClient, paginated } from './client';
export type { PagedResponse } from './client';
export * from './types';
export type { BitbucketRepositoryParser } from './BitbucketRepositoryParser';
export { defaultRepositoryParser } from './BitbucketRepositoryParser';
plugins/catalog-backend/src/ingestion/processors/bitbucket/types.ts
+28
View File
@@ -0,0 +1,28 @@
/*
* Copyright 2021 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export type Project = {
key: string;
};
export type Repository = {
project: Project;
slug: string;
links: Record<string, Link[]>;
};
export type Link = {
href: string;
};