Add Bitbucket Server discovery processor
Signed-off-by: Mathias Åhsberg <mathias.ahsberg@resurs.se>
This commit is contained in:
Mathias Åhsberg
@@ -0,0 +1,217 @@
|
||||
/*
|
||||
* Copyright 2021 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { getVoidLogger } from '@backstage/backend-common';
|
||||
import {
|
||||
BitbucketDiscoveryProcessor,
|
||||
readBitbucketOrg,
|
||||
} from './BitbucketDiscoveryProcessor';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { LocationSpec } from '@backstage/catalog-model';
|
||||
import { BitbucketClient, PagedResponse } from './bitbucket';
|
||||
|
||||
function pagedResponse(values: any): PagedResponse<any> {
|
||||
return {
|
||||
values: values,
|
||||
isLastPage: true,
|
||||
} as PagedResponse<any>;
|
||||
}
|
||||
|
||||
describe('BitbucketDiscoveryProcessor', () => {
|
||||
const client: jest.Mocked<BitbucketClient> = {
|
||||
listProjects: jest.fn(),
|
||||
listRepositories: jest.fn(),
|
||||
} as any;
|
||||
|
||||
afterEach(() => jest.resetAllMocks());
|
||||
|
||||
describe('reject unrelated entries', () => {
|
||||
it('rejects unknown types', async () => {
|
||||
const processor = BitbucketDiscoveryProcessor.fromConfig(
|
||||
new ConfigReader({
|
||||
integrations: {
|
||||
bitbucket: [{ host: 'bitbucket.mycompany.com', token: 'blob' }],
|
||||
},
|
||||
}),
|
||||
{ logger: getVoidLogger() },
|
||||
);
|
||||
const location: LocationSpec = {
|
||||
type: 'not-bitbucket-discovery',
|
||||
target: 'https://bitbucket.mycompany.com',
|
||||
};
|
||||
await expect(
|
||||
processor.readLocation(location, false, () => {}),
|
||||
).resolves.toBeFalsy();
|
||||
});
|
||||
|
||||
it('rejects unknown targets', async () => {
|
||||
const processor = BitbucketDiscoveryProcessor.fromConfig(
|
||||
new ConfigReader({
|
||||
integrations: {
|
||||
bitbucket: [
|
||||
{ host: 'bitbucket.org', token: 'blob' },
|
||||
{ host: 'bitbucket.mycompany.com', token: 'blob' },
|
||||
],
|
||||
},
|
||||
}),
|
||||
{ logger: getVoidLogger() },
|
||||
);
|
||||
const location: LocationSpec = {
|
||||
type: 'bitbucket-discovery',
|
||||
target: 'https://not.bitbucket.mycompany.com/foobar',
|
||||
};
|
||||
await expect(
|
||||
processor.readLocation(location, false, () => {}),
|
||||
).rejects.toThrow(
|
||||
/There is no Bitbucket integration that matches https:\/\/not.bitbucket.mycompany.com\/foobar/,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('handles repositories', () => {
|
||||
it('output all repositories', async () => {
|
||||
const target =
|
||||
'https://bitbucket.mycompany.com/projects/*/repos/*/catalog.yaml';
|
||||
|
||||
client.listProjects.mockResolvedValue(
|
||||
pagedResponse([{ key: 'backstage' }, { key: 'demo' }]),
|
||||
);
|
||||
client.listRepositories.mockResolvedValueOnce(
|
||||
pagedResponse([
|
||||
{
|
||||
slug: 'backstage',
|
||||
links: {
|
||||
self: [
|
||||
{
|
||||
href:
|
||||
'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
]),
|
||||
);
|
||||
client.listRepositories.mockResolvedValueOnce(
|
||||
pagedResponse([
|
||||
{
|
||||
slug: 'demo',
|
||||
links: {
|
||||
self: [
|
||||
{
|
||||
href:
|
||||
'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
]),
|
||||
);
|
||||
|
||||
const actual = await readBitbucketOrg(client, target);
|
||||
expect(actual).toContainEqual({
|
||||
type: 'url',
|
||||
target:
|
||||
'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse/catalog.yaml',
|
||||
});
|
||||
expect(actual).toContainEqual({
|
||||
type: 'url',
|
||||
target:
|
||||
'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse/catalog.yaml',
|
||||
});
|
||||
});
|
||||
|
||||
it('output repositories with wildcards', async () => {
|
||||
const target =
|
||||
'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-*/catalog.yaml';
|
||||
|
||||
client.listProjects.mockResolvedValue(
|
||||
pagedResponse([{ key: 'backstage' }]),
|
||||
);
|
||||
client.listRepositories.mockResolvedValueOnce(
|
||||
pagedResponse([
|
||||
{ slug: 'backstage' },
|
||||
{
|
||||
slug: 'techdocs-cli',
|
||||
links: {
|
||||
self: [
|
||||
{
|
||||
href:
|
||||
'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'techdocs-container',
|
||||
links: {
|
||||
self: [
|
||||
{
|
||||
href:
|
||||
'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
]),
|
||||
);
|
||||
|
||||
const actual = await readBitbucketOrg(client, target);
|
||||
|
||||
expect(actual).toContainEqual({
|
||||
type: 'url',
|
||||
target:
|
||||
'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse/catalog.yaml',
|
||||
});
|
||||
expect(actual).toContainEqual({
|
||||
type: 'url',
|
||||
target:
|
||||
'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse/catalog.yaml',
|
||||
});
|
||||
});
|
||||
it('filter unrelated repositories', async () => {
|
||||
const target =
|
||||
'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml';
|
||||
|
||||
client.listProjects.mockResolvedValue(
|
||||
pagedResponse([{ key: 'backstage' }]),
|
||||
);
|
||||
client.listRepositories.mockResolvedValue(
|
||||
pagedResponse([
|
||||
{ slug: 'abstest' },
|
||||
{ slug: 'testxyz' },
|
||||
{
|
||||
slug: 'test',
|
||||
links: {
|
||||
self: [
|
||||
{
|
||||
href:
|
||||
'https://bitbucket.mycompany.com/projects/backstage/repos/test',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
]),
|
||||
);
|
||||
|
||||
const actual = await readBitbucketOrg(client, target);
|
||||
|
||||
expect(actual).toContainEqual({
|
||||
type: 'url',
|
||||
target:
|
||||
'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml',
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,147 @@
|
||||
/*
|
||||
* Copyright 2021 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { Logger } from 'winston';
|
||||
import { Config } from '@backstage/config';
|
||||
|
||||
import { ScmIntegrations } from '@backstage/integration';
|
||||
import { LocationSpec } from '@backstage/catalog-model';
|
||||
import { BitbucketClient, pageIterator } from './bitbucket';
|
||||
import { CatalogProcessor, CatalogProcessorEmit } from './types';
|
||||
import { results } from './index';
|
||||
|
||||
export class BitbucketDiscoveryProcessor implements CatalogProcessor {
|
||||
private readonly integrations: ScmIntegrations;
|
||||
private readonly logger: Logger;
|
||||
|
||||
static fromConfig(config: Config, options: { logger: Logger }) {
|
||||
const integrations = ScmIntegrations.fromConfig(config);
|
||||
|
||||
return new BitbucketDiscoveryProcessor({
|
||||
...options,
|
||||
integrations,
|
||||
});
|
||||
}
|
||||
|
||||
constructor(options: { integrations: ScmIntegrations; logger: Logger }) {
|
||||
this.integrations = options.integrations;
|
||||
this.logger = options.logger;
|
||||
}
|
||||
|
||||
async readLocation(
|
||||
location: LocationSpec,
|
||||
_optional: boolean,
|
||||
emit: CatalogProcessorEmit,
|
||||
): Promise<boolean> {
|
||||
if (location.type !== 'bitbucket-discovery') {
|
||||
return false;
|
||||
}
|
||||
|
||||
const bitbucketConfig = this.integrations.bitbucket.byUrl(location.target)
|
||||
?.config;
|
||||
if (!bitbucketConfig) {
|
||||
throw new Error(
|
||||
`There is no Bitbucket integration that matches ${location.target}. Please add a configuration entry for it under integrations.bitbucket`,
|
||||
);
|
||||
} else if (bitbucketConfig.host === 'bitbucket.org') {
|
||||
throw new Error(
|
||||
`Component discovery for Bitbucket Cloud is not yet supported`,
|
||||
);
|
||||
}
|
||||
|
||||
const client = new BitbucketClient({
|
||||
config: bitbucketConfig,
|
||||
logger: this.logger,
|
||||
});
|
||||
const startTimestamp = Date.now();
|
||||
this.logger.info(`Reading Bitbucket repositories from ${location.target}`);
|
||||
|
||||
const repositories = await readBitbucketOrg(client, location.target);
|
||||
|
||||
for (const repository of repositories) {
|
||||
emit(
|
||||
results.location(
|
||||
repository,
|
||||
// Not all locations may actually exist, since the user defined them as a wildcard pattern.
|
||||
// Thus, we emit them as optional and let the downstream processor find them while not outputting
|
||||
// an error if it couldn't.
|
||||
true,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1);
|
||||
this.logger.info(
|
||||
`Read ${repositories.length} Bitbucket repositories in ${duration} seconds`,
|
||||
);
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
export async function readBitbucketOrg(
|
||||
client: BitbucketClient,
|
||||
target: string,
|
||||
): Promise<LocationSpec[]> {
|
||||
const { projectSearchPath, repoSearchPath, catalogPath } = parseUrl(target);
|
||||
const projectIterator = pageIterator(options => client.listProjects(options));
|
||||
let result: LocationSpec[] = [];
|
||||
|
||||
for await (const page of projectIterator) {
|
||||
for (const project of page.values) {
|
||||
if (!projectSearchPath.test(project.key)) {
|
||||
continue;
|
||||
}
|
||||
const repoIterator = pageIterator(options =>
|
||||
client.listRepositories(project.key, options),
|
||||
);
|
||||
for await (const repoPage of repoIterator) {
|
||||
result = result.concat(
|
||||
repoPage.values
|
||||
.filter(v => repoSearchPath.test(v.slug))
|
||||
.map(repo => {
|
||||
return {
|
||||
type: 'url',
|
||||
target: `${repo.links.self[0].href}${catalogPath}`,
|
||||
};
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function parseUrl(
|
||||
urlString: string,
|
||||
): { projectSearchPath: RegExp; repoSearchPath: RegExp; catalogPath: string } {
|
||||
const url = new URL(urlString);
|
||||
const path = url.pathname.substr(1).split('/');
|
||||
|
||||
// /backstage/techdocs-*/blob/master/catalog-info.yaml
|
||||
if (path.length > 3 && path[1].length && path[3].length) {
|
||||
return {
|
||||
projectSearchPath: escapeRegExp(decodeURIComponent(path[1])),
|
||||
repoSearchPath: escapeRegExp(decodeURIComponent(path[3])),
|
||||
catalogPath: `/${decodeURIComponent(path.slice(4).join('/'))}`,
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error(`Failed to parse ${urlString}`);
|
||||
}
|
||||
|
||||
function escapeRegExp(str: string): RegExp {
|
||||
return new RegExp(`^${str.replace(/\*/g, '.*')}$`);
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
/*
|
||||
* Copyright 2021 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { Logger } from 'winston';
|
||||
import fetch from 'cross-fetch';
|
||||
|
||||
import {
|
||||
BitbucketIntegrationConfig,
|
||||
getBitbucketRequestOptions,
|
||||
} from '@backstage/integration';
|
||||
|
||||
export class BitbucketClient {
|
||||
private readonly config: BitbucketIntegrationConfig;
|
||||
private readonly logger: Logger;
|
||||
|
||||
constructor(options: { config: BitbucketIntegrationConfig; logger: Logger }) {
|
||||
this.config = options.config;
|
||||
this.logger = options.logger;
|
||||
}
|
||||
|
||||
async listProjects(options?: ListOptions): Promise<PagedResponse<any>> {
|
||||
return this.pagedRequest(`${this.config.apiBaseUrl}/projects`, options);
|
||||
}
|
||||
|
||||
async listRepositories(
|
||||
projectKey: string,
|
||||
options?: ListOptions,
|
||||
): Promise<PagedResponse<any>> {
|
||||
return this.pagedRequest(
|
||||
`${this.config.apiBaseUrl}/projects/${projectKey}/repos`,
|
||||
options,
|
||||
);
|
||||
}
|
||||
|
||||
private async pagedRequest(
|
||||
endpoint: string,
|
||||
options?: ListOptions,
|
||||
): Promise<PagedResponse<any>> {
|
||||
const request = new URL(endpoint);
|
||||
if (options) {
|
||||
(Object.keys(options) as Array<keyof typeof options>).forEach(key => {
|
||||
const value: any = options[key] as any;
|
||||
if (value) {
|
||||
request.searchParams.append(key, value);
|
||||
}
|
||||
});
|
||||
}
|
||||
const response = await fetch(
|
||||
request.toString(),
|
||||
getBitbucketRequestOptions(this.config),
|
||||
);
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
`Unexpected response when fetching ${request.toString()}. Expected 200 but got ${
|
||||
response.status
|
||||
} - ${response.statusText}`,
|
||||
);
|
||||
}
|
||||
return response.json().then(repositories => {
|
||||
return repositories as PagedResponse<any>;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export type ListOptions = {
|
||||
limit?: number | undefined;
|
||||
start?: number | undefined;
|
||||
};
|
||||
|
||||
export type PagedResponse<T> = {
|
||||
size: number;
|
||||
limit: number;
|
||||
start: number;
|
||||
isLastPage: boolean;
|
||||
values: T[];
|
||||
nextPageStart: number;
|
||||
};
|
||||
|
||||
export function pageIterator(
|
||||
pagedRequest: (options: ListOptions) => Promise<PagedResponse<any>>,
|
||||
options?: ListOptions,
|
||||
): AsyncIterable<PagedResponse<any>> {
|
||||
return {
|
||||
[Symbol.asyncIterator]: () => {
|
||||
const opts = options || { start: 0 };
|
||||
let finished = false;
|
||||
return {
|
||||
async next() {
|
||||
if (!finished) {
|
||||
try {
|
||||
const response = await pagedRequest(opts);
|
||||
finished = response.isLastPage;
|
||||
opts.start = response.nextPageStart;
|
||||
return Promise.resolve({
|
||||
value: response,
|
||||
done: false,
|
||||
});
|
||||
} catch (error) {
|
||||
return Promise.reject({
|
||||
value: undefined,
|
||||
done: true,
|
||||
error: error,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
opts.start = 0;
|
||||
finished = false;
|
||||
return Promise.resolve({
|
||||
value: undefined,
|
||||
done: true,
|
||||
});
|
||||
}
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
/*
|
||||
* Copyright 2021 Spotify AB
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
export { BitbucketClient, pageIterator } from './client';
|
||||
export type { PagedResponse } from './client';
|
||||
Reference in New Issue
Block a user