paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 29 Packages Projects Releases Wiki Activity

Changed format for target location so that query can be passed directly to bitbucket api

Browse Source 
Signed-off-by: Bart Breen <bart.breen@twobulls.com>
This commit is contained in:
Bart Breen
2021-07-28 23:27:11 +10:00
parent d02ecd3eb3
commit b6801e6d70
4 changed files with 201 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/integrations/bitbucket/discovery.md
+33 -19
View File
@@ -54,30 +54,44 @@ you can add a location target to the catalog configuration:
catalog:
locations:
- type: bitbucket-discovery
target: https://bitbucket.org/workspaces/my-workspace/projects/my-project/repos/service-*/catalog-info.yaml
target: https://bitbucket.org/workspaces/my-workspace
```
Note the `bitbucket-discovery` type, as this is not a regular `url` processor.
The target is composed of four parts:
The target is composed of the following parts:
- The base instance URL, `https://bitbucket.org` in this case
- The workspace name to scan, which must match a workspace accessible with the
username of your integration.
- The project key to scan, which accepts \* wildcard tokens. This can simply be
`*` to include all repositories. This example only returns repositories in the
`my-project` project.
- The repository blob to scan, which accepts \* wildcard tokens. This can simply
be `*` to scan all repositories in the workspace. This example only looks for
repositories prefixed with `service-`.
- The path within each repository to find the catalog YAML file. This will
usually be `/catalog-info.yaml` or a similar variation for catalog files
stored in the root directory of each repository. If omitted, the default value
`catalog-info.yaml` will be used. E.g. given that `my-project` and `service-a`
exists,
`https://bitbucket.org/workspaces/my-workspace/projects/my-project/repos/service-*/`
will result in:
`https://bitbucket.org/my-workspace/service-a/src/master/catalog-info.yaml`.
- The base URL for Bitbucket, `https://bitbucket.org`
- The workspace name to scan (following the `workspaces/` path part), which must
match a workspace accessible with the username of your integration.
- (Optional) The project key to scan (following the `projects/` path part),
which accepts \* wildcard tokens. If ommitted, repositories from all projects
in the workspace are included.
- (Optional) The repository blob to scan (following the `repos/` path part),
which accepts \* wildcard tokens. If ommitted, all repositories in the
workspace are included.
- (Optional) The `catalogPath` query argument to specify the location within
each repository to find the catalog YAML file. This will usually be
`/catalog-info.yaml` or a similar variation for catalog files stored in the
root directory of each repository. If omitted, the default value
`catalog-info.yaml` will be used.
- (Optional) The `q` query argument to be passed through to Bitbucket for
filtering results via the API. This is the most flexible option and will
reduce the amount of API calls if you have a large workspace.
[See here for the specification](https://developer.atlassian.com/bitbucket/api/2/reference/meta/filtering)
for the query argument (will be passed as the `q` query parameter).
Examples:
- `https://bitbucket.org/workspaces/my-workspace/projects/my-project` will find
all repositories in the `my-project` project in the `my-workspace` workspace.
- `https://bitbucket.org/workspaces/my-workspace/repos/service-*` will find all
repositories starting with `service-` in the `my-workspace` workspace.
- `https://bitbucket.org/workspaces/my-workspace/projects/apis-*/repos/service-*`
will find all repositories starting with `service-`, in all projects starting
with `apis-` in the `my-workspace` workspace.
- `https://bitbucket.org/workspaces/my-workspace?q=project.key ~ "my-project"`
will find all repositories in a project containing `my-project` in its key.
## Custom repository processing
plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts
+127 -4
View File
@@ -88,6 +88,7 @@ function setupBitbucketCloudStubs(
workspace: string,
repositories: Pick<BitbucketRepository20, 'slug' | 'project'>[],
) {
const stubCallerFn = jest.fn();
function pagedResponse(values: any): PagedResponse20<any> {
return {
values: values,
@@ -99,6 +100,7 @@ function setupBitbucketCloudStubs(
rest.get(
`https://api.bitbucket.org/2.0/repositories/${workspace}`,
(_, res, ctx) => {
stubCallerFn(_);
return res(
ctx.json(
pagedResponse(
@@ -116,6 +118,7 @@ function setupBitbucketCloudStubs(
},
),
);
return stubCallerFn;
}
describe('BitbucketDiscoveryProcessor', () => {
@@ -310,6 +313,77 @@ describe('BitbucketDiscoveryProcessor', () => {
{ logger: getVoidLogger() },
);
it('output all repositories by default', async () => {
setupBitbucketCloudStubs('myworkspace', [
{ project: { key: 'prj-one' }, slug: 'repository-one' },
{ project: { key: 'prj-two' }, slug: 'repository-two' },
]);
const location: LocationSpec = {
type: 'bitbucket-discovery',
target: 'https://bitbucket.org/workspaces/myworkspace',
};
const emitter = jest.fn();
await processor.readLocation(location, false, emitter);
expect(emitter).toBeCalledTimes(2);
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.org/myworkspace/repository-one/src/master/catalog-info.yaml',
},
optional: true,
});
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.org/myworkspace/repository-two/src/master/catalog-info.yaml',
},
optional: true,
});
});
it('uses provided catalog path', async () => {
setupBitbucketCloudStubs('myworkspace', [
{ project: { key: 'prj-one' }, slug: 'repository-one' },
{ project: { key: 'prj-two' }, slug: 'repository-two' },
]);
const location: LocationSpec = {
type: 'bitbucket-discovery',
target:
'https://bitbucket.org/workspaces/myworkspace?catalogPath=my/nested/path/catalog.yaml',
};
const emitter = jest.fn();
await processor.readLocation(location, false, emitter);
expect(emitter).toBeCalledTimes(2);
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.org/myworkspace/repository-one/src/master/my/nested/path/catalog.yaml',
},
optional: true,
});
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.org/myworkspace/repository-two/src/master/my/nested/path/catalog.yaml',
},
optional: true,
});
});
it('output all repositories', async () => {
setupBitbucketCloudStubs('myworkspace', [
{ project: { key: 'prj-one' }, slug: 'repository-one' },
@@ -318,7 +392,7 @@ describe('BitbucketDiscoveryProcessor', () => {
const location: LocationSpec = {
type: 'bitbucket-discovery',
target:
'https://bitbucket.org/workspaces/myworkspace/projects/*/repos/*/catalog.yaml',
'https://bitbucket.org/workspaces/myworkspace/projects/*/repos/*?catalogPath=catalog.yaml',
};
const emitter = jest.fn();
@@ -354,7 +428,7 @@ describe('BitbucketDiscoveryProcessor', () => {
const location: LocationSpec = {
type: 'bitbucket-discovery',
target:
'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*/catalog.yaml',
'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*?catalogPath=catalog.yaml',
};
const emitter = jest.fn();
@@ -371,6 +445,7 @@ describe('BitbucketDiscoveryProcessor', () => {
optional: true,
});
});
it('filter unrelated repositories', async () => {
setupBitbucketCloudStubs('myworkspace', [
{ project: { key: 'prj-one' }, slug: 'repository-one' },
@@ -380,7 +455,7 @@ describe('BitbucketDiscoveryProcessor', () => {
const location: LocationSpec = {
type: 'bitbucket-discovery',
target:
'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/repository-three/catalog.yaml',
'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/repository-three?catalogPath=catalog.yaml',
};
const emitter = jest.fn();
@@ -398,13 +473,42 @@ describe('BitbucketDiscoveryProcessor', () => {
});
});
it('submits query', async () => {
const mockCall = setupBitbucketCloudStubs('myworkspace', [
{ project: { key: 'prj-one' }, slug: 'repository-one' },
]);
const location: LocationSpec = {
type: 'bitbucket-discovery',
target:
'https://bitbucket.org/workspaces/myworkspace?q=project.key ~ "prj-one"',
};
const emitter = jest.fn();
await processor.readLocation(location, false, emitter);
expect(emitter).toBeCalledTimes(1);
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.org/myworkspace/repository-one/src/master/catalog-info.yaml',
},
optional: true,
});
expect(mockCall).toBeCalledTimes(1);
// it should be possible to do this via an `expect.objectContaining` check but seems to fail with some encoding issue.
expect(mockCall.mock.calls[0][0].url).toMatchInlineSnapshot(
`"https://api.bitbucket.org/2.0/repositories/myworkspace?page=1&pagelen=100&q=project.key+%7E+%22prj-one%22"`,
);
});
it.each`
target
${'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*'}
${'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*/'}
${'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/repository-*/'}
`("target '$target' adds default path to catalog", async ({ target }) => {
setupStubs([{ key: 'backstage', repos: ['techdocs-cli'] }]);
setupBitbucketCloudStubs('myworkspace', [
{ project: { key: 'prj-one' }, slug: 'repository-one' },
]);
@@ -428,6 +532,25 @@ describe('BitbucketDiscoveryProcessor', () => {
optional: true,
});
});
it.each`
target
${'https://bitbucket.org/test'}
`("target '$target' is rejected", async ({ target }) => {
setupBitbucketCloudStubs('myworkspace', [
{ project: { key: 'prj-one' }, slug: 'repository-one' },
]);
const location: LocationSpec = {
type: 'bitbucket-discovery',
target: target,
};
const emitter = jest.fn();
await expect(
processor.readLocation(location, false, emitter),
).rejects.toThrow(/Failed to parse /);
});
});
describe('Custom repository parser', () => {
plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts
+36 -16
View File
@@ -196,11 +196,16 @@ export async function readBitbucketCloud(
): Promise<Result<BitbucketRepository20>> {
const {
workspacePath,
queryParam: q,
projectSearchPath,
repoSearchPath,
} = parseBitbucketCloudUrl(target);
const repositories = paginated20(options =>
client.listRepositoriesByWorkspace20(workspacePath, options),
const repositories = paginated20(
options => client.listRepositoriesByWorkspace20(workspacePath, options),
{
q,
},
);
const result: Result<BitbucketRepository20> = {
scanned: 0,
@@ -210,8 +215,8 @@ export async function readBitbucketCloud(
for await (const repository of repositories) {
result.scanned++;
if (
projectSearchPath.test(repository.project.key) &&
repoSearchPath.test(repository.slug)
(!projectSearchPath || projectSearchPath.test(repository.project.key)) &&
(!repoSearchPath || repoSearchPath.test(repository.slug))
) {
result.matches.push(repository);
}
@@ -237,28 +242,43 @@ function parseUrl(
throw new Error(`Failed to parse ${urlString}`);
}
function readPathParameters(pathParts: string[]): Map<string, string> {
const vals: Record<string, any> = {};
for (let i = 0; i < pathParts.length; i += 2) {
if (i + 1 >= pathParts.length) continue;
vals[pathParts[i]] = decodeURIComponent(pathParts[i + 1]);
}
return new Map<string, string>(Object.entries(vals));
}
function parseBitbucketCloudUrl(
urlString: string,
): {
workspacePath: string;
projectSearchPath: RegExp;
repoSearchPath: RegExp;
catalogPath: string;
projectSearchPath?: RegExp;
repoSearchPath?: RegExp;
queryParam?: string;
} {
const url = new URL(urlString);
const path = url.pathname.substr(1).split('/');
const pathMap = readPathParameters(url.pathname.substr(1).split('/'));
const query = url.searchParams;
// workspaces/{workspacePath}/projects/{projectSearchPath}/repos/{repoSearchPath}/{catalogPath=catalog-info.yaml}
if (path.length > 5 && path[1].length && path[3].length) {
return {
workspacePath: decodeURIComponent(path[1]),
projectSearchPath: escapeRegExp(decodeURIComponent(path[3])),
repoSearchPath: escapeRegExp(decodeURIComponent(path[5])),
catalogPath: `/${decodeURIComponent(path.slice(6).join('/'))}`,
};
if (!pathMap.has('workspaces')) {
throw new Error(`Failed to parse workspace from ${urlString}`);
}
throw new Error(`Failed to parse ${urlString}`);
return {
workspacePath: pathMap.get('workspaces')!,
projectSearchPath: pathMap.has('projects')
? escapeRegExp(pathMap.get('projects')!)
: undefined,
repoSearchPath: pathMap.has('repos')
? escapeRegExp(pathMap.get('repos')!)
: undefined,
catalogPath: `/${query.get('catalogPath') || ''}`,
queryParam: query.get('q') || undefined,
};
}
function escapeRegExp(str: string): RegExp {
plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts
+5 -5
View File
@@ -34,7 +34,7 @@ export class BitbucketClient {
async listRepositoriesByWorkspace20(
workspace: string,
options?: ListOptions,
options?: ListOptions20,
): Promise<PagedResponse20<BitbucketRepository20>> {
return this.pagedRequest20<BitbucketRepository20>(
`${this.config.apiBaseUrl}/repositories/${workspace}`,
@@ -81,7 +81,7 @@ export class BitbucketClient {
private async pagedRequest20<T = any>(
endpoint: string,
options?: ListOptions,
options?: ListOptions20,
): Promise<PagedResponse20<T>> {
const request = new URL(endpoint);
for (const key in options) {
@@ -123,7 +123,7 @@ export type PagedResponse<T> = {
};
export type ListOptions20 = {
[key: string]: number | undefined;
[key: string]: string | number | undefined;
page?: number | undefined;
pagelen?: number | undefined;
};
@@ -155,11 +155,11 @@ export async function* paginated20<T = any>(
request: (options: ListOptions20) => Promise<PagedResponse20<T>>,
options?: ListOptions20,
) {
const opts = options || { page: 1, pagelen: 100 };
const opts = { page: 1, pagelen: 100, ...options };
let res;
do {
res = await request(opts);
opts.page = (opts.page || 1) + 1;
opts.page = opts.page + 1;
for (const item of res.values) {
yield item;
}