feat(integration,bitbucket): discover catalog files using Bitbucket Cloud's code search

Support discovering catalog files using Bitbucket Cloud's code search
which allows finding multiple catalog files per repository (e.g., monorepo)
as well as creating Location entities for existing locations only.
(compared to creating an optional Location per matching repository using the
defined static path for the catalog file).

Closes: #9765
Signed-off-by: Patrick Jungermann <Patrick.Jungermann@gmail.com>
This commit is contained in:
Patrick Jungermann
2022-03-03 00:43:37 +01:00
parent f73c71980d
commit 616f02ade2
7 changed files with 504 additions and 27 deletions
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-catalog-backend': patch
---
support Bitbucket Cloud's code search to discover catalog files (multiple per repo, Location entities for existing files only)
+29
View File
@@ -80,6 +80,18 @@ The target is composed of the following parts:
reduce the amount of API calls if you have a large workspace.
[See here for the specification](https://developer.atlassian.com/bitbucket/api/2/reference/meta/filtering)
for the query argument (will be passed as the `q` query parameter).
- (Optional) The `search=true` query argument to activate the mode utilizing code search.
- Is mutually exclusive to the `q` query argument.
- Allows providing values at `catalogPath` for finding catalog files as allowed by the `path` filter/modifier
[at Bitbucket Cloud's code search](https://confluence.atlassian.com/bitbucket/code-search-in-bitbucket-873876782.html#Search-Pathmodifier).
- `catalogPath=/catalog-info.yaml`
- `catalogPath=catalog-info.yaml` (anywhere in the repository)
- `catalogPath=/path/catalog-info.yaml`
- `catalogPath=path/catalog-info.yaml`
- `catalogPath=/path/*/catalog-info.yaml`
- `catalogPath=path/*/catalog-info.yaml`
- Supports multiple catalog files per repository depending on the `catalogPath` value.
- Registers `Location` entities for existing files only vs all matching repositories.
Examples:
@@ -95,6 +107,23 @@ Examples:
- `https://bitbucket.org/workspaces/my-workspace?catalogPath=my/nested/path/catalog.yaml`
will find all repositories in the `my-workspace` workspace and use the catalog
file at `my/nested/path/catalog.yaml`.
- `https://bitbucket.org/workspaces/my-workspace?search=true&catalogPath=/catalog.yaml`
will find all `catalog.yaml` files located in the root of repositories in the workspace `my-workspace`.
- `https://bitbucket.org/workspaces/my-workspace?search=true&catalogPath=catalog.yaml`
will find all `catalog.yaml` files located anywhere within repositories in the workspace `my-workspace`.
- `https://bitbucket.org/workspaces/my-workspace?search=true&catalogPath=/my/nested/path/catalog.yaml`
will find all `catalog.yaml` files located within the directory `/my/nested/path/` within
repositories in the workspace `my-workspace`.
- `https://bitbucket.org/workspaces/my-workspace?search=true&catalogPath=my/nested/path/catalog.yaml`
will find all `catalog.yaml` files located within the directory `my/nested/path/` located anywhere within
repositories in the workspace `my-workspace`.
- `https://bitbucket.org/workspaces/my-workspace?search=true&catalogPath=/my/*/path/catalog.yaml`
will find all `catalog.yaml` files located within a directory `path/` located within any (recursive) directory
within the directory `my/` in the root of repositories in the workspace `my-workspace`
(`/my/nested/path/catalog.yaml`, `/my/very/nested/path/catalog.yaml`, ...).
- `https://bitbucket.org/workspaces/my-workspace/projects/apis-*/repos/service-*?search=true&catalogPath=catalog.yaml`
will find all `catalog.yaml` files located anywhere within repositories starting with `service-`
in projects starting with `api-` in the workspace `my-workspace`.
## Custom repository processing
+3
View File
@@ -104,6 +104,7 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
parser?: (options: {
integration: BitbucketIntegration;
target: string;
presence?: 'optional' | 'required';
logger: Logger;
}) => AsyncIterable<CatalogProcessorResult>;
logger: Logger;
@@ -115,6 +116,7 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
parser?: (options: {
integration: BitbucketIntegration;
target: string;
presence?: 'optional' | 'required';
logger: Logger;
}) => AsyncIterable<CatalogProcessorResult>;
logger: Logger;
@@ -134,6 +136,7 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
export type BitbucketRepositoryParser = (options: {
integration: BitbucketIntegration;
target: string;
presence?: 'optional' | 'required';
logger: Logger;
}) => AsyncIterable<CatalogProcessorResult>;
@@ -116,6 +116,60 @@ function setupBitbucketCloudStubs(
return stubCallerFn;
}
function setupBitbucketCloudSearchStubs(
workspace: string,
repositories: Pick<BitbucketRepository20, 'slug' | 'project'>[],
catalogPath: string,
) {
const stubCallerFn = jest.fn();
function pagedResponse(values: any): PagedResponse20<any> {
return {
values: values,
page: 1,
} as PagedResponse20<any>;
}
server.use(
rest.get(
`https://api.bitbucket.org/2.0/workspaces/${workspace}/search/code`,
(req, res, ctx) => {
stubCallerFn(req);
return res(
ctx.json(
pagedResponse(
repositories.map(r => ({
type: 'code_search_result',
content_match_count: 0,
content_matches: [],
path_matches: [
catalogPath
.split('/')
.flatMap(seg => [{ text: '/' }, { text: seg, match: true }])
.slice(1),
],
file: {
commit: {
repository: {
...r,
links: {
html: {
href: `https://bitbucket.org/${workspace}/${r.slug}`,
},
},
},
},
path: catalogPath,
},
})),
),
),
);
},
),
);
return stubCallerFn;
}
describe('BitbucketDiscoveryProcessor', () => {
beforeAll(() => server.listen());
afterEach(() => server.resetHandlers());
@@ -736,6 +790,256 @@ describe('BitbucketDiscoveryProcessor', () => {
});
});
describe('handles cloud repositories using code search', () => {
const processor = BitbucketDiscoveryProcessor.fromConfig(
new ConfigReader({
integrations: {
bitbucket: [
{
host: 'bitbucket.org',
username: 'myuser',
appPassword: 'blob',
},
],
},
}),
{ logger: getVoidLogger() },
);
it('output all repositories by default', async () => {
setupBitbucketCloudSearchStubs(
'myworkspace',
[
{ project: { key: 'prj-one' }, slug: 'repository-one' },
{ project: { key: 'prj-two' }, slug: 'repository-two' },
],
'catalog-info.yaml',
);
const location: LocationSpec = {
type: 'bitbucket-discovery',
target: 'https://bitbucket.org/workspaces/myworkspace?search=true',
};
const emitter = jest.fn();
await processor.readLocation(location, false, emitter);
expect(emitter).toBeCalledTimes(2);
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.org/myworkspace/repository-one/src/master/catalog-info.yaml',
presence: 'required',
},
});
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.org/myworkspace/repository-two/src/master/catalog-info.yaml',
presence: 'required',
},
});
});
it('uses provided catalog path', async () => {
setupBitbucketCloudSearchStubs(
'myworkspace',
[
{ project: { key: 'prj-one' }, slug: 'repository-one' },
{ project: { key: 'prj-two' }, slug: 'repository-two' },
],
'my/nested/path/catalog.yaml',
);
const location: LocationSpec = {
type: 'bitbucket-discovery',
target:
'https://bitbucket.org/workspaces/myworkspace?search=true&catalogPath=my/nested/path/catalog.yaml',
};
const emitter = jest.fn();
await processor.readLocation(location, false, emitter);
expect(emitter).toBeCalledTimes(2);
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.org/myworkspace/repository-one/src/master/my/nested/path/catalog.yaml',
presence: 'required',
},
});
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.org/myworkspace/repository-two/src/master/my/nested/path/catalog.yaml',
presence: 'required',
},
});
});
it('output all repositories', async () => {
setupBitbucketCloudSearchStubs(
'myworkspace',
[
{ project: { key: 'prj-one' }, slug: 'repository-one' },
{ project: { key: 'prj-two' }, slug: 'repository-two' },
],
'catalog.yaml',
);
const location: LocationSpec = {
type: 'bitbucket-discovery',
target:
'https://bitbucket.org/workspaces/myworkspace/projects/*/repos/*?search=true&catalogPath=catalog.yaml',
};
const emitter = jest.fn();
await processor.readLocation(location, false, emitter);
expect(emitter).toBeCalledTimes(2);
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.org/myworkspace/repository-one/src/master/catalog.yaml',
presence: 'required',
},
});
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.org/myworkspace/repository-two/src/master/catalog.yaml',
presence: 'required',
},
});
});
it('output repositories with wildcards', async () => {
setupBitbucketCloudSearchStubs(
'myworkspace',
[
{ project: { key: 'prj-one' }, slug: 'repository-one' },
{ project: { key: 'prj-two' }, slug: 'repository-two' },
],
'catalog.yaml',
);
const location: LocationSpec = {
type: 'bitbucket-discovery',
target:
'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*?search=true&catalogPath=catalog.yaml',
};
const emitter = jest.fn();
await processor.readLocation(location, false, emitter);
expect(emitter).toBeCalledTimes(1);
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.org/myworkspace/repository-one/src/master/catalog.yaml',
presence: 'required',
},
});
});
it('filter unrelated repositories', async () => {
setupBitbucketCloudSearchStubs(
'myworkspace',
[
{ project: { key: 'prj-one' }, slug: 'repository-one' },
{ project: { key: 'prj-one' }, slug: 'repository-two' },
{ project: { key: 'prj-one' }, slug: 'repository-three' },
],
'catalog.yaml',
);
const location: LocationSpec = {
type: 'bitbucket-discovery',
target:
'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/repository-three?search=true&catalogPath=catalog.yaml',
};
const emitter = jest.fn();
await processor.readLocation(location, false, emitter);
expect(emitter).toBeCalledTimes(1);
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.org/myworkspace/repository-three/src/master/catalog.yaml',
presence: 'required',
},
});
});
it.each`
target
${'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*?search=true'}
${'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*/?search=true'}
${'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/repository-*/?search=true'}
`("target '$target' adds default path to catalog", async ({ target }) => {
setupBitbucketCloudSearchStubs(
'myworkspace',
[{ project: { key: 'prj-one' }, slug: 'repository-one' }],
'catalog-info.yaml',
);
const location: LocationSpec = {
type: 'bitbucket-discovery',
target: target,
};
const emitter = jest.fn();
await processor.readLocation(location, false, emitter);
expect(emitter).toHaveBeenCalledTimes(1);
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.org/myworkspace/repository-one/src/master/catalog-info.yaml',
presence: 'required',
},
});
});
it.each`
target
${'https://bitbucket.org/test?search=true'}
`("target '$target' is rejected", async ({ target }) => {
setupBitbucketCloudSearchStubs(
'myworkspace',
[{ project: { key: 'prj-one' }, slug: 'repository-one' }],
'catalog-info.yaml',
);
const location: LocationSpec = {
type: 'bitbucket-discovery',
target: target,
};
const emitter = jest.fn();
await expect(
processor.readLocation(location, false, emitter),
).rejects.toThrow(/Failed to parse /);
});
});
describe('Custom repository parser', () => {
const processor = BitbucketDiscoveryProcessor.fromConfig(
new ConfigReader({
@@ -39,7 +39,6 @@ import {
const DEFAULT_BRANCH = 'master';
const DEFAULT_CATALOG_LOCATION = '/catalog-info.yaml';
const EMPTY_CATALOG_LOCATION = '/';
/** @public */
export class BitbucketDiscoveryProcessor implements CatalogProcessor {
@@ -47,6 +46,7 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
private readonly parser: (options: {
integration: BitbucketIntegration;
target: string;
presence?: 'optional' | 'required';
logger: Logger;
}) => AsyncIterable<CatalogProcessorResult>;
private readonly logger: Logger;
@@ -57,6 +57,7 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
parser?: (options: {
integration: BitbucketIntegration;
target: string;
presence?: 'optional' | 'required';
logger: Logger;
}) => AsyncIterable<CatalogProcessorResult>;
logger: Logger;
@@ -75,6 +76,7 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
parser?: (options: {
integration: BitbucketIntegration;
target: string;
presence?: 'optional' | 'required';
logger: Logger;
}) => AsyncIterable<CatalogProcessorResult>;
logger: Logger;
@@ -136,19 +138,17 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
): Promise<ResultSummary> {
const { client, location, integration, emit } = options;
const { catalogPath: requestedCatalogPath } = parseBitbucketCloudUrl(
location.target,
);
const catalogPath =
requestedCatalogPath === EMPTY_CATALOG_LOCATION
? DEFAULT_CATALOG_LOCATION
: requestedCatalogPath;
const result = await readBitbucketCloud(client, location.target);
for (const repository of result.matches) {
const mainbranch = repository.mainbranch?.name ?? DEFAULT_BRANCH;
const { searchEnabled } = parseBitbucketCloudUrl(location.target);
const result = searchEnabled
? await searchBitbucketCloudLocations(client, location.target)
: await readBitbucketCloudLocations(client, location.target);
for (const locationTarget of result.matches) {
for await (const entity of this.parser({
integration,
target: `${repository.links.html.href}/src/${mainbranch}${catalogPath}`,
target: locationTarget,
presence: searchEnabled ? 'required' : 'optional',
logger: this.logger,
})) {
emit(entity);
@@ -165,10 +165,10 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
): Promise<ResultSummary> {
const { client, location, integration, emit } = options;
const { catalogPath: requestedCatalogPath } = parseUrl(location.target);
const catalogPath =
requestedCatalogPath === EMPTY_CATALOG_LOCATION
? DEFAULT_CATALOG_LOCATION
: requestedCatalogPath;
const catalogPath = requestedCatalogPath
? `/${requestedCatalogPath}`
: DEFAULT_CATALOG_LOCATION;
const result = await readBitbucketOrg(client, location.target);
for (const repository of result.matches) {
for await (const entity of this.parser({
@@ -214,6 +214,81 @@ export async function readBitbucketOrg(
return result;
}
export async function searchBitbucketCloudLocations(
client: BitbucketClient,
target: string,
): Promise<Result<string>> {
const {
workspacePath,
catalogPath: requestedCatalogPath,
projectSearchPath,
repoSearchPath,
} = parseBitbucketCloudUrl(target);
const result: Result<string> = {
scanned: 0,
matches: [],
};
const catalogPath = requestedCatalogPath
? requestedCatalogPath
: DEFAULT_CATALOG_LOCATION;
const catalogFilename = catalogPath.substring(
catalogPath.lastIndexOf('/') + 1,
);
const searchResults = paginated20(options =>
client.searchCode(
workspacePath,
`"${catalogFilename}" path:${catalogPath}`,
options,
),
);
for await (const searchResult of searchResults) {
// not a file match, but a code match
if (searchResult.path_matches.length === 0) {
continue;
}
const repository = searchResult.file.commit.repository;
if (!matchesPostFilters(repository, projectSearchPath, repoSearchPath)) {
continue;
}
const repoUrl = repository.links.html.href;
const branch = repository.mainbranch?.name ?? DEFAULT_BRANCH;
const filePath = searchResult.file.path;
const location = `${repoUrl}/src/${branch}/${filePath}`;
result.matches.push(location);
}
return result;
}
export async function readBitbucketCloudLocations(
client: BitbucketClient,
target: string,
): Promise<Result<string>> {
const { catalogPath: requestedCatalogPath } = parseBitbucketCloudUrl(target);
const catalogPath = requestedCatalogPath
? `/${requestedCatalogPath}`
: DEFAULT_CATALOG_LOCATION;
return readBitbucketCloud(client, target).then(result => {
const matches = result.matches.map(repository => {
const branch = repository.mainbranch?.name ?? DEFAULT_BRANCH;
return `${repository.links.html.href}/src/${branch}${catalogPath}`;
});
return {
scanned: result.scanned,
matches,
};
});
}
export async function readBitbucketCloud(
client: BitbucketClient,
target: string,
@@ -238,16 +313,24 @@ export async function readBitbucketCloud(
for await (const repository of repositories) {
result.scanned++;
if (
(!projectSearchPath || projectSearchPath.test(repository.project.key)) &&
(!repoSearchPath || repoSearchPath.test(repository.slug))
) {
if (matchesPostFilters(repository, projectSearchPath, repoSearchPath)) {
result.matches.push(repository);
}
}
return result;
}
function matchesPostFilters(
repository: BitbucketRepository20,
projectSearchPath: RegExp | undefined,
repoSearchPath: RegExp | undefined,
): boolean {
return (
(!projectSearchPath || projectSearchPath.test(repository.project.key)) &&
(!repoSearchPath || repoSearchPath.test(repository.slug))
);
}
function parseUrl(urlString: string): {
projectSearchPath: RegExp;
repoSearchPath: RegExp;
@@ -263,9 +346,7 @@ function parseUrl(urlString: string): {
return {
projectSearchPath: escapeRegExp(decodeURIComponent(path[1])),
repoSearchPath: escapeRegExp(decodeURIComponent(path[3])),
catalogPath: `/${decodeURIComponent(
path.slice(4).join('/') + url.search,
)}`,
catalogPath: decodeURIComponent(path.slice(4).join('/') + url.search),
};
}
@@ -282,10 +363,11 @@ function readPathParameters(pathParts: string[]): Map<string, string> {
function parseBitbucketCloudUrl(urlString: string): {
workspacePath: string;
catalogPath: string;
catalogPath?: string;
projectSearchPath?: RegExp;
repoSearchPath?: RegExp;
queryParam?: string;
searchEnabled: boolean;
} {
const url = new URL(urlString);
const pathMap = readPathParameters(url.pathname.substr(1).split('/'));
@@ -303,8 +385,9 @@ function parseBitbucketCloudUrl(urlString: string): {
repoSearchPath: pathMap.has('repos')
? escapeRegExp(pathMap.get('repos')!)
: undefined,
catalogPath: `/${query.get('catalogPath') || ''}`,
catalogPath: query.get('catalogPath') || undefined,
queryParam: query.get('q') || undefined,
searchEnabled: query.get('search')?.toLowerCase() === 'true',
};
}
@@ -25,17 +25,21 @@ import { CatalogProcessorResult, processingResult } from '../../../api';
export type BitbucketRepositoryParser = (options: {
integration: BitbucketIntegration;
target: string;
presence?: 'optional' | 'required';
logger: Logger;
}) => AsyncIterable<CatalogProcessorResult>;
export const defaultRepositoryParser =
async function* defaultRepositoryParser(options: { target: string }) {
async function* defaultRepositoryParser(options: {
target: string;
presence?: 'optional' | 'required';
}) {
yield processingResult.location({
type: 'url',
target: options.target,
// Not all locations may actually exist, since the user defined them as a wildcard pattern.
// Thus, we emit them as optional and let the downstream processor find them while not outputting
// an error if it couldn't.
presence: 'optional',
presence: options.presence ?? 'optional',
});
};
@@ -28,6 +28,39 @@ export class BitbucketClient {
this.config = options.config;
}
async searchCode(
workspace: string,
query: string,
options?: ListOptions20,
): Promise<PagedResponse20<CodeSearchResultItem>> {
// load all fields relevant for creating refs later, but not more
const fields = [
// exclude code/content match details
'-values.content_matches',
// include/add relevant repository details
'+values.file.commit.repository.mainbranch.name',
'+values.file.commit.repository.project.key',
'+values.file.commit.repository.slug',
// remove irrelevant links
'-values.*.links',
'-values.*.*.links',
'-values.*.*.*.links',
// ...except the one we need
'+values.file.commit.repository.links.html.href',
].join(',');
return this.pagedRequest20<CodeSearchResultItem>(
`${this.config.apiBaseUrl}/workspaces/${encodeURIComponent(
workspace,
)}/search/code`,
{
...options,
fields: fields,
search_query: query,
},
);
}
async listProjects(options?: ListOptions): Promise<PagedResponse<any>> {
return this.pagedRequest(`${this.config.apiBaseUrl}/projects`, options);
}
@@ -105,6 +138,22 @@ export class BitbucketClient {
}
}
export type CodeSearchResultItem = {
type: string;
content_match_count: number;
path_matches: Array<{
text: string;
match?: boolean;
}>;
file: {
path: string;
type: string;
commit: {
repository: BitbucketRepository20;
};
};
};
export type ListOptions = {
[key: string]: number | undefined;
limit?: number | undefined;