paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 42 Packages Projects Releases Wiki Activity

Merge pull request #9924 from Bonial-International-GmbH/PJ_bitbucket_codeSearch

Browse Source 
feat(integration,bitbucket): discover catalog files using Bitbucket Cloud's code search
This commit is contained in:
Fredrik Adelöw
2022-03-08 09:18:24 +01:00
committed by GitHub
parent 5bef88248c 616f02ade2
commit 562df0fccb
7 changed files with 504 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/fair-turtles-fetch.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-catalog-backend': patch
---
support Bitbucket Cloud's code search to discover catalog files (multiple per repo, Location entities for existing files only)
docs/integrations/bitbucket/discovery.md
+29
View File
@@ -80,6 +80,18 @@ The target is composed of the following parts:
reduce the amount of API calls if you have a large workspace.
[See here for the specification](https://developer.atlassian.com/bitbucket/api/2/reference/meta/filtering)
for the query argument (will be passed as the `q` query parameter).
- (Optional) The `search=true` query argument to activate the mode utilizing code search.
- Is mutually exclusive to the `q` query argument.
- Allows providing values at `catalogPath` for finding catalog files as allowed by the `path` filter/modifier
[at Bitbucket Cloud's code search](https://confluence.atlassian.com/bitbucket/code-search-in-bitbucket-873876782.html#Search-Pathmodifier).
- `catalogPath=/catalog-info.yaml`
- `catalogPath=catalog-info.yaml` (anywhere in the repository)
- `catalogPath=/path/catalog-info.yaml`
- `catalogPath=path/catalog-info.yaml`
- `catalogPath=/path/*/catalog-info.yaml`
- `catalogPath=path/*/catalog-info.yaml`
- Supports multiple catalog files per repository depending on the `catalogPath` value.
- Registers `Location` entities for existing files only vs all matching repositories.
Examples:
@@ -95,6 +107,23 @@ Examples:
- `https://bitbucket.org/workspaces/my-workspace?catalogPath=my/nested/path/catalog.yaml`
will find all repositories in the `my-workspace` workspace and use the catalog
file at `my/nested/path/catalog.yaml`.
- `https://bitbucket.org/workspaces/my-workspace?search=true&catalogPath=/catalog.yaml`
will find all `catalog.yaml` files located in the root of repositories in the workspace `my-workspace`.
- `https://bitbucket.org/workspaces/my-workspace?search=true&catalogPath=catalog.yaml`
will find all `catalog.yaml` files located anywhere within repositories in the workspace `my-workspace`.
- `https://bitbucket.org/workspaces/my-workspace?search=true&catalogPath=/my/nested/path/catalog.yaml`
will find all `catalog.yaml` files located within the directory `/my/nested/path/` within
repositories in the workspace `my-workspace`.
- `https://bitbucket.org/workspaces/my-workspace?search=true&catalogPath=my/nested/path/catalog.yaml`
will find all `catalog.yaml` files located within the directory `my/nested/path/` located anywhere within
repositories in the workspace `my-workspace`.
- `https://bitbucket.org/workspaces/my-workspace?search=true&catalogPath=/my/*/path/catalog.yaml`
will find all `catalog.yaml` files located within a directory `path/` located within any (recursive) directory
within the directory `my/` in the root of repositories in the workspace `my-workspace`
(`/my/nested/path/catalog.yaml`, `/my/very/nested/path/catalog.yaml`, ...).
- `https://bitbucket.org/workspaces/my-workspace/projects/apis-*/repos/service-*?search=true&catalogPath=catalog.yaml`
will find all `catalog.yaml` files located anywhere within repositories starting with `service-`
in projects starting with `api-` in the workspace `my-workspace`.
## Custom repository processing
plugins/catalog-backend/api-report.md
+3
View File
@@ -104,6 +104,7 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
parser?: (options: {
integration: BitbucketIntegration;
target: string;
presence?: 'optional' | 'required';
logger: Logger;
}) => AsyncIterable<CatalogProcessorResult>;
logger: Logger;
@@ -115,6 +116,7 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
parser?: (options: {
integration: BitbucketIntegration;
target: string;
presence?: 'optional' | 'required';
logger: Logger;
}) => AsyncIterable<CatalogProcessorResult>;
logger: Logger;
@@ -134,6 +136,7 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
export type BitbucketRepositoryParser = (options: {
integration: BitbucketIntegration;
target: string;
presence?: 'optional' | 'required';
logger: Logger;
}) => AsyncIterable<CatalogProcessorResult>;
plugins/catalog-backend/src/modules/bitbucket/BitbucketDiscoveryProcessor.test.ts
+304
View File
@@ -116,6 +116,60 @@ function setupBitbucketCloudStubs(
return stubCallerFn;
}
function setupBitbucketCloudSearchStubs(
workspace: string,
repositories: Pick<BitbucketRepository20, 'slug' | 'project'>[],
catalogPath: string,
) {
const stubCallerFn = jest.fn();
function pagedResponse(values: any): PagedResponse20<any> {
return {
values: values,
page: 1,
} as PagedResponse20<any>;
}
server.use(
rest.get(
`https://api.bitbucket.org/2.0/workspaces/${workspace}/search/code`,
(req, res, ctx) => {
stubCallerFn(req);
return res(
ctx.json(
pagedResponse(
repositories.map(r => ({
type: 'code_search_result',
content_match_count: 0,
content_matches: [],
path_matches: [
catalogPath
.split('/')
.flatMap(seg => [{ text: '/' }, { text: seg, match: true }])
.slice(1),
],
file: {
commit: {
repository: {
...r,
links: {
html: {
href: `https://bitbucket.org/${workspace}/${r.slug}`,
},
},
},
},
path: catalogPath,
},
})),
),
),
);
},
),
);
return stubCallerFn;
}
describe('BitbucketDiscoveryProcessor', () => {
beforeAll(() => server.listen());
afterEach(() => server.resetHandlers());
@@ -736,6 +790,256 @@ describe('BitbucketDiscoveryProcessor', () => {
});
});
describe('handles cloud repositories using code search', () => {
const processor = BitbucketDiscoveryProcessor.fromConfig(
new ConfigReader({
integrations: {
bitbucket: [
{
host: 'bitbucket.org',
username: 'myuser',
appPassword: 'blob',
},
],
},
}),
{ logger: getVoidLogger() },
);
it('output all repositories by default', async () => {
setupBitbucketCloudSearchStubs(
'myworkspace',
[
{ project: { key: 'prj-one' }, slug: 'repository-one' },
{ project: { key: 'prj-two' }, slug: 'repository-two' },
],
'catalog-info.yaml',
);
const location: LocationSpec = {
type: 'bitbucket-discovery',
target: 'https://bitbucket.org/workspaces/myworkspace?search=true',
};
const emitter = jest.fn();
await processor.readLocation(location, false, emitter);
expect(emitter).toBeCalledTimes(2);
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.org/myworkspace/repository-one/src/master/catalog-info.yaml',
presence: 'required',
},
});
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.org/myworkspace/repository-two/src/master/catalog-info.yaml',
presence: 'required',
},
});
});
it('uses provided catalog path', async () => {
setupBitbucketCloudSearchStubs(
'myworkspace',
[
{ project: { key: 'prj-one' }, slug: 'repository-one' },
{ project: { key: 'prj-two' }, slug: 'repository-two' },
],
'my/nested/path/catalog.yaml',
);
const location: LocationSpec = {
type: 'bitbucket-discovery',
target:
'https://bitbucket.org/workspaces/myworkspace?search=true&catalogPath=my/nested/path/catalog.yaml',
};
const emitter = jest.fn();
await processor.readLocation(location, false, emitter);
expect(emitter).toBeCalledTimes(2);
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.org/myworkspace/repository-one/src/master/my/nested/path/catalog.yaml',
presence: 'required',
},
});
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.org/myworkspace/repository-two/src/master/my/nested/path/catalog.yaml',
presence: 'required',
},
});
});
it('output all repositories', async () => {
setupBitbucketCloudSearchStubs(
'myworkspace',
[
{ project: { key: 'prj-one' }, slug: 'repository-one' },
{ project: { key: 'prj-two' }, slug: 'repository-two' },
],
'catalog.yaml',
);
const location: LocationSpec = {
type: 'bitbucket-discovery',
target:
'https://bitbucket.org/workspaces/myworkspace/projects/*/repos/*?search=true&catalogPath=catalog.yaml',
};
const emitter = jest.fn();
await processor.readLocation(location, false, emitter);
expect(emitter).toBeCalledTimes(2);
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.org/myworkspace/repository-one/src/master/catalog.yaml',
presence: 'required',
},
});
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.org/myworkspace/repository-two/src/master/catalog.yaml',
presence: 'required',
},
});
});
it('output repositories with wildcards', async () => {
setupBitbucketCloudSearchStubs(
'myworkspace',
[
{ project: { key: 'prj-one' }, slug: 'repository-one' },
{ project: { key: 'prj-two' }, slug: 'repository-two' },
],
'catalog.yaml',
);
const location: LocationSpec = {
type: 'bitbucket-discovery',
target:
'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*?search=true&catalogPath=catalog.yaml',
};
const emitter = jest.fn();
await processor.readLocation(location, false, emitter);
expect(emitter).toBeCalledTimes(1);
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.org/myworkspace/repository-one/src/master/catalog.yaml',
presence: 'required',
},
});
});
it('filter unrelated repositories', async () => {
setupBitbucketCloudSearchStubs(
'myworkspace',
[
{ project: { key: 'prj-one' }, slug: 'repository-one' },
{ project: { key: 'prj-one' }, slug: 'repository-two' },
{ project: { key: 'prj-one' }, slug: 'repository-three' },
],
'catalog.yaml',
);
const location: LocationSpec = {
type: 'bitbucket-discovery',
target:
'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/repository-three?search=true&catalogPath=catalog.yaml',
};
const emitter = jest.fn();
await processor.readLocation(location, false, emitter);
expect(emitter).toBeCalledTimes(1);
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.org/myworkspace/repository-three/src/master/catalog.yaml',
presence: 'required',
},
});
});
it.each`
target
${'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*?search=true'}
${'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/*/?search=true'}
${'https://bitbucket.org/workspaces/myworkspace/projects/prj-one/repos/repository-*/?search=true'}
`("target '$target' adds default path to catalog", async ({ target }) => {
setupBitbucketCloudSearchStubs(
'myworkspace',
[{ project: { key: 'prj-one' }, slug: 'repository-one' }],
'catalog-info.yaml',
);
const location: LocationSpec = {
type: 'bitbucket-discovery',
target: target,
};
const emitter = jest.fn();
await processor.readLocation(location, false, emitter);
expect(emitter).toHaveBeenCalledTimes(1);
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://bitbucket.org/myworkspace/repository-one/src/master/catalog-info.yaml',
presence: 'required',
},
});
});
it.each`
target
${'https://bitbucket.org/test?search=true'}
`("target '$target' is rejected", async ({ target }) => {
setupBitbucketCloudSearchStubs(
'myworkspace',
[{ project: { key: 'prj-one' }, slug: 'repository-one' }],
'catalog-info.yaml',
);
const location: LocationSpec = {
type: 'bitbucket-discovery',
target: target,
};
const emitter = jest.fn();
await expect(
processor.readLocation(location, false, emitter),
).rejects.toThrow(/Failed to parse /);
});
});
describe('Custom repository parser', () => {
const processor = BitbucketDiscoveryProcessor.fromConfig(
new ConfigReader({
plugins/catalog-backend/src/modules/bitbucket/BitbucketDiscoveryProcessor.ts
+108 -25
View File
@@ -39,7 +39,6 @@ import {
const DEFAULT_BRANCH = 'master';
const DEFAULT_CATALOG_LOCATION = '/catalog-info.yaml';
const EMPTY_CATALOG_LOCATION = '/';
/** @public */
export class BitbucketDiscoveryProcessor implements CatalogProcessor {
@@ -47,6 +46,7 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
private readonly parser: (options: {
integration: BitbucketIntegration;
target: string;
presence?: 'optional' | 'required';
logger: Logger;
}) => AsyncIterable<CatalogProcessorResult>;
private readonly logger: Logger;
@@ -57,6 +57,7 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
parser?: (options: {
integration: BitbucketIntegration;
target: string;
presence?: 'optional' | 'required';
logger: Logger;
}) => AsyncIterable<CatalogProcessorResult>;
logger: Logger;
@@ -75,6 +76,7 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
parser?: (options: {
integration: BitbucketIntegration;
target: string;
presence?: 'optional' | 'required';
logger: Logger;
}) => AsyncIterable<CatalogProcessorResult>;
logger: Logger;
@@ -136,19 +138,17 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
): Promise<ResultSummary> {
const { client, location, integration, emit } = options;
const { catalogPath: requestedCatalogPath } = parseBitbucketCloudUrl(
location.target,
);
const catalogPath =
requestedCatalogPath === EMPTY_CATALOG_LOCATION
? DEFAULT_CATALOG_LOCATION
: requestedCatalogPath;
const result = await readBitbucketCloud(client, location.target);
for (const repository of result.matches) {
const mainbranch = repository.mainbranch?.name ?? DEFAULT_BRANCH;
const { searchEnabled } = parseBitbucketCloudUrl(location.target);
const result = searchEnabled
? await searchBitbucketCloudLocations(client, location.target)
: await readBitbucketCloudLocations(client, location.target);
for (const locationTarget of result.matches) {
for await (const entity of this.parser({
integration,
target: `${repository.links.html.href}/src/${mainbranch}${catalogPath}`,
target: locationTarget,
presence: searchEnabled ? 'required' : 'optional',
logger: this.logger,
})) {
emit(entity);
@@ -165,10 +165,10 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
): Promise<ResultSummary> {
const { client, location, integration, emit } = options;
const { catalogPath: requestedCatalogPath } = parseUrl(location.target);
const catalogPath =
requestedCatalogPath === EMPTY_CATALOG_LOCATION
? DEFAULT_CATALOG_LOCATION
: requestedCatalogPath;
const catalogPath = requestedCatalogPath
? `/${requestedCatalogPath}`
: DEFAULT_CATALOG_LOCATION;
const result = await readBitbucketOrg(client, location.target);
for (const repository of result.matches) {
for await (const entity of this.parser({
@@ -214,6 +214,81 @@ export async function readBitbucketOrg(
return result;
}
export async function searchBitbucketCloudLocations(
client: BitbucketClient,
target: string,
): Promise<Result<string>> {
const {
workspacePath,
catalogPath: requestedCatalogPath,
projectSearchPath,
repoSearchPath,
} = parseBitbucketCloudUrl(target);
const result: Result<string> = {
scanned: 0,
matches: [],
};
const catalogPath = requestedCatalogPath
? requestedCatalogPath
: DEFAULT_CATALOG_LOCATION;
const catalogFilename = catalogPath.substring(
catalogPath.lastIndexOf('/') + 1,
);
const searchResults = paginated20(options =>
client.searchCode(
workspacePath,
`"${catalogFilename}" path:${catalogPath}`,
options,
),
);
for await (const searchResult of searchResults) {
// not a file match, but a code match
if (searchResult.path_matches.length === 0) {
continue;
}
const repository = searchResult.file.commit.repository;
if (!matchesPostFilters(repository, projectSearchPath, repoSearchPath)) {
continue;
}
const repoUrl = repository.links.html.href;
const branch = repository.mainbranch?.name ?? DEFAULT_BRANCH;
const filePath = searchResult.file.path;
const location = `${repoUrl}/src/${branch}/${filePath}`;
result.matches.push(location);
}
return result;
}
export async function readBitbucketCloudLocations(
client: BitbucketClient,
target: string,
): Promise<Result<string>> {
const { catalogPath: requestedCatalogPath } = parseBitbucketCloudUrl(target);
const catalogPath = requestedCatalogPath
? `/${requestedCatalogPath}`
: DEFAULT_CATALOG_LOCATION;
return readBitbucketCloud(client, target).then(result => {
const matches = result.matches.map(repository => {
const branch = repository.mainbranch?.name ?? DEFAULT_BRANCH;
return `${repository.links.html.href}/src/${branch}${catalogPath}`;
});
return {
scanned: result.scanned,
matches,
};
});
}
export async function readBitbucketCloud(
client: BitbucketClient,
target: string,
@@ -238,16 +313,24 @@ export async function readBitbucketCloud(
for await (const repository of repositories) {
result.scanned++;
if (
(!projectSearchPath || projectSearchPath.test(repository.project.key)) &&
(!repoSearchPath || repoSearchPath.test(repository.slug))
) {
if (matchesPostFilters(repository, projectSearchPath, repoSearchPath)) {
result.matches.push(repository);
}
}
return result;
}
function matchesPostFilters(
repository: BitbucketRepository20,
projectSearchPath: RegExp | undefined,
repoSearchPath: RegExp | undefined,
): boolean {
return (
(!projectSearchPath || projectSearchPath.test(repository.project.key)) &&
(!repoSearchPath || repoSearchPath.test(repository.slug))
);
}
function parseUrl(urlString: string): {
projectSearchPath: RegExp;
repoSearchPath: RegExp;
@@ -263,9 +346,7 @@ function parseUrl(urlString: string): {
return {
projectSearchPath: escapeRegExp(decodeURIComponent(path[1])),
repoSearchPath: escapeRegExp(decodeURIComponent(path[3])),
catalogPath: `/${decodeURIComponent(
path.slice(4).join('/') + url.search,
)}`,
catalogPath: decodeURIComponent(path.slice(4).join('/') + url.search),
};
}
@@ -282,10 +363,11 @@ function readPathParameters(pathParts: string[]): Map<string, string> {
function parseBitbucketCloudUrl(urlString: string): {
workspacePath: string;
catalogPath: string;
catalogPath?: string;
projectSearchPath?: RegExp;
repoSearchPath?: RegExp;
queryParam?: string;
searchEnabled: boolean;
} {
const url = new URL(urlString);
const pathMap = readPathParameters(url.pathname.substr(1).split('/'));
@@ -303,8 +385,9 @@ function parseBitbucketCloudUrl(urlString: string): {
repoSearchPath: pathMap.has('repos')
? escapeRegExp(pathMap.get('repos')!)
: undefined,
catalogPath: `/${query.get('catalogPath') || ''}`,
catalogPath: query.get('catalogPath') || undefined,
queryParam: query.get('q') || undefined,
searchEnabled: query.get('search')?.toLowerCase() === 'true',
};
}
plugins/catalog-backend/src/modules/bitbucket/lib/BitbucketRepositoryParser.ts
+6 -2
View File
@@ -25,17 +25,21 @@ import { CatalogProcessorResult, processingResult } from '../../../api';
export type BitbucketRepositoryParser = (options: {
integration: BitbucketIntegration;
target: string;
presence?: 'optional' | 'required';
logger: Logger;
}) => AsyncIterable<CatalogProcessorResult>;
export const defaultRepositoryParser =
async function* defaultRepositoryParser(options: { target: string }) {
async function* defaultRepositoryParser(options: {
target: string;
presence?: 'optional' | 'required';
}) {
yield processingResult.location({
type: 'url',
target: options.target,
// Not all locations may actually exist, since the user defined them as a wildcard pattern.
// Thus, we emit them as optional and let the downstream processor find them while not outputting
// an error if it couldn't.
presence: 'optional',
presence: options.presence ?? 'optional',
});
};
plugins/catalog-backend/src/modules/bitbucket/lib/client.ts
+49
View File
@@ -28,6 +28,39 @@ export class BitbucketClient {
this.config = options.config;
}
async searchCode(
workspace: string,
query: string,
options?: ListOptions20,
): Promise<PagedResponse20<CodeSearchResultItem>> {
// load all fields relevant for creating refs later, but not more
const fields = [
// exclude code/content match details
'-values.content_matches',
// include/add relevant repository details
'+values.file.commit.repository.mainbranch.name',
'+values.file.commit.repository.project.key',
'+values.file.commit.repository.slug',
// remove irrelevant links
'-values.*.links',
'-values.*.*.links',
'-values.*.*.*.links',
// ...except the one we need
'+values.file.commit.repository.links.html.href',
].join(',');
return this.pagedRequest20<CodeSearchResultItem>(
`${this.config.apiBaseUrl}/workspaces/${encodeURIComponent(
workspace,
)}/search/code`,
{
...options,
fields: fields,
search_query: query,
},
);
}
async listProjects(options?: ListOptions): Promise<PagedResponse<any>> {
return this.pagedRequest(`${this.config.apiBaseUrl}/projects`, options);
}
@@ -105,6 +138,22 @@ export class BitbucketClient {
}
}
export type CodeSearchResultItem = {
type: string;
content_match_count: number;
path_matches: Array<{
text: string;
match?: boolean;
}>;
file: {
path: string;
type: string;
commit: {
repository: BitbucketRepository20;
};
};
};
export type ListOptions = {
[key: string]: number | undefined;
limit?: number | undefined;