paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 46 Packages Projects Releases Wiki Activity

Merge pull request #22996 from RagnarHal/azure-make-branches-work

Browse Source 
fix(catalog-backend-module-azure): Fixed issue where specifying a branch for discovery did not work
This commit is contained in:
Fredrik Adelöw
2024-02-19 17:41:19 +01:00
committed by GitHub
parent 5d8eb6cd71 49c759e1c1
commit d94ebb80e8
7 changed files with 200 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/real-keys-juggle.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-catalog-backend-module-azure': patch
---
Fixed issue where specifying a branch for discovery did not work
docs/integrations/azure/discovery.md
+7 -1
View File
@@ -93,6 +93,8 @@ _Note:_
5. In the window that appears, enter the name of the branch you want to add and click "Add".
6. The added branch will now appear in the "Searchable branches" list.
It may take some time before the branch is indexed and searchable.
As this provider is not one of the default providers, you will first need to install
the Azure catalog plugin:
@@ -160,11 +162,14 @@ catalog:
# Or use a custom file format and location
- type: azure-discovery
target: https://dev.azure.com/myorg/myproject/_git/*?path=/src/*/catalog-info.yaml
# And optionally provide a specific branch name using the version parameter
- type: azure-discovery
target: https://dev.azure.com/myorg/myproject/_git/*?path=/catalog-info.yaml&version=GBtopic/catalog-info
```
Note the `azure-discovery` type, as this is not a regular `url` processor.
When using a custom pattern, the target is composed of five parts:
When using a custom pattern, the target is composed of these parts:
- The base instance URL, `https://dev.azure.com` in this case
- The organization name which is required, `myorg` in this case
@@ -175,3 +180,4 @@ When using a custom pattern, the target is composed of five parts:
- The path within each repository to find the catalog YAML file. This will
usually be `/catalog-info.yaml`, `/src/*/catalog-info.yaml` or a similar
variation for catalog files stored in the root directory of each repository.
- The repository branch to scan which is optional, `topic/catalog-info` in this case. If omitted, the repo's default branch will be scanned. The `GB` prefix is required, as this is how Azure DevOps identifies the version as a branch.
plugins/catalog-backend-module-azure/src/lib/azure.test.ts
+65
View File
@@ -89,6 +89,7 @@ describe('azure', () => {
'engineering',
'',
'/catalog-info.yaml',
'',
),
).resolves.toEqual([]);
});
@@ -154,6 +155,7 @@ describe('azure', () => {
'engineering',
'',
'/catalog-info.yaml',
'',
),
).resolves.toEqual(response.results);
});
@@ -210,6 +212,67 @@ describe('azure', () => {
'engineering',
'backstage',
'/catalog-info.yaml',
'',
),
).resolves.toEqual(response.results);
});
it('searches in specific branch if parameter is set', async () => {
const response: CodeSearchResponse = {
count: 1,
results: [
{
fileName: 'catalog-info.yaml',
path: '/catalog-info.yaml',
project: {
name: '*',
},
repository: {
name: 'backstage',
},
},
],
};
server.use(
rest.post(
`https://almsearch.dev.azure.com/shopify/_apis/search/codesearchresults`,
(req, res, ctx) => {
expect(req.headers.get('Authorization')).toBe('Basic OkFCQw==');
expect(req.body).toEqual({
searchText:
'path:/catalog-info.yaml repo:backstage proj:engineering',
$orderBy: [
{
field: 'path',
sortOrder: 'ASC',
},
],
$skip: 0,
$top: 1000,
filters: {
Branch: ['topic/catalog-info'],
},
});
return res(ctx.json(response));
},
),
);
const { credentialsProvider, azureConfig } = createFixture(
'dev.azure.com',
'ABC',
);
await expect(
codeSearch(
credentialsProvider,
azureConfig,
'shopify',
'engineering',
'backstage',
'/catalog-info.yaml',
'topic/catalog-info',
),
).resolves.toEqual(response.results);
});
@@ -265,6 +328,7 @@ describe('azure', () => {
'engineering',
'',
'/catalog-info.yaml',
'',
),
).resolves.toEqual(response.results);
});
@@ -324,6 +388,7 @@ describe('azure', () => {
'engineering',
'backstage',
'/catalog-info.yaml',
'',
),
).resolves.toHaveLength(totalCount);
});
plugins/catalog-backend-module-azure/src/lib/azure.ts
+28 -11
View File
@@ -37,6 +37,16 @@ export interface CodeSearchResultItem {
branch?: string;
}
interface CodeSearchRequest {
searchText: string;
$orderBy: Array<{ field: string; sortOrder: string }>;
$skip: number;
$top: number;
filters?: {
Branch: string[];
};
}
const isCloud = (host: string) => host === 'dev.azure.com';
const PAGE_SIZE = 1000;
@@ -48,6 +58,7 @@ export async function codeSearch(
project: string,
repo: string,
path: string,
branch: string,
): Promise<CodeSearchResultItem[]> {
const searchBaseUrl = isCloud(azureConfig.host)
? 'https://almsearch.dev.azure.com'
@@ -62,23 +73,29 @@ export async function codeSearch(
url: `https://${azureConfig.host}/${org}`,
});
const searchRequestBody: CodeSearchRequest = {
searchText: `path:${path} repo:${repo || '*'} proj:${project || '*'}`,
$orderBy: [
{
field: 'path',
sortOrder: 'ASC',
},
],
$skip: items.length,
$top: PAGE_SIZE,
};
if (branch) {
searchRequestBody.filters = { Branch: [branch] };
}
const response = await fetch(searchUrl, {
headers: {
...credentials?.headers,
'Content-Type': 'application/json',
},
method: 'POST',
body: JSON.stringify({
searchText: `path:${path} repo:${repo || '*'} proj:${project || '*'}`,
$orderBy: [
{
field: 'path',
sortOrder: 'ASC',
},
],
$skip: items.length,
$top: PAGE_SIZE,
}),
body: JSON.stringify(searchRequestBody),
});
if (response.status !== 200) {
plugins/catalog-backend-module-azure/src/processors/AzureDevOpsDiscoveryProcessor.test.ts
+76
View File
@@ -35,6 +35,7 @@ describe('AzureDevOpsDiscoveryProcessor', () => {
project: 'my-proj',
repo: '',
catalogPath: '/catalog-info.yaml',
branch: '',
});
expect(
@@ -47,6 +48,7 @@ describe('AzureDevOpsDiscoveryProcessor', () => {
project: 'engineering',
repo: 'backstage',
catalogPath: '/catalog.yaml',
branch: '',
});
expect(
@@ -59,6 +61,33 @@ describe('AzureDevOpsDiscoveryProcessor', () => {
project: 'engineering',
repo: 'backstage',
catalogPath: '/src/*/catalog.yaml',
branch: '',
});
expect(
parseUrl(
'https://azuredevops.mycompany.com/spotify/engineering/_git/backstage?path=/src/*/catalog.yaml&version=GBtopic/catalog-info',
),
).toEqual({
baseUrl: 'https://azuredevops.mycompany.com',
org: 'spotify',
project: 'engineering',
repo: 'backstage',
catalogPath: '/src/*/catalog.yaml',
branch: 'topic/catalog-info',
});
expect(
parseUrl(
'https://azuredevops.mycompany.com/spotify/engineering/_git/backstage?path=/src/*/catalog.yaml&version=GBtopic%2Fcatalog-info',
),
).toEqual({
baseUrl: 'https://azuredevops.mycompany.com',
org: 'spotify',
project: 'engineering',
repo: 'backstage',
catalogPath: '/src/*/catalog.yaml',
branch: 'topic/catalog-info',
});
});
@@ -164,6 +193,7 @@ describe('AzureDevOpsDiscoveryProcessor', () => {
'engineering',
'',
'/catalog-info.yaml',
'',
);
expect(emitter).toHaveBeenCalledTimes(2);
expect(emitter).toHaveBeenCalledWith({
@@ -214,6 +244,7 @@ describe('AzureDevOpsDiscoveryProcessor', () => {
'engineering',
'backstage',
'/catalog-info.yaml',
'',
);
expect(emitter).toHaveBeenCalledTimes(1);
expect(emitter).toHaveBeenCalledWith({
@@ -227,6 +258,49 @@ describe('AzureDevOpsDiscoveryProcessor', () => {
});
});
it('output locations with branch if specified in target', async () => {
const location: LocationSpec = {
type: 'azure-discovery',
target:
'https://dev.azure.com/shopify/engineering/_git/backstage?path=/catalog-info.yaml&version=GBtopic/catalog-info',
};
mockCodeSearch.mockResolvedValueOnce([
{
fileName: 'catalog-info.yaml',
path: '/catalog-info.yaml',
repository: {
name: 'backstage',
},
project: {
name: '*',
},
},
]);
const emitter = jest.fn();
await processor.readLocation(location, false, emitter);
expect(mockCodeSearch).toHaveBeenCalledWith(
expect.anything(),
{ host: 'dev.azure.com' },
'shopify',
'engineering',
'backstage',
'/catalog-info.yaml',
'topic/catalog-info',
);
expect(emitter).toHaveBeenCalledTimes(1);
expect(emitter).toHaveBeenCalledWith({
type: 'location',
location: {
type: 'url',
target:
'https://dev.azure.com/shopify/engineering/_git/backstage?path=/catalog-info.yaml&version=GBtopic/catalog-info',
presence: 'optional',
},
});
});
it('output single locations with different file name from code search', async () => {
const location: LocationSpec = {
type: 'azure-discovery',
@@ -256,6 +330,7 @@ describe('AzureDevOpsDiscoveryProcessor', () => {
'engineering',
'',
'/src/*/catalog.yaml',
'',
);
expect(emitter).toHaveBeenCalledTimes(1);
expect(emitter).toHaveBeenCalledWith({
@@ -286,6 +361,7 @@ describe('AzureDevOpsDiscoveryProcessor', () => {
'engineering',
'backstage',
'/catalog-info.yaml',
'',
);
expect(emitter).not.toHaveBeenCalled();
});
plugins/catalog-backend-module-azure/src/processors/AzureDevOpsDiscoveryProcessor.ts
+18 -2
View File
@@ -92,7 +92,7 @@ export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor {
);
}
const { baseUrl, org, project, repo, catalogPath } = parseUrl(
const { baseUrl, org, project, repo, catalogPath, branch } = parseUrl(
location.target,
);
this.logger.info(
@@ -106,6 +106,7 @@ export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor {
project,
repo,
catalogPath,
branch,
);
this.logger.debug(
@@ -113,10 +114,16 @@ export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor {
);
for (const file of files) {
let target = `${baseUrl}/${org}/${project}/_git/${file.repository.name}?path=${file.path}`;
if (branch) {
target += `&version=GB${branch}`;
}
emit(
processingResult.location({
type: 'url',
target: `${baseUrl}/${org}/${project}/_git/${file.repository.name}?path=${file.path}`,
target,
// Not all locations may actually exist, since the user defined them as a wildcard pattern.
// Thus, we emit them as optional and let the downstream processor find them while not outputting
// an error if it couldn't.
@@ -138,11 +145,18 @@ export function parseUrl(urlString: string): {
project: string;
repo: string;
catalogPath: string;
branch: string;
} {
const url = new URL(urlString);
const path = url.pathname.slice(1).split('/');
const catalogPath = url.searchParams.get('path') || '/catalog-info.yaml';
let branch = url.searchParams.get('version') || '';
if (branch.startsWith('GB')) {
// DevOps prefixes branch names with 'GB' in URLs
branch = branch.slice(2);
}
if (path.length === 2 && path[0].length && path[1].length) {
return {
@@ -151,6 +165,7 @@ export function parseUrl(urlString: string): {
project: decodeURIComponent(path[1]),
repo: '',
catalogPath,
branch,
};
} else if (
path.length === 4 &&
@@ -165,6 +180,7 @@ export function parseUrl(urlString: string): {
project: decodeURIComponent(path[1]),
repo: decodeURIComponent(path[3]),
catalogPath,
branch,
};
}
plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.ts
+1
View File
@@ -158,6 +158,7 @@ export class AzureDevOpsEntityProvider implements EntityProvider {
this.config.project,
this.config.repository,
this.config.path,
this.config.branch || '',
);
logger.info(`Discovered ${files.length} catalog files`);