diff --git a/.changeset/strong-seahorses-scream.md b/.changeset/strong-seahorses-scream.md index 445ed65c81..bbd1d44378 100644 --- a/.changeset/strong-seahorses-scream.md +++ b/.changeset/strong-seahorses-scream.md @@ -1,5 +1,5 @@ --- -'@backstage/plugin-catalog-backend': minor +'@backstage/plugin-catalog-backend': patch --- Added Azure DevOps discovery processor diff --git a/microsite/sidebars.json b/microsite/sidebars.json index ec5c5d8d68..bc026ec196 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -124,7 +124,11 @@ { "type": "subcategory", "label": "Azure", - "ids": ["integrations/azure/locations", "integrations/azure/org"] + "ids": [ + "integrations/azure/locations", + "integrations/azure/discovery", + "integrations/azure/org" + ] }, { "type": "subcategory", diff --git a/mkdocs.yml b/mkdocs.yml index 8c7114abbc..08107f255d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -88,6 +88,7 @@ nav: - Discovery: 'integrations/aws-s3/discovery.md' - Azure: - Locations: 'integrations/azure/locations.md' + - Discovery: 'integrations/azure/discovery.md' - Org Data: 'integrations/azure/org.md' - Bitbucket: - Locations: 'integrations/bitbucket/locations.md' diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts index b86e3f9c82..58ee2155e9 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts @@ -85,7 +85,9 @@ export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { catalogPath, ); - this.logger.info(`Found ${files.length} files in Azure DevOps.`); + this.logger.debug( + `Found ${files.length} files in Azure DevOps from ${location.target}.`, + ); for (const file of files) { emit( diff --git a/plugins/catalog-backend/src/ingestion/processors/azure/azure.test.ts b/plugins/catalog-backend/src/ingestion/processors/azure/azure.test.ts index d9dfe5d1f4..67cc120bab 100644 --- a/plugins/catalog-backend/src/ingestion/processors/azure/azure.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/azure/azure.test.ts @@ -34,6 +34,7 @@ describe('azure', () => { expect(req.headers.get('Authorization')).toBe('Basic OkFCQw=='); expect(req.body).toEqual({ searchText: 'path:/catalog-info.yaml repo:*', + $skip: 0, $top: 1000, }); return res(ctx.json(response)); @@ -81,6 +82,7 @@ describe('azure', () => { expect(req.headers.get('Authorization')).toBe('Basic OkFCQw=='); expect(req.body).toEqual({ searchText: 'path:/catalog-info.yaml repo:*', + $skip: 0, $top: 1000, }); return res(ctx.json(response)); @@ -120,6 +122,7 @@ describe('azure', () => { expect(req.headers.get('Authorization')).toBe('Basic OkFCQw=='); expect(req.body).toEqual({ searchText: 'path:/catalog-info.yaml repo:backstage', + $skip: 0, $top: 1000, }); return res(ctx.json(response)); @@ -137,4 +140,93 @@ describe('azure', () => { ), ).resolves.toEqual(response.results); }); + + it('can search using onpremise api', async () => { + const response: CodeSearchResponse = { + count: 1, + results: [ + { + fileName: 'catalog-info.yaml', + path: '/catalog-info.yaml', + repository: { + name: 'backstage', + }, + }, + ], + }; + + server.use( + rest.post( + `https://azuredevops.mycompany.com/shopify/engineering/_apis/search/codesearchresults`, + (req, res, ctx) => { + expect(req.headers.get('Authorization')).toBe('Basic OkFCQw=='); + expect(req.body).toEqual({ + searchText: 'path:/catalog-info.yaml repo:*', + $skip: 0, + $top: 1000, + }); + return res(ctx.json(response)); + }, + ), + ); + + await expect( + codeSearch( + { host: 'azuredevops.mycompany.com', token: 'ABC' }, + 'shopify', + 'engineering', + '', + '/catalog-info.yaml', + ), + ).resolves.toEqual(response.results); + }); + + it('searches multiple pages if response contains many items', async () => { + const totalCount = 2401; + const generateItems = (count: number) => { + return Array.from(Array(count).keys()).map(_ => ({ + fileName: 'catalog-info.yaml', + path: '/catalog-info.yaml', + repository: { + name: 'backstage', + }, + })); + }; + + server.use( + rest.post( + `https://almsearch.dev.azure.com/shopify/engineering/_apis/search/codesearchresults`, + (req, res, ctx) => { + expect(req.headers.get('Authorization')).toBe('Basic OkFCQw=='); + expect(req.body).toMatchObject({ + searchText: 'path:/catalog-info.yaml repo:backstage', + $top: 1000, + }); + + const body = req.body as { $skip: number; $top: number }; + const countItemsToReturn = + body.$top + body.$skip > totalCount + ? totalCount - body.$skip + : body.$top; + + return res( + ctx.json({ + count: totalCount, + results: generateItems(countItemsToReturn), + }), + ); + }, + ), + ); + + await expect( + codeSearch( + { host: 'dev.azure.com', token: 'ABC' }, + 'shopify', + 'engineering', + 'backstage', + '/catalog-info.yaml', + ), + ).resolves.toHaveLength(totalCount); + }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/azure/azure.ts b/plugins/catalog-backend/src/ingestion/processors/azure/azure.ts index ec1b9ba0fb..0c7b17483a 100644 --- a/plugins/catalog-backend/src/ingestion/processors/azure/azure.ts +++ b/plugins/catalog-backend/src/ingestion/processors/azure/azure.ts @@ -33,6 +33,9 @@ export interface CodeSearchResultItem { }; } +const isCloud = (host: string) => host === 'dev.azure.com'; +const PAGE_SIZE = 1000; + // codeSearch returns all files that matches the given search path. export async function codeSearch( azureConfig: AzureIntegrationConfig, @@ -41,27 +44,37 @@ export async function codeSearch( repo: string, path: string, ): Promise { - // TODO: What's the search URL for onpremises DevOps? - const searchUrl = `https://almsearch.dev.azure.com/${org}/${project}/_apis/search/codesearchresults?api-version=6.0-preview.1`; - const opts = getAzureRequestOptions(azureConfig); - const response = await fetch(searchUrl, { - method: 'POST', - headers: { - ...opts.headers, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - searchText: `path:${path} repo:${repo || '*'}`, - $top: 1000, - }), - }); + const searchBaseUrl = isCloud(azureConfig.host) + ? 'https://almsearch.dev.azure.com' + : `https://${azureConfig.host}`; + const searchUrl = `${searchBaseUrl}/${org}/${project}/_apis/search/codesearchresults?api-version=6.0-preview.1`; - if (response.status !== 200) { - throw new Error( - `Azure DevOps search failed with response status ${response.status}`, - ); - } + let items: CodeSearchResultItem[] = []; + let hasMorePages = true; - const responseBody: CodeSearchResponse = await response.json(); - return responseBody.results; + do { + const response = await fetch(searchUrl, { + ...getAzureRequestOptions(azureConfig, { + 'Content-Type': 'application/json', + }), + method: 'POST', + body: JSON.stringify({ + searchText: `path:${path} repo:${repo || '*'}`, + $skip: items.length, + $top: PAGE_SIZE, + }), + }); + + if (response.status !== 200) { + throw new Error( + `Azure DevOps search failed with response status ${response.status}`, + ); + } + + const body: CodeSearchResponse = await response.json(); + items = [...items, ...body.results]; + hasMorePages = body.count > items.length; + } while (hasMorePages); + + return items; }