diff --git a/.changeset/strong-seahorses-scream.md b/.changeset/strong-seahorses-scream.md new file mode 100644 index 0000000000..bbd1d44378 --- /dev/null +++ b/.changeset/strong-seahorses-scream.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Added Azure DevOps discovery processor diff --git a/docs/integrations/azure/discovery.md b/docs/integrations/azure/discovery.md new file mode 100644 index 0000000000..ad452f47af --- /dev/null +++ b/docs/integrations/azure/discovery.md @@ -0,0 +1,49 @@ +--- +id: discovery +title: Azure DevOps Discovery +sidebar_label: Discovery +# prettier-ignore +description: Automatically discovering catalog entities from repositories in an Azure DevOps organization +--- + +The Azure DevOps integration has a special discovery processor for discovering +catalog entities within an Azure DevOps. The processor will crawl the Azure +DevOps organization and register entities matching the configured path. This can +be useful as an alternative to static locations or manually adding things to the +catalog. + +To use the discovery processor, you'll need a GitHub integration +[set up](locations.md) with a `AZURE_TOKEN`. Then you can add a location target +to the catalog configuration: + +```yaml +catalog: + locations: + # Scan all repositories for a catalog-info.yaml in the root of the default branch + - type: azure-discovery + target: https://dev.azure.com/myorg/myproject + # Or use a custom pattern for a subset of all repositories with default repository + - type: azure-discovery + target: https://dev.azure.com/myorg/myproject/_git/service-* + # Or use a custom file format and location + - type: azure-discovery + target: https://dev.azure.com/myorg/myproject/_git/*?path=/src/*/catalog-info.yaml +``` + +Note the `azure-discovery` type, as this is not a regular `url` processor. + +When using a custom pattern, the target is composed of five parts: + +- The base instance URL, `https://dev.azure.com` in this case +- The organization name which is required, `myorg` in this case +- The project name which is required, `myproject` in this case +- The repository blob to scan, which accepts \* wildcard tokens and must be + added after `_git/`. This can simply be `*` to scan all repositories in the + project. +- The path within each repository to find the catalog YAML file. This will + usually be `/catalog-info.yaml`, `/src/*/catalog-info.yaml` or a similar + variation for catalog files stored in the root directory of each repository. + +_Note:_ the path parameter follows the same rules as the search on Azure DevOps +web interface. For more details visit the +[official search documentation](https://docs.microsoft.com/en-us/azure/devops/project/search/get-started-search?view=azure-devops) diff --git a/microsite/sidebars.json b/microsite/sidebars.json index a24d5897ab..89eb282d14 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -125,7 +125,11 @@ { "type": "subcategory", "label": "Azure", - "ids": ["integrations/azure/locations", "integrations/azure/org"] + "ids": [ + "integrations/azure/locations", + "integrations/azure/discovery", + "integrations/azure/org" + ] }, { "type": "subcategory", diff --git a/mkdocs.yml b/mkdocs.yml index 48df25afef..2eed377819 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -88,6 +88,7 @@ nav: - Discovery: 'integrations/aws-s3/discovery.md' - Azure: - Locations: 'integrations/azure/locations.md' + - Discovery: 'integrations/azure/discovery.md' - Org Data: 'integrations/azure/org.md' - Bitbucket: - Locations: 'integrations/bitbucket/locations.md' diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 5008cf02f1..7363a803c8 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -174,6 +174,26 @@ export class AwsS3DiscoveryProcessor implements CatalogProcessor { ): Promise; } +// Warning: (ae-missing-release-tag) "AzureDevOpsDiscoveryProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { + constructor(options: { integrations: ScmIntegrations; logger: Logger_2 }); + // (undocumented) + static fromConfig( + config: Config, + options: { + logger: Logger_2; + }, + ): AzureDevOpsDiscoveryProcessor; + // (undocumented) + readLocation( + location: LocationSpec, + _optional: boolean, + emit: CatalogProcessorEmit, + ): Promise; +} + // Warning: (ae-missing-release-tag) "BitbucketDiscoveryProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts new file mode 100644 index 0000000000..ab053e8748 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts @@ -0,0 +1,277 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { LocationSpec } from '@backstage/catalog-model'; +import { ConfigReader } from '@backstage/config'; +import { codeSearch } from './azure'; +import { + AzureDevOpsDiscoveryProcessor, + parseUrl, +} from './AzureDevOpsDiscoveryProcessor'; + +jest.mock('./azure'); +const mockCodeSearch = codeSearch as jest.MockedFunction; + +describe('AzureDevOpsDiscoveryProcessor', () => { + describe('parseUrl', () => { + it('parses well formed URLs', () => { + expect(parseUrl('https://dev.azure.com/my-org/my-proj')).toEqual({ + baseUrl: 'https://dev.azure.com', + org: 'my-org', + project: 'my-proj', + repo: '', + catalogPath: '/catalog-info.yaml', + }); + + expect( + parseUrl( + 'https://dev.azure.com/spotify/engineering/_git/backstage?path=/catalog.yaml', + ), + ).toEqual({ + baseUrl: 'https://dev.azure.com', + org: 'spotify', + project: 'engineering', + repo: 'backstage', + catalogPath: '/catalog.yaml', + }); + + expect( + parseUrl( + 'https://azuredevops.mycompany.com/spotify/engineering/_git/backstage?path=/src/*/catalog.yaml', + ), + ).toEqual({ + baseUrl: 'https://azuredevops.mycompany.com', + org: 'spotify', + project: 'engineering', + repo: 'backstage', + catalogPath: '/src/*/catalog.yaml', + }); + }); + + it('throws on incorrectly formed URLs', () => { + expect(() => parseUrl('https://dev.azure.com')).toThrow(); + expect(() => parseUrl('https://dev.azure.com//')).toThrow(); + expect(() => parseUrl('https://dev.azure.com//foo')).toThrow(); + }); + }); + + describe('reject unrelated entries', () => { + it('rejects unknown types', async () => { + const processor = AzureDevOpsDiscoveryProcessor.fromConfig( + new ConfigReader({ + integrations: { + azure: [{ host: 'dev.azure.com', token: 'blob' }], + }, + }), + { logger: getVoidLogger() }, + ); + const location: LocationSpec = { + type: 'not-azure-discovery', + target: 'https://dev.azure.com', + }; + await expect( + processor.readLocation(location, false, () => {}), + ).resolves.toBeFalsy(); + }); + }); + + it('rejects unknown targets', async () => { + const processor = AzureDevOpsDiscoveryProcessor.fromConfig( + new ConfigReader({ + integrations: { + github: [ + { host: 'dev.azure.com', token: 'blob' }, + { host: 'azure.myorg.com', token: 'blob' }, + ], + }, + }), + { logger: getVoidLogger() }, + ); + const location: LocationSpec = { + type: 'azure-discovery', + target: 'https://not.azure.com/org/project', + }; + await expect( + processor.readLocation(location, false, () => {}), + ).rejects.toThrow( + /There is no Azure integration that matches https:\/\/not.azure.com\/org\/project. Please add a configuration entry for it under integrations.azure/, + ); + }); + + describe('handles repositories', () => { + const processor = AzureDevOpsDiscoveryProcessor.fromConfig( + new ConfigReader({ + integrations: { + github: [{ host: 'dev.azure.com', token: 'blob' }], + }, + }), + { logger: getVoidLogger() }, + ); + + beforeEach(() => { + mockCodeSearch.mockClear(); + }); + + it('output all locations found on from code search', async () => { + const location: LocationSpec = { + type: 'azure-discovery', + target: 'https://dev.azure.com/shopify/engineering', + }; + mockCodeSearch.mockResolvedValueOnce([ + { + fileName: 'catalog-info.yaml', + path: '/catalog-info.yaml', + repository: { + name: 'backstage', + }, + }, + { + fileName: 'catalog-info.yaml', + path: '/src/catalog-info.yaml', + repository: { + name: 'ios-app', + }, + }, + ]); + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(mockCodeSearch).toHaveBeenCalledWith( + { host: 'dev.azure.com' }, + 'shopify', + 'engineering', + '', + '/catalog-info.yaml', + ); + expect(emitter).toHaveBeenCalledTimes(2); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://dev.azure.com/shopify/engineering/_git/backstage?path=/catalog-info.yaml', + }, + optional: true, + }); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://dev.azure.com/shopify/engineering/_git/ios-app?path=/src/catalog-info.yaml', + }, + optional: true, + }); + }); + + it('output single locations from code search', async () => { + const location: LocationSpec = { + type: 'azure-discovery', + target: 'https://dev.azure.com/shopify/engineering/_git/backstage', + }; + mockCodeSearch.mockResolvedValueOnce([ + { + fileName: 'catalog-info.yaml', + path: '/catalog-info.yaml', + repository: { + name: 'backstage', + }, + }, + ]); + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(mockCodeSearch).toHaveBeenCalledWith( + { host: 'dev.azure.com' }, + 'shopify', + 'engineering', + 'backstage', + '/catalog-info.yaml', + ); + expect(emitter).toHaveBeenCalledTimes(1); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://dev.azure.com/shopify/engineering/_git/backstage?path=/catalog-info.yaml', + }, + optional: true, + }); + }); + + it('output single locations with different file name from code search', async () => { + const location: LocationSpec = { + type: 'azure-discovery', + target: + 'https://dev.azure.com/shopify/engineering?path=/src/*/catalog.yaml', + }; + mockCodeSearch.mockResolvedValueOnce([ + { + fileName: 'catalog.yaml', + path: '/src/main/catalog.yaml', + repository: { + name: 'backstage', + }, + }, + ]); + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(mockCodeSearch).toHaveBeenCalledWith( + { host: 'dev.azure.com' }, + 'shopify', + 'engineering', + '', + '/src/*/catalog.yaml', + ); + expect(emitter).toHaveBeenCalledTimes(1); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://dev.azure.com/shopify/engineering/_git/backstage?path=/src/main/catalog.yaml', + }, + optional: true, + }); + }); + + it('output nothing when code search does not find anything', async () => { + const location: LocationSpec = { + type: 'azure-discovery', + target: 'https://dev.azure.com/shopify/engineering/_git/backstage', + }; + mockCodeSearch.mockResolvedValueOnce([]); + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(mockCodeSearch).toHaveBeenCalledWith( + { host: 'dev.azure.com' }, + 'shopify', + 'engineering', + 'backstage', + '/catalog-info.yaml', + ); + expect(emitter).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts new file mode 100644 index 0000000000..58ee2155e9 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts @@ -0,0 +1,151 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { ScmIntegrations } from '@backstage/integration'; +import { Logger } from 'winston'; +import * as results from './results'; +import { CatalogProcessor, CatalogProcessorEmit } from './types'; +import { codeSearch } from './azure'; + +/** + * Extracts repositories out of an Azure DevOps org. + * + * The following will create locations for all projects which have a catalog-info.yaml + * on the default branch. The first is shorthand for the second. + * + * target: "https://dev.azure.com/org/project" + * or + * target: https://dev.azure.com/org/project?path=/catalog-info.yaml + * + * You may also explicitly specify a single repo: + * + * target: https://dev.azure.com/org/project/_git/repo + **/ +export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { + private readonly integrations: ScmIntegrations; + private readonly logger: Logger; + + static fromConfig(config: Config, options: { logger: Logger }) { + const integrations = ScmIntegrations.fromConfig(config); + + return new AzureDevOpsDiscoveryProcessor({ + ...options, + integrations, + }); + } + + constructor(options: { integrations: ScmIntegrations; logger: Logger }) { + this.integrations = options.integrations; + this.logger = options.logger; + } + + async readLocation( + location: LocationSpec, + _optional: boolean, + emit: CatalogProcessorEmit, + ): Promise { + if (location.type !== 'azure-discovery') { + return false; + } + + const azureConfig = this.integrations.azure.byUrl(location.target)?.config; + if (!azureConfig) { + throw new Error( + `There is no Azure integration that matches ${location.target}. Please add a configuration entry for it under integrations.azure`, + ); + } + + const { baseUrl, org, project, repo, catalogPath } = parseUrl( + location.target, + ); + this.logger.info( + `Reading Azure DevOps repositories from ${location.target}`, + ); + + const files = await codeSearch( + azureConfig, + org, + project, + repo, + catalogPath, + ); + + this.logger.debug( + `Found ${files.length} files in Azure DevOps from ${location.target}.`, + ); + + for (const file of files) { + emit( + results.location( + { + type: 'url', + target: `${baseUrl}/${org}/${project}/_git/${file.repository.name}?path=${file.path}`, + }, + // Not all locations may actually exist, since the user defined them as a wildcard pattern. + // Thus, we emit them as optional and let the downstream processor find them while not outputting + // an error if it couldn't. + true, + ), + ); + } + + return true; + } +} + +/** + * parseUrl extracts segments from the Azure DevOps URL. + **/ +export function parseUrl(urlString: string): { + baseUrl: string; + org: string; + project: string; + repo: string; + catalogPath: string; +} { + const url = new URL(urlString); + const path = url.pathname.substr(1).split('/'); + + const catalogPath = url.searchParams.get('path') || '/catalog-info.yaml'; + + if (path.length === 2 && path[0].length && path[1].length) { + return { + baseUrl: url.origin, + org: decodeURIComponent(path[0]), + project: decodeURIComponent(path[1]), + repo: '', + catalogPath, + }; + } else if ( + path.length === 4 && + path[0].length && + path[1].length && + path[2].length && + path[3].length + ) { + return { + baseUrl: url.origin, + org: decodeURIComponent(path[0]), + project: decodeURIComponent(path[1]), + repo: decodeURIComponent(path[3]), + catalogPath, + }; + } + + throw new Error(`Failed to parse ${urlString}`); +} diff --git a/plugins/catalog-backend/src/ingestion/processors/azure/azure.test.ts b/plugins/catalog-backend/src/ingestion/processors/azure/azure.test.ts new file mode 100644 index 0000000000..67cc120bab --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/azure/azure.test.ts @@ -0,0 +1,232 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { setupServer } from 'msw/node'; +import { setupRequestMockHandlers } from '@backstage/test-utils'; +import { rest } from 'msw'; +import { codeSearch, CodeSearchResponse } from './azure'; + +describe('azure', () => { + const server = setupServer(); + setupRequestMockHandlers(server); + + describe('codeSearch', () => { + it('returns empty when nothing is found', async () => { + const response: CodeSearchResponse = { count: 0, results: [] }; + + server.use( + rest.post( + `https://almsearch.dev.azure.com/shopify/engineering/_apis/search/codesearchresults`, + (req, res, ctx) => { + expect(req.headers.get('Authorization')).toBe('Basic OkFCQw=='); + expect(req.body).toEqual({ + searchText: 'path:/catalog-info.yaml repo:*', + $skip: 0, + $top: 1000, + }); + return res(ctx.json(response)); + }, + ), + ); + + await expect( + codeSearch( + { host: 'dev.azure.com', token: 'ABC' }, + 'shopify', + 'engineering', + '', + '/catalog-info.yaml', + ), + ).resolves.toEqual([]); + }); + }); + + it('returns entries when request matches some files', async () => { + const response: CodeSearchResponse = { + count: 2, + results: [ + { + fileName: 'catalog-info.yaml', + path: '/catalog-info.yaml', + repository: { + name: 'backstage', + }, + }, + { + fileName: 'catalog-info.yaml', + path: '/catalog-info.yaml', + repository: { + name: 'ios-app', + }, + }, + ], + }; + + server.use( + rest.post( + `https://almsearch.dev.azure.com/shopify/engineering/_apis/search/codesearchresults`, + (req, res, ctx) => { + expect(req.headers.get('Authorization')).toBe('Basic OkFCQw=='); + expect(req.body).toEqual({ + searchText: 'path:/catalog-info.yaml repo:*', + $skip: 0, + $top: 1000, + }); + return res(ctx.json(response)); + }, + ), + ); + + await expect( + codeSearch( + { host: 'dev.azure.com', token: 'ABC' }, + 'shopify', + 'engineering', + '', + '/catalog-info.yaml', + ), + ).resolves.toEqual(response.results); + }); + + it('searches in specific repo if parameter is set', async () => { + const response: CodeSearchResponse = { + count: 1, + results: [ + { + fileName: 'catalog-info.yaml', + path: '/catalog-info.yaml', + repository: { + name: 'backstage', + }, + }, + ], + }; + + server.use( + rest.post( + `https://almsearch.dev.azure.com/shopify/engineering/_apis/search/codesearchresults`, + (req, res, ctx) => { + expect(req.headers.get('Authorization')).toBe('Basic OkFCQw=='); + expect(req.body).toEqual({ + searchText: 'path:/catalog-info.yaml repo:backstage', + $skip: 0, + $top: 1000, + }); + return res(ctx.json(response)); + }, + ), + ); + + await expect( + codeSearch( + { host: 'dev.azure.com', token: 'ABC' }, + 'shopify', + 'engineering', + 'backstage', + '/catalog-info.yaml', + ), + ).resolves.toEqual(response.results); + }); + + it('can search using onpremise api', async () => { + const response: CodeSearchResponse = { + count: 1, + results: [ + { + fileName: 'catalog-info.yaml', + path: '/catalog-info.yaml', + repository: { + name: 'backstage', + }, + }, + ], + }; + + server.use( + rest.post( + `https://azuredevops.mycompany.com/shopify/engineering/_apis/search/codesearchresults`, + (req, res, ctx) => { + expect(req.headers.get('Authorization')).toBe('Basic OkFCQw=='); + expect(req.body).toEqual({ + searchText: 'path:/catalog-info.yaml repo:*', + $skip: 0, + $top: 1000, + }); + return res(ctx.json(response)); + }, + ), + ); + + await expect( + codeSearch( + { host: 'azuredevops.mycompany.com', token: 'ABC' }, + 'shopify', + 'engineering', + '', + '/catalog-info.yaml', + ), + ).resolves.toEqual(response.results); + }); + + it('searches multiple pages if response contains many items', async () => { + const totalCount = 2401; + const generateItems = (count: number) => { + return Array.from(Array(count).keys()).map(_ => ({ + fileName: 'catalog-info.yaml', + path: '/catalog-info.yaml', + repository: { + name: 'backstage', + }, + })); + }; + + server.use( + rest.post( + `https://almsearch.dev.azure.com/shopify/engineering/_apis/search/codesearchresults`, + (req, res, ctx) => { + expect(req.headers.get('Authorization')).toBe('Basic OkFCQw=='); + expect(req.body).toMatchObject({ + searchText: 'path:/catalog-info.yaml repo:backstage', + $top: 1000, + }); + + const body = req.body as { $skip: number; $top: number }; + const countItemsToReturn = + body.$top + body.$skip > totalCount + ? totalCount - body.$skip + : body.$top; + + return res( + ctx.json({ + count: totalCount, + results: generateItems(countItemsToReturn), + }), + ); + }, + ), + ); + + await expect( + codeSearch( + { host: 'dev.azure.com', token: 'ABC' }, + 'shopify', + 'engineering', + 'backstage', + '/catalog-info.yaml', + ), + ).resolves.toHaveLength(totalCount); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/azure/azure.ts b/plugins/catalog-backend/src/ingestion/processors/azure/azure.ts new file mode 100644 index 0000000000..0c7b17483a --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/azure/azure.ts @@ -0,0 +1,80 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fetch from 'cross-fetch'; +import { + AzureIntegrationConfig, + getAzureRequestOptions, +} from '@backstage/integration'; + +export interface CodeSearchResponse { + count: number; + results: CodeSearchResultItem[]; +} + +export interface CodeSearchResultItem { + fileName: string; + path: string; + repository: { + name: string; + }; +} + +const isCloud = (host: string) => host === 'dev.azure.com'; +const PAGE_SIZE = 1000; + +// codeSearch returns all files that matches the given search path. +export async function codeSearch( + azureConfig: AzureIntegrationConfig, + org: string, + project: string, + repo: string, + path: string, +): Promise { + const searchBaseUrl = isCloud(azureConfig.host) + ? 'https://almsearch.dev.azure.com' + : `https://${azureConfig.host}`; + const searchUrl = `${searchBaseUrl}/${org}/${project}/_apis/search/codesearchresults?api-version=6.0-preview.1`; + + let items: CodeSearchResultItem[] = []; + let hasMorePages = true; + + do { + const response = await fetch(searchUrl, { + ...getAzureRequestOptions(azureConfig, { + 'Content-Type': 'application/json', + }), + method: 'POST', + body: JSON.stringify({ + searchText: `path:${path} repo:${repo || '*'}`, + $skip: items.length, + $top: PAGE_SIZE, + }), + }); + + if (response.status !== 200) { + throw new Error( + `Azure DevOps search failed with response status ${response.status}`, + ); + } + + const body: CodeSearchResponse = await response.json(); + items = [...items, ...body.results]; + hasMorePages = body.count > items.length; + } while (hasMorePages); + + return items; +} diff --git a/plugins/catalog-backend/src/ingestion/processors/azure/index.ts b/plugins/catalog-backend/src/ingestion/processors/azure/index.ts new file mode 100644 index 0000000000..d4ff56c4c0 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/azure/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './azure'; diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index 4f3c502a8e..63feac51af 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -26,6 +26,7 @@ export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor'; export { CodeOwnersProcessor } from './CodeOwnersProcessor'; export { FileReaderProcessor } from './FileReaderProcessor'; export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor'; +export { AzureDevOpsDiscoveryProcessor } from './AzureDevOpsDiscoveryProcessor'; export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; export { GithubMultiOrgReaderProcessor } from './GithubMultiOrgReaderProcessor'; export { GitLabDiscoveryProcessor } from './GitLabDiscoveryProcessor'; diff --git a/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts b/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts index b8e45c7cf3..6fb52e245e 100644 --- a/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts @@ -42,6 +42,7 @@ import { CodeOwnersProcessor, FileReaderProcessor, GithubDiscoveryProcessor, + AzureDevOpsDiscoveryProcessor, GithubOrgReaderProcessor, GitLabDiscoveryProcessor, LocationEntityProcessor, @@ -317,6 +318,7 @@ export class CatalogBuilder { new FileReaderProcessor(), BitbucketDiscoveryProcessor.fromConfig(config, { logger }), GithubDiscoveryProcessor.fromConfig(config, { logger }), + AzureDevOpsDiscoveryProcessor.fromConfig(config, { logger }), GithubOrgReaderProcessor.fromConfig(config, { logger }), GitLabDiscoveryProcessor.fromConfig(config, { logger }), new UrlReaderProcessor({ reader, logger }), diff --git a/plugins/catalog-backend/src/service/NextCatalogBuilder.ts b/plugins/catalog-backend/src/service/NextCatalogBuilder.ts index eed420cd53..c9ebc8f723 100644 --- a/plugins/catalog-backend/src/service/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/NextCatalogBuilder.ts @@ -44,6 +44,7 @@ import { CatalogProcessorParser, CodeOwnersProcessor, FileReaderProcessor, + AzureDevOpsDiscoveryProcessor, GithubDiscoveryProcessor, GithubOrgReaderProcessor, GitLabDiscoveryProcessor, @@ -292,6 +293,7 @@ export class NextCatalogBuilder { return [ new FileReaderProcessor(), BitbucketDiscoveryProcessor.fromConfig(config, { logger }), + AzureDevOpsDiscoveryProcessor.fromConfig(config, { logger }), GithubDiscoveryProcessor.fromConfig(config, { logger }), GithubOrgReaderProcessor.fromConfig(config, { logger }), GitLabDiscoveryProcessor.fromConfig(config, { logger }),