diff --git a/docs/integrations/azure/discovery.md b/docs/integrations/azure/discovery.md new file mode 100644 index 0000000000..ad452f47af --- /dev/null +++ b/docs/integrations/azure/discovery.md @@ -0,0 +1,49 @@ +--- +id: discovery +title: Azure DevOps Discovery +sidebar_label: Discovery +# prettier-ignore +description: Automatically discovering catalog entities from repositories in an Azure DevOps organization +--- + +The Azure DevOps integration has a special discovery processor for discovering +catalog entities within an Azure DevOps. The processor will crawl the Azure +DevOps organization and register entities matching the configured path. This can +be useful as an alternative to static locations or manually adding things to the +catalog. + +To use the discovery processor, you'll need a GitHub integration +[set up](locations.md) with a `AZURE_TOKEN`. Then you can add a location target +to the catalog configuration: + +```yaml +catalog: + locations: + # Scan all repositories for a catalog-info.yaml in the root of the default branch + - type: azure-discovery + target: https://dev.azure.com/myorg/myproject + # Or use a custom pattern for a subset of all repositories with default repository + - type: azure-discovery + target: https://dev.azure.com/myorg/myproject/_git/service-* + # Or use a custom file format and location + - type: azure-discovery + target: https://dev.azure.com/myorg/myproject/_git/*?path=/src/*/catalog-info.yaml +``` + +Note the `azure-discovery` type, as this is not a regular `url` processor. + +When using a custom pattern, the target is composed of five parts: + +- The base instance URL, `https://dev.azure.com` in this case +- The organization name which is required, `myorg` in this case +- The project name which is required, `myproject` in this case +- The repository blob to scan, which accepts \* wildcard tokens and must be + added after `_git/`. This can simply be `*` to scan all repositories in the + project. +- The path within each repository to find the catalog YAML file. This will + usually be `/catalog-info.yaml`, `/src/*/catalog-info.yaml` or a similar + variation for catalog files stored in the root directory of each repository. + +_Note:_ the path parameter follows the same rules as the search on Azure DevOps +web interface. For more details visit the +[official search documentation](https://docs.microsoft.com/en-us/azure/devops/project/search/get-started-search?view=azure-devops) diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts new file mode 100644 index 0000000000..9d468acf15 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts @@ -0,0 +1,63 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { parseUrl } from './AzureDevOpsDiscoveryProcessor'; + +describe('AzureDevOpsDiscoveryProcessor', () => { + describe('parseUrl', () => { + it('parses well formed URLs', () => { + expect(parseUrl('https://dev.azure.com/my-org/my-proj')).toEqual({ + baseUrl: 'https://dev.azure.com', + org: 'my-org', + project: 'my-proj', + repo: '', + catalogPath: '/catalog-info.yaml', + }); + + expect( + parseUrl( + 'https://dev.azure.com/spotify/engineering/_git/backstage?path=/catalog.yaml', + ), + ).toEqual({ + baseUrl: 'https://dev.azure.com', + org: 'spotify', + project: 'engineering', + repo: 'backstage', + catalogPath: '/catalog.yaml', + }); + + expect( + parseUrl( + 'https://azuredevops.mycompany.com/spotify/engineering/_git/backstage?path=/src/*/catalog.yaml', + ), + ).toEqual({ + baseUrl: 'https://azuredevops.mycompany.com', + org: 'spotify', + project: 'engineering', + repo: 'backstage', + catalogPath: '/src/*/catalog.yaml', + }); + }); + + it('throws on incorrectly formed URLs', () => { + expect(() => parseUrl('https://dev.azure.com')).toThrow(); + expect(() => parseUrl('https://dev.azure.com//')).toThrow(); + expect(() => parseUrl('https://dev.azure.com//foo')).toThrow(); + }); + }); + + // TODO: add tests for AzureDevOpsDiscoveryProcessor +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts index 8300b503f3..a938e230e7 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts @@ -18,6 +18,7 @@ import { LocationSpec } from '@backstage/catalog-model'; import fetch from 'cross-fetch'; import { Config } from '@backstage/config'; import { + AzureIntegrationConfig, getAzureRequestOptions, ScmIntegrations, } from '@backstage/integration'; @@ -26,7 +27,18 @@ import * as results from './results'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; /** - * TODO + * Extracts repositories out of an Azure DevOps org. + * + * The following will create locations for all projects which have a catalog-info.yaml + * on the default branch. The first is shorthand for the second. + * + * target: "https://dev.azure.com/org/project" + * or + * target: https://dev.azure.com/org/project?path=/catalog-info.yaml + * + * You may also explicitly specify a single repo: + * + * target: https://dev.azure.com/org/project/_git/repo **/ export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { private readonly integrations: ScmIntegrations; @@ -62,13 +74,49 @@ export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { ); } - // TODO: extract this from configured URL - const { org, project } = { org: 'myOrg', project: 'myProject' }; + const { baseUrl, org, project, repo, catalogPath } = parseUrl( + location.target, + ); + this.logger.info( + `Reading Azure DevOps repositories from ${location.target}`, + ); - // TODO: What's the search URL for self hosted DevOps? + const files = await this.search( + azureConfig, + org, + project, + repo, + catalogPath, + ); + for (const file of files) { + emit( + results.location( + { + type: 'url', + target: `${baseUrl}/${org}/${project}/_git/${file.repository.name}?path=${file.path}`, + }, + // Not all locations may actually exist, since the user defined them as a wildcard pattern. + // Thus, we emit them as optional and let the downstream processor find them while not outputting + // an error if it couldn't. + true, + ), + ); + } + + return true; + } + + // search returns all files that matches the given search path. + async search( + azureConfig: AzureIntegrationConfig, + org: string, + project: string, + repo: string, + path: string, + ): Promise { + // TODO: What's the search URL for onpremises DevOps? const searchUrl = `https://almsearch.dev.azure.com/${org}/${project}/_apis/search/codesearchresults?api-version=6.0-preview.1`; const opts = getAzureRequestOptions(azureConfig); - const response = await fetch(searchUrl, { method: 'POST', headers: { @@ -76,53 +124,81 @@ export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { 'Content-Type': 'application/json', }, body: JSON.stringify({ - searchText: 'catalog-info.yaml', + searchText: `path:${path} repo:${repo || '*'}`, $top: 1000, }), }); - // TODO: more logging - if (response.status === 200) { - const responseBody: AzureDevOpsCodeSearchResults = await response.json(); - - // TODO: should we support different file names? - const matches = responseBody.results.filter( - r => r.fileName === 'catalog-info.yaml', + if (response.status !== 200) { + this.logger.warn( + `Azure DevOps search failed with response status ${response.status}`, ); - - for (const match of matches) { - emit( - results.location( - { - type: 'url', - // TODO: Do we need to support non-default branches? - target: `${location.target}/_git/${match.repository.name}?path=${match.path}`, - }, - // Not all locations may actually exist, since the user defined them as a wildcard pattern. - // Thus, we emit them as optional and let the downstream processor find them while not outputting - // an error if it couldn't. - true, - ), - ); - } + return []; } - return true; + const responseBody: CodeSearchResponse = await response.json(); + this.logger.info(`Azure DevOps search found ${responseBody.count} files`); + + return responseBody.results; } } -interface AzureDevOpsCodeSearchResults { - count: number; - results: Array<{ - fileName: string; - path: string; - project: { - id: string; - name: string; +/** + * parseUrl extracts segments from the Azure DevOps URL. + **/ +export function parseUrl(urlString: string): { + baseUrl: string; + org: string; + project: string; + repo: string; + catalogPath: string; +} { + const url = new URL(urlString); + const path = url.pathname.substr(1).split('/'); + + const catalogPath = url.searchParams.get('path') || '/catalog-info.yaml'; + + if (path.length === 2 && path[0].length && path[1].length) { + return { + baseUrl: url.origin, + org: decodeURIComponent(path[0]), + project: decodeURIComponent(path[1]), + repo: '', + catalogPath, }; - repository: { - id: string; - name: string; + } else if ( + path.length === 4 && + path[0].length && + path[1].length && + path[2].length && + path[3].length + ) { + return { + baseUrl: url.origin, + org: decodeURIComponent(path[0]), + project: decodeURIComponent(path[1]), + repo: decodeURIComponent(path[3]), + catalogPath, }; - }>; + } + + throw new Error(`Failed to parse ${urlString}`); +} + +interface CodeSearchResponse { + count: number; + results: CodeSearchResultItem[]; +} + +interface CodeSearchResultItem { + fileName: string; + path: string; + project: { + id: string; + name: string; + }; + repository: { + id: string; + name: string; + }; }