From 0179bd1f139b9e7383a965b0adf53075ad0eaf7a Mon Sep 17 00:00:00 2001 From: goenning Date: Sat, 20 Nov 2021 09:37:57 +0000 Subject: [PATCH 1/5] draft implementation for AzureDevOps Discovery Signed-off-by: goenning --- .../AzureDevOpsDiscoveryProcessor.ts | 128 ++++++++++++++++++ .../src/ingestion/processors/index.ts | 1 + .../src/legacy/service/CatalogBuilder.ts | 2 + .../src/service/NextCatalogBuilder.ts | 2 + 4 files changed, 133 insertions(+) create mode 100644 plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts new file mode 100644 index 0000000000..8300b503f3 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts @@ -0,0 +1,128 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import fetch from 'cross-fetch'; +import { Config } from '@backstage/config'; +import { + getAzureRequestOptions, + ScmIntegrations, +} from '@backstage/integration'; +import { Logger } from 'winston'; +import * as results from './results'; +import { CatalogProcessor, CatalogProcessorEmit } from './types'; + +/** + * TODO + **/ +export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { + private readonly integrations: ScmIntegrations; + private readonly logger: Logger; + + static fromConfig(config: Config, options: { logger: Logger }) { + const integrations = ScmIntegrations.fromConfig(config); + + return new AzureDevOpsDiscoveryProcessor({ + ...options, + integrations, + }); + } + + constructor(options: { integrations: ScmIntegrations; logger: Logger }) { + this.integrations = options.integrations; + this.logger = options.logger; + } + + async readLocation( + location: LocationSpec, + _optional: boolean, + emit: CatalogProcessorEmit, + ): Promise { + if (location.type !== 'azure-discovery') { + return false; + } + + const azureConfig = this.integrations.azure.byUrl(location.target)?.config; + if (!azureConfig) { + throw new Error( + `There is no Azure integration that matches ${location.target}. Please add a configuration entry for it under integrations.azure`, + ); + } + + // TODO: extract this from configured URL + const { org, project } = { org: 'myOrg', project: 'myProject' }; + + // TODO: What's the search URL for self hosted DevOps? + const searchUrl = `https://almsearch.dev.azure.com/${org}/${project}/_apis/search/codesearchresults?api-version=6.0-preview.1`; + const opts = getAzureRequestOptions(azureConfig); + + const response = await fetch(searchUrl, { + method: 'POST', + headers: { + ...opts.headers, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + searchText: 'catalog-info.yaml', + $top: 1000, + }), + }); + + // TODO: more logging + if (response.status === 200) { + const responseBody: AzureDevOpsCodeSearchResults = await response.json(); + + // TODO: should we support different file names? + const matches = responseBody.results.filter( + r => r.fileName === 'catalog-info.yaml', + ); + + for (const match of matches) { + emit( + results.location( + { + type: 'url', + // TODO: Do we need to support non-default branches? + target: `${location.target}/_git/${match.repository.name}?path=${match.path}`, + }, + // Not all locations may actually exist, since the user defined them as a wildcard pattern. + // Thus, we emit them as optional and let the downstream processor find them while not outputting + // an error if it couldn't. + true, + ), + ); + } + } + + return true; + } +} + +interface AzureDevOpsCodeSearchResults { + count: number; + results: Array<{ + fileName: string; + path: string; + project: { + id: string; + name: string; + }; + repository: { + id: string; + name: string; + }; + }>; +} diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index 4f3c502a8e..63feac51af 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -26,6 +26,7 @@ export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor'; export { CodeOwnersProcessor } from './CodeOwnersProcessor'; export { FileReaderProcessor } from './FileReaderProcessor'; export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor'; +export { AzureDevOpsDiscoveryProcessor } from './AzureDevOpsDiscoveryProcessor'; export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; export { GithubMultiOrgReaderProcessor } from './GithubMultiOrgReaderProcessor'; export { GitLabDiscoveryProcessor } from './GitLabDiscoveryProcessor'; diff --git a/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts b/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts index b8e45c7cf3..6fb52e245e 100644 --- a/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts @@ -42,6 +42,7 @@ import { CodeOwnersProcessor, FileReaderProcessor, GithubDiscoveryProcessor, + AzureDevOpsDiscoveryProcessor, GithubOrgReaderProcessor, GitLabDiscoveryProcessor, LocationEntityProcessor, @@ -317,6 +318,7 @@ export class CatalogBuilder { new FileReaderProcessor(), BitbucketDiscoveryProcessor.fromConfig(config, { logger }), GithubDiscoveryProcessor.fromConfig(config, { logger }), + AzureDevOpsDiscoveryProcessor.fromConfig(config, { logger }), GithubOrgReaderProcessor.fromConfig(config, { logger }), GitLabDiscoveryProcessor.fromConfig(config, { logger }), new UrlReaderProcessor({ reader, logger }), diff --git a/plugins/catalog-backend/src/service/NextCatalogBuilder.ts b/plugins/catalog-backend/src/service/NextCatalogBuilder.ts index bbda6f070c..a694030701 100644 --- a/plugins/catalog-backend/src/service/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/NextCatalogBuilder.ts @@ -44,6 +44,7 @@ import { CatalogProcessorParser, CodeOwnersProcessor, FileReaderProcessor, + AzureDevOpsDiscoveryProcessor, GithubDiscoveryProcessor, GithubOrgReaderProcessor, GitLabDiscoveryProcessor, @@ -291,6 +292,7 @@ export class NextCatalogBuilder { return [ new FileReaderProcessor(), BitbucketDiscoveryProcessor.fromConfig(config, { logger }), + AzureDevOpsDiscoveryProcessor.fromConfig(config, { logger }), GithubDiscoveryProcessor.fromConfig(config, { logger }), GithubOrgReaderProcessor.fromConfig(config, { logger }), GitLabDiscoveryProcessor.fromConfig(config, { logger }), From 51e42bc2c81846765b11ca9288259225a0b4346e Mon Sep 17 00:00:00 2001 From: goenning Date: Sat, 20 Nov 2021 21:20:47 +0000 Subject: [PATCH 2/5] add support for search queries Signed-off-by: goenning --- docs/integrations/azure/discovery.md | 49 ++++++ .../AzureDevOpsDiscoveryProcessor.test.ts | 63 +++++++ .../AzureDevOpsDiscoveryProcessor.ts | 160 +++++++++++++----- 3 files changed, 230 insertions(+), 42 deletions(-) create mode 100644 docs/integrations/azure/discovery.md create mode 100644 plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts diff --git a/docs/integrations/azure/discovery.md b/docs/integrations/azure/discovery.md new file mode 100644 index 0000000000..ad452f47af --- /dev/null +++ b/docs/integrations/azure/discovery.md @@ -0,0 +1,49 @@ +--- +id: discovery +title: Azure DevOps Discovery +sidebar_label: Discovery +# prettier-ignore +description: Automatically discovering catalog entities from repositories in an Azure DevOps organization +--- + +The Azure DevOps integration has a special discovery processor for discovering +catalog entities within an Azure DevOps. The processor will crawl the Azure +DevOps organization and register entities matching the configured path. This can +be useful as an alternative to static locations or manually adding things to the +catalog. + +To use the discovery processor, you'll need a GitHub integration +[set up](locations.md) with a `AZURE_TOKEN`. Then you can add a location target +to the catalog configuration: + +```yaml +catalog: + locations: + # Scan all repositories for a catalog-info.yaml in the root of the default branch + - type: azure-discovery + target: https://dev.azure.com/myorg/myproject + # Or use a custom pattern for a subset of all repositories with default repository + - type: azure-discovery + target: https://dev.azure.com/myorg/myproject/_git/service-* + # Or use a custom file format and location + - type: azure-discovery + target: https://dev.azure.com/myorg/myproject/_git/*?path=/src/*/catalog-info.yaml +``` + +Note the `azure-discovery` type, as this is not a regular `url` processor. + +When using a custom pattern, the target is composed of five parts: + +- The base instance URL, `https://dev.azure.com` in this case +- The organization name which is required, `myorg` in this case +- The project name which is required, `myproject` in this case +- The repository blob to scan, which accepts \* wildcard tokens and must be + added after `_git/`. This can simply be `*` to scan all repositories in the + project. +- The path within each repository to find the catalog YAML file. This will + usually be `/catalog-info.yaml`, `/src/*/catalog-info.yaml` or a similar + variation for catalog files stored in the root directory of each repository. + +_Note:_ the path parameter follows the same rules as the search on Azure DevOps +web interface. For more details visit the +[official search documentation](https://docs.microsoft.com/en-us/azure/devops/project/search/get-started-search?view=azure-devops) diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts new file mode 100644 index 0000000000..9d468acf15 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts @@ -0,0 +1,63 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { parseUrl } from './AzureDevOpsDiscoveryProcessor'; + +describe('AzureDevOpsDiscoveryProcessor', () => { + describe('parseUrl', () => { + it('parses well formed URLs', () => { + expect(parseUrl('https://dev.azure.com/my-org/my-proj')).toEqual({ + baseUrl: 'https://dev.azure.com', + org: 'my-org', + project: 'my-proj', + repo: '', + catalogPath: '/catalog-info.yaml', + }); + + expect( + parseUrl( + 'https://dev.azure.com/spotify/engineering/_git/backstage?path=/catalog.yaml', + ), + ).toEqual({ + baseUrl: 'https://dev.azure.com', + org: 'spotify', + project: 'engineering', + repo: 'backstage', + catalogPath: '/catalog.yaml', + }); + + expect( + parseUrl( + 'https://azuredevops.mycompany.com/spotify/engineering/_git/backstage?path=/src/*/catalog.yaml', + ), + ).toEqual({ + baseUrl: 'https://azuredevops.mycompany.com', + org: 'spotify', + project: 'engineering', + repo: 'backstage', + catalogPath: '/src/*/catalog.yaml', + }); + }); + + it('throws on incorrectly formed URLs', () => { + expect(() => parseUrl('https://dev.azure.com')).toThrow(); + expect(() => parseUrl('https://dev.azure.com//')).toThrow(); + expect(() => parseUrl('https://dev.azure.com//foo')).toThrow(); + }); + }); + + // TODO: add tests for AzureDevOpsDiscoveryProcessor +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts index 8300b503f3..a938e230e7 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts @@ -18,6 +18,7 @@ import { LocationSpec } from '@backstage/catalog-model'; import fetch from 'cross-fetch'; import { Config } from '@backstage/config'; import { + AzureIntegrationConfig, getAzureRequestOptions, ScmIntegrations, } from '@backstage/integration'; @@ -26,7 +27,18 @@ import * as results from './results'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; /** - * TODO + * Extracts repositories out of an Azure DevOps org. + * + * The following will create locations for all projects which have a catalog-info.yaml + * on the default branch. The first is shorthand for the second. + * + * target: "https://dev.azure.com/org/project" + * or + * target: https://dev.azure.com/org/project?path=/catalog-info.yaml + * + * You may also explicitly specify a single repo: + * + * target: https://dev.azure.com/org/project/_git/repo **/ export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { private readonly integrations: ScmIntegrations; @@ -62,13 +74,49 @@ export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { ); } - // TODO: extract this from configured URL - const { org, project } = { org: 'myOrg', project: 'myProject' }; + const { baseUrl, org, project, repo, catalogPath } = parseUrl( + location.target, + ); + this.logger.info( + `Reading Azure DevOps repositories from ${location.target}`, + ); - // TODO: What's the search URL for self hosted DevOps? + const files = await this.search( + azureConfig, + org, + project, + repo, + catalogPath, + ); + for (const file of files) { + emit( + results.location( + { + type: 'url', + target: `${baseUrl}/${org}/${project}/_git/${file.repository.name}?path=${file.path}`, + }, + // Not all locations may actually exist, since the user defined them as a wildcard pattern. + // Thus, we emit them as optional and let the downstream processor find them while not outputting + // an error if it couldn't. + true, + ), + ); + } + + return true; + } + + // search returns all files that matches the given search path. + async search( + azureConfig: AzureIntegrationConfig, + org: string, + project: string, + repo: string, + path: string, + ): Promise { + // TODO: What's the search URL for onpremises DevOps? const searchUrl = `https://almsearch.dev.azure.com/${org}/${project}/_apis/search/codesearchresults?api-version=6.0-preview.1`; const opts = getAzureRequestOptions(azureConfig); - const response = await fetch(searchUrl, { method: 'POST', headers: { @@ -76,53 +124,81 @@ export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { 'Content-Type': 'application/json', }, body: JSON.stringify({ - searchText: 'catalog-info.yaml', + searchText: `path:${path} repo:${repo || '*'}`, $top: 1000, }), }); - // TODO: more logging - if (response.status === 200) { - const responseBody: AzureDevOpsCodeSearchResults = await response.json(); - - // TODO: should we support different file names? - const matches = responseBody.results.filter( - r => r.fileName === 'catalog-info.yaml', + if (response.status !== 200) { + this.logger.warn( + `Azure DevOps search failed with response status ${response.status}`, ); - - for (const match of matches) { - emit( - results.location( - { - type: 'url', - // TODO: Do we need to support non-default branches? - target: `${location.target}/_git/${match.repository.name}?path=${match.path}`, - }, - // Not all locations may actually exist, since the user defined them as a wildcard pattern. - // Thus, we emit them as optional and let the downstream processor find them while not outputting - // an error if it couldn't. - true, - ), - ); - } + return []; } - return true; + const responseBody: CodeSearchResponse = await response.json(); + this.logger.info(`Azure DevOps search found ${responseBody.count} files`); + + return responseBody.results; } } -interface AzureDevOpsCodeSearchResults { - count: number; - results: Array<{ - fileName: string; - path: string; - project: { - id: string; - name: string; +/** + * parseUrl extracts segments from the Azure DevOps URL. + **/ +export function parseUrl(urlString: string): { + baseUrl: string; + org: string; + project: string; + repo: string; + catalogPath: string; +} { + const url = new URL(urlString); + const path = url.pathname.substr(1).split('/'); + + const catalogPath = url.searchParams.get('path') || '/catalog-info.yaml'; + + if (path.length === 2 && path[0].length && path[1].length) { + return { + baseUrl: url.origin, + org: decodeURIComponent(path[0]), + project: decodeURIComponent(path[1]), + repo: '', + catalogPath, }; - repository: { - id: string; - name: string; + } else if ( + path.length === 4 && + path[0].length && + path[1].length && + path[2].length && + path[3].length + ) { + return { + baseUrl: url.origin, + org: decodeURIComponent(path[0]), + project: decodeURIComponent(path[1]), + repo: decodeURIComponent(path[3]), + catalogPath, }; - }>; + } + + throw new Error(`Failed to parse ${urlString}`); +} + +interface CodeSearchResponse { + count: number; + results: CodeSearchResultItem[]; +} + +interface CodeSearchResultItem { + fileName: string; + path: string; + project: { + id: string; + name: string; + }; + repository: { + id: string; + name: string; + }; } From ac724ea8aee28c2deec0099fa46347853113be5a Mon Sep 17 00:00:00 2001 From: goenning Date: Sun, 21 Nov 2021 14:45:41 +0000 Subject: [PATCH 3/5] adding tests Signed-off-by: goenning --- .../AzureDevOpsDiscoveryProcessor.test.ts | 218 +++++++++++++++++- .../AzureDevOpsDiscoveryProcessor.ts | 67 +----- .../ingestion/processors/azure/azure.test.ts | 140 +++++++++++ .../src/ingestion/processors/azure/azure.ts | 67 ++++++ .../src/ingestion/processors/azure/index.ts | 17 ++ 5 files changed, 446 insertions(+), 63 deletions(-) create mode 100644 plugins/catalog-backend/src/ingestion/processors/azure/azure.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/azure/azure.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/azure/index.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts index 9d468acf15..ab053e8748 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts @@ -14,7 +14,17 @@ * limitations under the License. */ -import { parseUrl } from './AzureDevOpsDiscoveryProcessor'; +import { getVoidLogger } from '@backstage/backend-common'; +import { LocationSpec } from '@backstage/catalog-model'; +import { ConfigReader } from '@backstage/config'; +import { codeSearch } from './azure'; +import { + AzureDevOpsDiscoveryProcessor, + parseUrl, +} from './AzureDevOpsDiscoveryProcessor'; + +jest.mock('./azure'); +const mockCodeSearch = codeSearch as jest.MockedFunction; describe('AzureDevOpsDiscoveryProcessor', () => { describe('parseUrl', () => { @@ -59,5 +69,209 @@ describe('AzureDevOpsDiscoveryProcessor', () => { }); }); - // TODO: add tests for AzureDevOpsDiscoveryProcessor + describe('reject unrelated entries', () => { + it('rejects unknown types', async () => { + const processor = AzureDevOpsDiscoveryProcessor.fromConfig( + new ConfigReader({ + integrations: { + azure: [{ host: 'dev.azure.com', token: 'blob' }], + }, + }), + { logger: getVoidLogger() }, + ); + const location: LocationSpec = { + type: 'not-azure-discovery', + target: 'https://dev.azure.com', + }; + await expect( + processor.readLocation(location, false, () => {}), + ).resolves.toBeFalsy(); + }); + }); + + it('rejects unknown targets', async () => { + const processor = AzureDevOpsDiscoveryProcessor.fromConfig( + new ConfigReader({ + integrations: { + github: [ + { host: 'dev.azure.com', token: 'blob' }, + { host: 'azure.myorg.com', token: 'blob' }, + ], + }, + }), + { logger: getVoidLogger() }, + ); + const location: LocationSpec = { + type: 'azure-discovery', + target: 'https://not.azure.com/org/project', + }; + await expect( + processor.readLocation(location, false, () => {}), + ).rejects.toThrow( + /There is no Azure integration that matches https:\/\/not.azure.com\/org\/project. Please add a configuration entry for it under integrations.azure/, + ); + }); + + describe('handles repositories', () => { + const processor = AzureDevOpsDiscoveryProcessor.fromConfig( + new ConfigReader({ + integrations: { + github: [{ host: 'dev.azure.com', token: 'blob' }], + }, + }), + { logger: getVoidLogger() }, + ); + + beforeEach(() => { + mockCodeSearch.mockClear(); + }); + + it('output all locations found on from code search', async () => { + const location: LocationSpec = { + type: 'azure-discovery', + target: 'https://dev.azure.com/shopify/engineering', + }; + mockCodeSearch.mockResolvedValueOnce([ + { + fileName: 'catalog-info.yaml', + path: '/catalog-info.yaml', + repository: { + name: 'backstage', + }, + }, + { + fileName: 'catalog-info.yaml', + path: '/src/catalog-info.yaml', + repository: { + name: 'ios-app', + }, + }, + ]); + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(mockCodeSearch).toHaveBeenCalledWith( + { host: 'dev.azure.com' }, + 'shopify', + 'engineering', + '', + '/catalog-info.yaml', + ); + expect(emitter).toHaveBeenCalledTimes(2); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://dev.azure.com/shopify/engineering/_git/backstage?path=/catalog-info.yaml', + }, + optional: true, + }); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://dev.azure.com/shopify/engineering/_git/ios-app?path=/src/catalog-info.yaml', + }, + optional: true, + }); + }); + + it('output single locations from code search', async () => { + const location: LocationSpec = { + type: 'azure-discovery', + target: 'https://dev.azure.com/shopify/engineering/_git/backstage', + }; + mockCodeSearch.mockResolvedValueOnce([ + { + fileName: 'catalog-info.yaml', + path: '/catalog-info.yaml', + repository: { + name: 'backstage', + }, + }, + ]); + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(mockCodeSearch).toHaveBeenCalledWith( + { host: 'dev.azure.com' }, + 'shopify', + 'engineering', + 'backstage', + '/catalog-info.yaml', + ); + expect(emitter).toHaveBeenCalledTimes(1); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://dev.azure.com/shopify/engineering/_git/backstage?path=/catalog-info.yaml', + }, + optional: true, + }); + }); + + it('output single locations with different file name from code search', async () => { + const location: LocationSpec = { + type: 'azure-discovery', + target: + 'https://dev.azure.com/shopify/engineering?path=/src/*/catalog.yaml', + }; + mockCodeSearch.mockResolvedValueOnce([ + { + fileName: 'catalog.yaml', + path: '/src/main/catalog.yaml', + repository: { + name: 'backstage', + }, + }, + ]); + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(mockCodeSearch).toHaveBeenCalledWith( + { host: 'dev.azure.com' }, + 'shopify', + 'engineering', + '', + '/src/*/catalog.yaml', + ); + expect(emitter).toHaveBeenCalledTimes(1); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://dev.azure.com/shopify/engineering/_git/backstage?path=/src/main/catalog.yaml', + }, + optional: true, + }); + }); + + it('output nothing when code search does not find anything', async () => { + const location: LocationSpec = { + type: 'azure-discovery', + target: 'https://dev.azure.com/shopify/engineering/_git/backstage', + }; + mockCodeSearch.mockResolvedValueOnce([]); + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(mockCodeSearch).toHaveBeenCalledWith( + { host: 'dev.azure.com' }, + 'shopify', + 'engineering', + 'backstage', + '/catalog-info.yaml', + ); + expect(emitter).not.toHaveBeenCalled(); + }); + }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts index a938e230e7..b86e3f9c82 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts @@ -15,16 +15,12 @@ */ import { LocationSpec } from '@backstage/catalog-model'; -import fetch from 'cross-fetch'; import { Config } from '@backstage/config'; -import { - AzureIntegrationConfig, - getAzureRequestOptions, - ScmIntegrations, -} from '@backstage/integration'; +import { ScmIntegrations } from '@backstage/integration'; import { Logger } from 'winston'; import * as results from './results'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; +import { codeSearch } from './azure'; /** * Extracts repositories out of an Azure DevOps org. @@ -81,13 +77,16 @@ export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { `Reading Azure DevOps repositories from ${location.target}`, ); - const files = await this.search( + const files = await codeSearch( azureConfig, org, project, repo, catalogPath, ); + + this.logger.info(`Found ${files.length} files in Azure DevOps.`); + for (const file of files) { emit( results.location( @@ -105,42 +104,6 @@ export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { return true; } - - // search returns all files that matches the given search path. - async search( - azureConfig: AzureIntegrationConfig, - org: string, - project: string, - repo: string, - path: string, - ): Promise { - // TODO: What's the search URL for onpremises DevOps? - const searchUrl = `https://almsearch.dev.azure.com/${org}/${project}/_apis/search/codesearchresults?api-version=6.0-preview.1`; - const opts = getAzureRequestOptions(azureConfig); - const response = await fetch(searchUrl, { - method: 'POST', - headers: { - ...opts.headers, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - searchText: `path:${path} repo:${repo || '*'}`, - $top: 1000, - }), - }); - - if (response.status !== 200) { - this.logger.warn( - `Azure DevOps search failed with response status ${response.status}`, - ); - return []; - } - - const responseBody: CodeSearchResponse = await response.json(); - this.logger.info(`Azure DevOps search found ${responseBody.count} files`); - - return responseBody.results; - } } /** @@ -184,21 +147,3 @@ export function parseUrl(urlString: string): { throw new Error(`Failed to parse ${urlString}`); } - -interface CodeSearchResponse { - count: number; - results: CodeSearchResultItem[]; -} - -interface CodeSearchResultItem { - fileName: string; - path: string; - project: { - id: string; - name: string; - }; - repository: { - id: string; - name: string; - }; -} diff --git a/plugins/catalog-backend/src/ingestion/processors/azure/azure.test.ts b/plugins/catalog-backend/src/ingestion/processors/azure/azure.test.ts new file mode 100644 index 0000000000..d9dfe5d1f4 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/azure/azure.test.ts @@ -0,0 +1,140 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { setupServer } from 'msw/node'; +import { setupRequestMockHandlers } from '@backstage/test-utils'; +import { rest } from 'msw'; +import { codeSearch, CodeSearchResponse } from './azure'; + +describe('azure', () => { + const server = setupServer(); + setupRequestMockHandlers(server); + + describe('codeSearch', () => { + it('returns empty when nothing is found', async () => { + const response: CodeSearchResponse = { count: 0, results: [] }; + + server.use( + rest.post( + `https://almsearch.dev.azure.com/shopify/engineering/_apis/search/codesearchresults`, + (req, res, ctx) => { + expect(req.headers.get('Authorization')).toBe('Basic OkFCQw=='); + expect(req.body).toEqual({ + searchText: 'path:/catalog-info.yaml repo:*', + $top: 1000, + }); + return res(ctx.json(response)); + }, + ), + ); + + await expect( + codeSearch( + { host: 'dev.azure.com', token: 'ABC' }, + 'shopify', + 'engineering', + '', + '/catalog-info.yaml', + ), + ).resolves.toEqual([]); + }); + }); + + it('returns entries when request matches some files', async () => { + const response: CodeSearchResponse = { + count: 2, + results: [ + { + fileName: 'catalog-info.yaml', + path: '/catalog-info.yaml', + repository: { + name: 'backstage', + }, + }, + { + fileName: 'catalog-info.yaml', + path: '/catalog-info.yaml', + repository: { + name: 'ios-app', + }, + }, + ], + }; + + server.use( + rest.post( + `https://almsearch.dev.azure.com/shopify/engineering/_apis/search/codesearchresults`, + (req, res, ctx) => { + expect(req.headers.get('Authorization')).toBe('Basic OkFCQw=='); + expect(req.body).toEqual({ + searchText: 'path:/catalog-info.yaml repo:*', + $top: 1000, + }); + return res(ctx.json(response)); + }, + ), + ); + + await expect( + codeSearch( + { host: 'dev.azure.com', token: 'ABC' }, + 'shopify', + 'engineering', + '', + '/catalog-info.yaml', + ), + ).resolves.toEqual(response.results); + }); + + it('searches in specific repo if parameter is set', async () => { + const response: CodeSearchResponse = { + count: 1, + results: [ + { + fileName: 'catalog-info.yaml', + path: '/catalog-info.yaml', + repository: { + name: 'backstage', + }, + }, + ], + }; + + server.use( + rest.post( + `https://almsearch.dev.azure.com/shopify/engineering/_apis/search/codesearchresults`, + (req, res, ctx) => { + expect(req.headers.get('Authorization')).toBe('Basic OkFCQw=='); + expect(req.body).toEqual({ + searchText: 'path:/catalog-info.yaml repo:backstage', + $top: 1000, + }); + return res(ctx.json(response)); + }, + ), + ); + + await expect( + codeSearch( + { host: 'dev.azure.com', token: 'ABC' }, + 'shopify', + 'engineering', + 'backstage', + '/catalog-info.yaml', + ), + ).resolves.toEqual(response.results); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/azure/azure.ts b/plugins/catalog-backend/src/ingestion/processors/azure/azure.ts new file mode 100644 index 0000000000..ec1b9ba0fb --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/azure/azure.ts @@ -0,0 +1,67 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fetch from 'cross-fetch'; +import { + AzureIntegrationConfig, + getAzureRequestOptions, +} from '@backstage/integration'; + +export interface CodeSearchResponse { + count: number; + results: CodeSearchResultItem[]; +} + +export interface CodeSearchResultItem { + fileName: string; + path: string; + repository: { + name: string; + }; +} + +// codeSearch returns all files that matches the given search path. +export async function codeSearch( + azureConfig: AzureIntegrationConfig, + org: string, + project: string, + repo: string, + path: string, +): Promise { + // TODO: What's the search URL for onpremises DevOps? + const searchUrl = `https://almsearch.dev.azure.com/${org}/${project}/_apis/search/codesearchresults?api-version=6.0-preview.1`; + const opts = getAzureRequestOptions(azureConfig); + const response = await fetch(searchUrl, { + method: 'POST', + headers: { + ...opts.headers, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + searchText: `path:${path} repo:${repo || '*'}`, + $top: 1000, + }), + }); + + if (response.status !== 200) { + throw new Error( + `Azure DevOps search failed with response status ${response.status}`, + ); + } + + const responseBody: CodeSearchResponse = await response.json(); + return responseBody.results; +} diff --git a/plugins/catalog-backend/src/ingestion/processors/azure/index.ts b/plugins/catalog-backend/src/ingestion/processors/azure/index.ts new file mode 100644 index 0000000000..d4ff56c4c0 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/azure/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './azure'; From 563b039f0b6ea67ef7adc04912ef728af8117fd9 Mon Sep 17 00:00:00 2001 From: goenning Date: Sun, 21 Nov 2021 14:51:33 +0000 Subject: [PATCH 4/5] add api-report and changeset Signed-off-by: goenning --- .changeset/strong-seahorses-scream.md | 5 +++++ plugins/catalog-backend/api-report.md | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 .changeset/strong-seahorses-scream.md diff --git a/.changeset/strong-seahorses-scream.md b/.changeset/strong-seahorses-scream.md new file mode 100644 index 0000000000..445ed65c81 --- /dev/null +++ b/.changeset/strong-seahorses-scream.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Added Azure DevOps discovery processor diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 6660912dda..ece074fe94 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -174,6 +174,26 @@ export class AwsS3DiscoveryProcessor implements CatalogProcessor { ): Promise; } +// Warning: (ae-missing-release-tag) "AzureDevOpsDiscoveryProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { + constructor(options: { integrations: ScmIntegrations; logger: Logger_2 }); + // (undocumented) + static fromConfig( + config: Config, + options: { + logger: Logger_2; + }, + ): AzureDevOpsDiscoveryProcessor; + // (undocumented) + readLocation( + location: LocationSpec, + _optional: boolean, + emit: CatalogProcessorEmit, + ): Promise; +} + // Warning: (ae-missing-release-tag) "BitbucketDiscoveryProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) From f1ba6dfb1d7fa1dec651d9330a1ea2c17f1b5be1 Mon Sep 17 00:00:00 2001 From: goenning Date: Tue, 23 Nov 2021 19:03:46 +0000 Subject: [PATCH 5/5] fix comments, support onpremise and paging Signed-off-by: goenning --- .changeset/strong-seahorses-scream.md | 2 +- microsite/sidebars.json | 6 +- mkdocs.yml | 1 + .../AzureDevOpsDiscoveryProcessor.ts | 4 +- .../ingestion/processors/azure/azure.test.ts | 92 +++++++++++++++++++ .../src/ingestion/processors/azure/azure.ts | 55 ++++++----- 6 files changed, 136 insertions(+), 24 deletions(-) diff --git a/.changeset/strong-seahorses-scream.md b/.changeset/strong-seahorses-scream.md index 445ed65c81..bbd1d44378 100644 --- a/.changeset/strong-seahorses-scream.md +++ b/.changeset/strong-seahorses-scream.md @@ -1,5 +1,5 @@ --- -'@backstage/plugin-catalog-backend': minor +'@backstage/plugin-catalog-backend': patch --- Added Azure DevOps discovery processor diff --git a/microsite/sidebars.json b/microsite/sidebars.json index ec5c5d8d68..bc026ec196 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -124,7 +124,11 @@ { "type": "subcategory", "label": "Azure", - "ids": ["integrations/azure/locations", "integrations/azure/org"] + "ids": [ + "integrations/azure/locations", + "integrations/azure/discovery", + "integrations/azure/org" + ] }, { "type": "subcategory", diff --git a/mkdocs.yml b/mkdocs.yml index 8c7114abbc..08107f255d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -88,6 +88,7 @@ nav: - Discovery: 'integrations/aws-s3/discovery.md' - Azure: - Locations: 'integrations/azure/locations.md' + - Discovery: 'integrations/azure/discovery.md' - Org Data: 'integrations/azure/org.md' - Bitbucket: - Locations: 'integrations/bitbucket/locations.md' diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts index b86e3f9c82..58ee2155e9 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts @@ -85,7 +85,9 @@ export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { catalogPath, ); - this.logger.info(`Found ${files.length} files in Azure DevOps.`); + this.logger.debug( + `Found ${files.length} files in Azure DevOps from ${location.target}.`, + ); for (const file of files) { emit( diff --git a/plugins/catalog-backend/src/ingestion/processors/azure/azure.test.ts b/plugins/catalog-backend/src/ingestion/processors/azure/azure.test.ts index d9dfe5d1f4..67cc120bab 100644 --- a/plugins/catalog-backend/src/ingestion/processors/azure/azure.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/azure/azure.test.ts @@ -34,6 +34,7 @@ describe('azure', () => { expect(req.headers.get('Authorization')).toBe('Basic OkFCQw=='); expect(req.body).toEqual({ searchText: 'path:/catalog-info.yaml repo:*', + $skip: 0, $top: 1000, }); return res(ctx.json(response)); @@ -81,6 +82,7 @@ describe('azure', () => { expect(req.headers.get('Authorization')).toBe('Basic OkFCQw=='); expect(req.body).toEqual({ searchText: 'path:/catalog-info.yaml repo:*', + $skip: 0, $top: 1000, }); return res(ctx.json(response)); @@ -120,6 +122,7 @@ describe('azure', () => { expect(req.headers.get('Authorization')).toBe('Basic OkFCQw=='); expect(req.body).toEqual({ searchText: 'path:/catalog-info.yaml repo:backstage', + $skip: 0, $top: 1000, }); return res(ctx.json(response)); @@ -137,4 +140,93 @@ describe('azure', () => { ), ).resolves.toEqual(response.results); }); + + it('can search using onpremise api', async () => { + const response: CodeSearchResponse = { + count: 1, + results: [ + { + fileName: 'catalog-info.yaml', + path: '/catalog-info.yaml', + repository: { + name: 'backstage', + }, + }, + ], + }; + + server.use( + rest.post( + `https://azuredevops.mycompany.com/shopify/engineering/_apis/search/codesearchresults`, + (req, res, ctx) => { + expect(req.headers.get('Authorization')).toBe('Basic OkFCQw=='); + expect(req.body).toEqual({ + searchText: 'path:/catalog-info.yaml repo:*', + $skip: 0, + $top: 1000, + }); + return res(ctx.json(response)); + }, + ), + ); + + await expect( + codeSearch( + { host: 'azuredevops.mycompany.com', token: 'ABC' }, + 'shopify', + 'engineering', + '', + '/catalog-info.yaml', + ), + ).resolves.toEqual(response.results); + }); + + it('searches multiple pages if response contains many items', async () => { + const totalCount = 2401; + const generateItems = (count: number) => { + return Array.from(Array(count).keys()).map(_ => ({ + fileName: 'catalog-info.yaml', + path: '/catalog-info.yaml', + repository: { + name: 'backstage', + }, + })); + }; + + server.use( + rest.post( + `https://almsearch.dev.azure.com/shopify/engineering/_apis/search/codesearchresults`, + (req, res, ctx) => { + expect(req.headers.get('Authorization')).toBe('Basic OkFCQw=='); + expect(req.body).toMatchObject({ + searchText: 'path:/catalog-info.yaml repo:backstage', + $top: 1000, + }); + + const body = req.body as { $skip: number; $top: number }; + const countItemsToReturn = + body.$top + body.$skip > totalCount + ? totalCount - body.$skip + : body.$top; + + return res( + ctx.json({ + count: totalCount, + results: generateItems(countItemsToReturn), + }), + ); + }, + ), + ); + + await expect( + codeSearch( + { host: 'dev.azure.com', token: 'ABC' }, + 'shopify', + 'engineering', + 'backstage', + '/catalog-info.yaml', + ), + ).resolves.toHaveLength(totalCount); + }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/azure/azure.ts b/plugins/catalog-backend/src/ingestion/processors/azure/azure.ts index ec1b9ba0fb..0c7b17483a 100644 --- a/plugins/catalog-backend/src/ingestion/processors/azure/azure.ts +++ b/plugins/catalog-backend/src/ingestion/processors/azure/azure.ts @@ -33,6 +33,9 @@ export interface CodeSearchResultItem { }; } +const isCloud = (host: string) => host === 'dev.azure.com'; +const PAGE_SIZE = 1000; + // codeSearch returns all files that matches the given search path. export async function codeSearch( azureConfig: AzureIntegrationConfig, @@ -41,27 +44,37 @@ export async function codeSearch( repo: string, path: string, ): Promise { - // TODO: What's the search URL for onpremises DevOps? - const searchUrl = `https://almsearch.dev.azure.com/${org}/${project}/_apis/search/codesearchresults?api-version=6.0-preview.1`; - const opts = getAzureRequestOptions(azureConfig); - const response = await fetch(searchUrl, { - method: 'POST', - headers: { - ...opts.headers, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - searchText: `path:${path} repo:${repo || '*'}`, - $top: 1000, - }), - }); + const searchBaseUrl = isCloud(azureConfig.host) + ? 'https://almsearch.dev.azure.com' + : `https://${azureConfig.host}`; + const searchUrl = `${searchBaseUrl}/${org}/${project}/_apis/search/codesearchresults?api-version=6.0-preview.1`; - if (response.status !== 200) { - throw new Error( - `Azure DevOps search failed with response status ${response.status}`, - ); - } + let items: CodeSearchResultItem[] = []; + let hasMorePages = true; - const responseBody: CodeSearchResponse = await response.json(); - return responseBody.results; + do { + const response = await fetch(searchUrl, { + ...getAzureRequestOptions(azureConfig, { + 'Content-Type': 'application/json', + }), + method: 'POST', + body: JSON.stringify({ + searchText: `path:${path} repo:${repo || '*'}`, + $skip: items.length, + $top: PAGE_SIZE, + }), + }); + + if (response.status !== 200) { + throw new Error( + `Azure DevOps search failed with response status ${response.status}`, + ); + } + + const body: CodeSearchResponse = await response.json(); + items = [...items, ...body.results]; + hasMorePages = body.count > items.length; + } while (hasMorePages); + + return items; }