From ac724ea8aee28c2deec0099fa46347853113be5a Mon Sep 17 00:00:00 2001 From: goenning Date: Sun, 21 Nov 2021 14:45:41 +0000 Subject: [PATCH] adding tests Signed-off-by: goenning --- .../AzureDevOpsDiscoveryProcessor.test.ts | 218 +++++++++++++++++- .../AzureDevOpsDiscoveryProcessor.ts | 67 +----- .../ingestion/processors/azure/azure.test.ts | 140 +++++++++++ .../src/ingestion/processors/azure/azure.ts | 67 ++++++ .../src/ingestion/processors/azure/index.ts | 17 ++ 5 files changed, 446 insertions(+), 63 deletions(-) create mode 100644 plugins/catalog-backend/src/ingestion/processors/azure/azure.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/azure/azure.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/azure/index.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts index 9d468acf15..ab053e8748 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts @@ -14,7 +14,17 @@ * limitations under the License. */ -import { parseUrl } from './AzureDevOpsDiscoveryProcessor'; +import { getVoidLogger } from '@backstage/backend-common'; +import { LocationSpec } from '@backstage/catalog-model'; +import { ConfigReader } from '@backstage/config'; +import { codeSearch } from './azure'; +import { + AzureDevOpsDiscoveryProcessor, + parseUrl, +} from './AzureDevOpsDiscoveryProcessor'; + +jest.mock('./azure'); +const mockCodeSearch = codeSearch as jest.MockedFunction; describe('AzureDevOpsDiscoveryProcessor', () => { describe('parseUrl', () => { @@ -59,5 +69,209 @@ describe('AzureDevOpsDiscoveryProcessor', () => { }); }); - // TODO: add tests for AzureDevOpsDiscoveryProcessor + describe('reject unrelated entries', () => { + it('rejects unknown types', async () => { + const processor = AzureDevOpsDiscoveryProcessor.fromConfig( + new ConfigReader({ + integrations: { + azure: [{ host: 'dev.azure.com', token: 'blob' }], + }, + }), + { logger: getVoidLogger() }, + ); + const location: LocationSpec = { + type: 'not-azure-discovery', + target: 'https://dev.azure.com', + }; + await expect( + processor.readLocation(location, false, () => {}), + ).resolves.toBeFalsy(); + }); + }); + + it('rejects unknown targets', async () => { + const processor = AzureDevOpsDiscoveryProcessor.fromConfig( + new ConfigReader({ + integrations: { + github: [ + { host: 'dev.azure.com', token: 'blob' }, + { host: 'azure.myorg.com', token: 'blob' }, + ], + }, + }), + { logger: getVoidLogger() }, + ); + const location: LocationSpec = { + type: 'azure-discovery', + target: 'https://not.azure.com/org/project', + }; + await expect( + processor.readLocation(location, false, () => {}), + ).rejects.toThrow( + /There is no Azure integration that matches https:\/\/not.azure.com\/org\/project. Please add a configuration entry for it under integrations.azure/, + ); + }); + + describe('handles repositories', () => { + const processor = AzureDevOpsDiscoveryProcessor.fromConfig( + new ConfigReader({ + integrations: { + github: [{ host: 'dev.azure.com', token: 'blob' }], + }, + }), + { logger: getVoidLogger() }, + ); + + beforeEach(() => { + mockCodeSearch.mockClear(); + }); + + it('output all locations found on from code search', async () => { + const location: LocationSpec = { + type: 'azure-discovery', + target: 'https://dev.azure.com/shopify/engineering', + }; + mockCodeSearch.mockResolvedValueOnce([ + { + fileName: 'catalog-info.yaml', + path: '/catalog-info.yaml', + repository: { + name: 'backstage', + }, + }, + { + fileName: 'catalog-info.yaml', + path: '/src/catalog-info.yaml', + repository: { + name: 'ios-app', + }, + }, + ]); + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(mockCodeSearch).toHaveBeenCalledWith( + { host: 'dev.azure.com' }, + 'shopify', + 'engineering', + '', + '/catalog-info.yaml', + ); + expect(emitter).toHaveBeenCalledTimes(2); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://dev.azure.com/shopify/engineering/_git/backstage?path=/catalog-info.yaml', + }, + optional: true, + }); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://dev.azure.com/shopify/engineering/_git/ios-app?path=/src/catalog-info.yaml', + }, + optional: true, + }); + }); + + it('output single locations from code search', async () => { + const location: LocationSpec = { + type: 'azure-discovery', + target: 'https://dev.azure.com/shopify/engineering/_git/backstage', + }; + mockCodeSearch.mockResolvedValueOnce([ + { + fileName: 'catalog-info.yaml', + path: '/catalog-info.yaml', + repository: { + name: 'backstage', + }, + }, + ]); + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(mockCodeSearch).toHaveBeenCalledWith( + { host: 'dev.azure.com' }, + 'shopify', + 'engineering', + 'backstage', + '/catalog-info.yaml', + ); + expect(emitter).toHaveBeenCalledTimes(1); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://dev.azure.com/shopify/engineering/_git/backstage?path=/catalog-info.yaml', + }, + optional: true, + }); + }); + + it('output single locations with different file name from code search', async () => { + const location: LocationSpec = { + type: 'azure-discovery', + target: + 'https://dev.azure.com/shopify/engineering?path=/src/*/catalog.yaml', + }; + mockCodeSearch.mockResolvedValueOnce([ + { + fileName: 'catalog.yaml', + path: '/src/main/catalog.yaml', + repository: { + name: 'backstage', + }, + }, + ]); + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(mockCodeSearch).toHaveBeenCalledWith( + { host: 'dev.azure.com' }, + 'shopify', + 'engineering', + '', + '/src/*/catalog.yaml', + ); + expect(emitter).toHaveBeenCalledTimes(1); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://dev.azure.com/shopify/engineering/_git/backstage?path=/src/main/catalog.yaml', + }, + optional: true, + }); + }); + + it('output nothing when code search does not find anything', async () => { + const location: LocationSpec = { + type: 'azure-discovery', + target: 'https://dev.azure.com/shopify/engineering/_git/backstage', + }; + mockCodeSearch.mockResolvedValueOnce([]); + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(mockCodeSearch).toHaveBeenCalledWith( + { host: 'dev.azure.com' }, + 'shopify', + 'engineering', + 'backstage', + '/catalog-info.yaml', + ); + expect(emitter).not.toHaveBeenCalled(); + }); + }); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts index a938e230e7..b86e3f9c82 100644 --- a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts @@ -15,16 +15,12 @@ */ import { LocationSpec } from '@backstage/catalog-model'; -import fetch from 'cross-fetch'; import { Config } from '@backstage/config'; -import { - AzureIntegrationConfig, - getAzureRequestOptions, - ScmIntegrations, -} from '@backstage/integration'; +import { ScmIntegrations } from '@backstage/integration'; import { Logger } from 'winston'; import * as results from './results'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; +import { codeSearch } from './azure'; /** * Extracts repositories out of an Azure DevOps org. @@ -81,13 +77,16 @@ export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { `Reading Azure DevOps repositories from ${location.target}`, ); - const files = await this.search( + const files = await codeSearch( azureConfig, org, project, repo, catalogPath, ); + + this.logger.info(`Found ${files.length} files in Azure DevOps.`); + for (const file of files) { emit( results.location( @@ -105,42 +104,6 @@ export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { return true; } - - // search returns all files that matches the given search path. - async search( - azureConfig: AzureIntegrationConfig, - org: string, - project: string, - repo: string, - path: string, - ): Promise { - // TODO: What's the search URL for onpremises DevOps? - const searchUrl = `https://almsearch.dev.azure.com/${org}/${project}/_apis/search/codesearchresults?api-version=6.0-preview.1`; - const opts = getAzureRequestOptions(azureConfig); - const response = await fetch(searchUrl, { - method: 'POST', - headers: { - ...opts.headers, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - searchText: `path:${path} repo:${repo || '*'}`, - $top: 1000, - }), - }); - - if (response.status !== 200) { - this.logger.warn( - `Azure DevOps search failed with response status ${response.status}`, - ); - return []; - } - - const responseBody: CodeSearchResponse = await response.json(); - this.logger.info(`Azure DevOps search found ${responseBody.count} files`); - - return responseBody.results; - } } /** @@ -184,21 +147,3 @@ export function parseUrl(urlString: string): { throw new Error(`Failed to parse ${urlString}`); } - -interface CodeSearchResponse { - count: number; - results: CodeSearchResultItem[]; -} - -interface CodeSearchResultItem { - fileName: string; - path: string; - project: { - id: string; - name: string; - }; - repository: { - id: string; - name: string; - }; -} diff --git a/plugins/catalog-backend/src/ingestion/processors/azure/azure.test.ts b/plugins/catalog-backend/src/ingestion/processors/azure/azure.test.ts new file mode 100644 index 0000000000..d9dfe5d1f4 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/azure/azure.test.ts @@ -0,0 +1,140 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { setupServer } from 'msw/node'; +import { setupRequestMockHandlers } from '@backstage/test-utils'; +import { rest } from 'msw'; +import { codeSearch, CodeSearchResponse } from './azure'; + +describe('azure', () => { + const server = setupServer(); + setupRequestMockHandlers(server); + + describe('codeSearch', () => { + it('returns empty when nothing is found', async () => { + const response: CodeSearchResponse = { count: 0, results: [] }; + + server.use( + rest.post( + `https://almsearch.dev.azure.com/shopify/engineering/_apis/search/codesearchresults`, + (req, res, ctx) => { + expect(req.headers.get('Authorization')).toBe('Basic OkFCQw=='); + expect(req.body).toEqual({ + searchText: 'path:/catalog-info.yaml repo:*', + $top: 1000, + }); + return res(ctx.json(response)); + }, + ), + ); + + await expect( + codeSearch( + { host: 'dev.azure.com', token: 'ABC' }, + 'shopify', + 'engineering', + '', + '/catalog-info.yaml', + ), + ).resolves.toEqual([]); + }); + }); + + it('returns entries when request matches some files', async () => { + const response: CodeSearchResponse = { + count: 2, + results: [ + { + fileName: 'catalog-info.yaml', + path: '/catalog-info.yaml', + repository: { + name: 'backstage', + }, + }, + { + fileName: 'catalog-info.yaml', + path: '/catalog-info.yaml', + repository: { + name: 'ios-app', + }, + }, + ], + }; + + server.use( + rest.post( + `https://almsearch.dev.azure.com/shopify/engineering/_apis/search/codesearchresults`, + (req, res, ctx) => { + expect(req.headers.get('Authorization')).toBe('Basic OkFCQw=='); + expect(req.body).toEqual({ + searchText: 'path:/catalog-info.yaml repo:*', + $top: 1000, + }); + return res(ctx.json(response)); + }, + ), + ); + + await expect( + codeSearch( + { host: 'dev.azure.com', token: 'ABC' }, + 'shopify', + 'engineering', + '', + '/catalog-info.yaml', + ), + ).resolves.toEqual(response.results); + }); + + it('searches in specific repo if parameter is set', async () => { + const response: CodeSearchResponse = { + count: 1, + results: [ + { + fileName: 'catalog-info.yaml', + path: '/catalog-info.yaml', + repository: { + name: 'backstage', + }, + }, + ], + }; + + server.use( + rest.post( + `https://almsearch.dev.azure.com/shopify/engineering/_apis/search/codesearchresults`, + (req, res, ctx) => { + expect(req.headers.get('Authorization')).toBe('Basic OkFCQw=='); + expect(req.body).toEqual({ + searchText: 'path:/catalog-info.yaml repo:backstage', + $top: 1000, + }); + return res(ctx.json(response)); + }, + ), + ); + + await expect( + codeSearch( + { host: 'dev.azure.com', token: 'ABC' }, + 'shopify', + 'engineering', + 'backstage', + '/catalog-info.yaml', + ), + ).resolves.toEqual(response.results); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/azure/azure.ts b/plugins/catalog-backend/src/ingestion/processors/azure/azure.ts new file mode 100644 index 0000000000..ec1b9ba0fb --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/azure/azure.ts @@ -0,0 +1,67 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fetch from 'cross-fetch'; +import { + AzureIntegrationConfig, + getAzureRequestOptions, +} from '@backstage/integration'; + +export interface CodeSearchResponse { + count: number; + results: CodeSearchResultItem[]; +} + +export interface CodeSearchResultItem { + fileName: string; + path: string; + repository: { + name: string; + }; +} + +// codeSearch returns all files that matches the given search path. +export async function codeSearch( + azureConfig: AzureIntegrationConfig, + org: string, + project: string, + repo: string, + path: string, +): Promise { + // TODO: What's the search URL for onpremises DevOps? + const searchUrl = `https://almsearch.dev.azure.com/${org}/${project}/_apis/search/codesearchresults?api-version=6.0-preview.1`; + const opts = getAzureRequestOptions(azureConfig); + const response = await fetch(searchUrl, { + method: 'POST', + headers: { + ...opts.headers, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + searchText: `path:${path} repo:${repo || '*'}`, + $top: 1000, + }), + }); + + if (response.status !== 200) { + throw new Error( + `Azure DevOps search failed with response status ${response.status}`, + ); + } + + const responseBody: CodeSearchResponse = await response.json(); + return responseBody.results; +} diff --git a/plugins/catalog-backend/src/ingestion/processors/azure/index.ts b/plugins/catalog-backend/src/ingestion/processors/azure/index.ts new file mode 100644 index 0000000000..d4ff56c4c0 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/azure/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './azure';