From 0179bd1f139b9e7383a965b0adf53075ad0eaf7a Mon Sep 17 00:00:00 2001 From: goenning Date: Sat, 20 Nov 2021 09:37:57 +0000 Subject: [PATCH] draft implementation for AzureDevOps Discovery Signed-off-by: goenning --- .../AzureDevOpsDiscoveryProcessor.ts | 128 ++++++++++++++++++ .../src/ingestion/processors/index.ts | 1 + .../src/legacy/service/CatalogBuilder.ts | 2 + .../src/service/NextCatalogBuilder.ts | 2 + 4 files changed, 133 insertions(+) create mode 100644 plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts new file mode 100644 index 0000000000..8300b503f3 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts @@ -0,0 +1,128 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import fetch from 'cross-fetch'; +import { Config } from '@backstage/config'; +import { + getAzureRequestOptions, + ScmIntegrations, +} from '@backstage/integration'; +import { Logger } from 'winston'; +import * as results from './results'; +import { CatalogProcessor, CatalogProcessorEmit } from './types'; + +/** + * TODO + **/ +export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { + private readonly integrations: ScmIntegrations; + private readonly logger: Logger; + + static fromConfig(config: Config, options: { logger: Logger }) { + const integrations = ScmIntegrations.fromConfig(config); + + return new AzureDevOpsDiscoveryProcessor({ + ...options, + integrations, + }); + } + + constructor(options: { integrations: ScmIntegrations; logger: Logger }) { + this.integrations = options.integrations; + this.logger = options.logger; + } + + async readLocation( + location: LocationSpec, + _optional: boolean, + emit: CatalogProcessorEmit, + ): Promise { + if (location.type !== 'azure-discovery') { + return false; + } + + const azureConfig = this.integrations.azure.byUrl(location.target)?.config; + if (!azureConfig) { + throw new Error( + `There is no Azure integration that matches ${location.target}. Please add a configuration entry for it under integrations.azure`, + ); + } + + // TODO: extract this from configured URL + const { org, project } = { org: 'myOrg', project: 'myProject' }; + + // TODO: What's the search URL for self hosted DevOps? + const searchUrl = `https://almsearch.dev.azure.com/${org}/${project}/_apis/search/codesearchresults?api-version=6.0-preview.1`; + const opts = getAzureRequestOptions(azureConfig); + + const response = await fetch(searchUrl, { + method: 'POST', + headers: { + ...opts.headers, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + searchText: 'catalog-info.yaml', + $top: 1000, + }), + }); + + // TODO: more logging + if (response.status === 200) { + const responseBody: AzureDevOpsCodeSearchResults = await response.json(); + + // TODO: should we support different file names? + const matches = responseBody.results.filter( + r => r.fileName === 'catalog-info.yaml', + ); + + for (const match of matches) { + emit( + results.location( + { + type: 'url', + // TODO: Do we need to support non-default branches? + target: `${location.target}/_git/${match.repository.name}?path=${match.path}`, + }, + // Not all locations may actually exist, since the user defined them as a wildcard pattern. + // Thus, we emit them as optional and let the downstream processor find them while not outputting + // an error if it couldn't. + true, + ), + ); + } + } + + return true; + } +} + +interface AzureDevOpsCodeSearchResults { + count: number; + results: Array<{ + fileName: string; + path: string; + project: { + id: string; + name: string; + }; + repository: { + id: string; + name: string; + }; + }>; +} diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index 4f3c502a8e..63feac51af 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -26,6 +26,7 @@ export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor'; export { CodeOwnersProcessor } from './CodeOwnersProcessor'; export { FileReaderProcessor } from './FileReaderProcessor'; export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor'; +export { AzureDevOpsDiscoveryProcessor } from './AzureDevOpsDiscoveryProcessor'; export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; export { GithubMultiOrgReaderProcessor } from './GithubMultiOrgReaderProcessor'; export { GitLabDiscoveryProcessor } from './GitLabDiscoveryProcessor'; diff --git a/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts b/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts index b8e45c7cf3..6fb52e245e 100644 --- a/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts @@ -42,6 +42,7 @@ import { CodeOwnersProcessor, FileReaderProcessor, GithubDiscoveryProcessor, + AzureDevOpsDiscoveryProcessor, GithubOrgReaderProcessor, GitLabDiscoveryProcessor, LocationEntityProcessor, @@ -317,6 +318,7 @@ export class CatalogBuilder { new FileReaderProcessor(), BitbucketDiscoveryProcessor.fromConfig(config, { logger }), GithubDiscoveryProcessor.fromConfig(config, { logger }), + AzureDevOpsDiscoveryProcessor.fromConfig(config, { logger }), GithubOrgReaderProcessor.fromConfig(config, { logger }), GitLabDiscoveryProcessor.fromConfig(config, { logger }), new UrlReaderProcessor({ reader, logger }), diff --git a/plugins/catalog-backend/src/service/NextCatalogBuilder.ts b/plugins/catalog-backend/src/service/NextCatalogBuilder.ts index bbda6f070c..a694030701 100644 --- a/plugins/catalog-backend/src/service/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/NextCatalogBuilder.ts @@ -44,6 +44,7 @@ import { CatalogProcessorParser, CodeOwnersProcessor, FileReaderProcessor, + AzureDevOpsDiscoveryProcessor, GithubDiscoveryProcessor, GithubOrgReaderProcessor, GitLabDiscoveryProcessor, @@ -291,6 +292,7 @@ export class NextCatalogBuilder { return [ new FileReaderProcessor(), BitbucketDiscoveryProcessor.fromConfig(config, { logger }), + AzureDevOpsDiscoveryProcessor.fromConfig(config, { logger }), GithubDiscoveryProcessor.fromConfig(config, { logger }), GithubOrgReaderProcessor.fromConfig(config, { logger }), GitLabDiscoveryProcessor.fromConfig(config, { logger }),