paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 46 Packages Projects Releases Wiki Activity

add support for search queries

Browse Source 
Signed-off-by: goenning <me@goenning.net>
This commit is contained in:
goenning
2021-11-20 21:20:47 +00:00
parent 0179bd1f13
commit 51e42bc2c8
3 changed files with 230 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/integrations/azure/discovery.md
+49
View File
@@ -0,0 +1,49 @@
---
id: discovery
title: Azure DevOps Discovery
sidebar_label: Discovery
# prettier-ignore
description: Automatically discovering catalog entities from repositories in an Azure DevOps organization
---
The Azure DevOps integration has a special discovery processor for discovering
catalog entities within an Azure DevOps. The processor will crawl the Azure
DevOps organization and register entities matching the configured path. This can
be useful as an alternative to static locations or manually adding things to the
catalog.
To use the discovery processor, you'll need a GitHub integration
[set up](locations.md) with a `AZURE_TOKEN`. Then you can add a location target
to the catalog configuration:
```yaml
catalog:
locations:
# Scan all repositories for a catalog-info.yaml in the root of the default branch
- type: azure-discovery
target: https://dev.azure.com/myorg/myproject
# Or use a custom pattern for a subset of all repositories with default repository
- type: azure-discovery
target: https://dev.azure.com/myorg/myproject/_git/service-*
# Or use a custom file format and location
- type: azure-discovery
target: https://dev.azure.com/myorg/myproject/_git/*?path=/src/*/catalog-info.yaml
```
Note the `azure-discovery` type, as this is not a regular `url` processor.
When using a custom pattern, the target is composed of five parts:
- The base instance URL, `https://dev.azure.com` in this case
- The organization name which is required, `myorg` in this case
- The project name which is required, `myproject` in this case
- The repository blob to scan, which accepts \* wildcard tokens and must be
added after `_git/`. This can simply be `*` to scan all repositories in the
project.
- The path within each repository to find the catalog YAML file. This will
usually be `/catalog-info.yaml`, `/src/*/catalog-info.yaml` or a similar
variation for catalog files stored in the root directory of each repository.
_Note:_ the path parameter follows the same rules as the search on Azure DevOps
web interface. For more details visit the
[official search documentation](https://docs.microsoft.com/en-us/azure/devops/project/search/get-started-search?view=azure-devops)
plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts
+63
View File
@@ -0,0 +1,63 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { parseUrl } from './AzureDevOpsDiscoveryProcessor';
describe('AzureDevOpsDiscoveryProcessor', () => {
describe('parseUrl', () => {
it('parses well formed URLs', () => {
expect(parseUrl('https://dev.azure.com/my-org/my-proj')).toEqual({
baseUrl: 'https://dev.azure.com',
org: 'my-org',
project: 'my-proj',
repo: '',
catalogPath: '/catalog-info.yaml',
});
expect(
parseUrl(
'https://dev.azure.com/spotify/engineering/_git/backstage?path=/catalog.yaml',
),
).toEqual({
baseUrl: 'https://dev.azure.com',
org: 'spotify',
project: 'engineering',
repo: 'backstage',
catalogPath: '/catalog.yaml',
});
expect(
parseUrl(
'https://azuredevops.mycompany.com/spotify/engineering/_git/backstage?path=/src/*/catalog.yaml',
),
).toEqual({
baseUrl: 'https://azuredevops.mycompany.com',
org: 'spotify',
project: 'engineering',
repo: 'backstage',
catalogPath: '/src/*/catalog.yaml',
});
});
it('throws on incorrectly formed URLs', () => {
expect(() => parseUrl('https://dev.azure.com')).toThrow();
expect(() => parseUrl('https://dev.azure.com//')).toThrow();
expect(() => parseUrl('https://dev.azure.com//foo')).toThrow();
});
});
// TODO: add tests for AzureDevOpsDiscoveryProcessor
});
plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts
+118 -42
View File
@@ -18,6 +18,7 @@ import { LocationSpec } from '@backstage/catalog-model';
import fetch from 'cross-fetch';
import { Config } from '@backstage/config';
import {
AzureIntegrationConfig,
getAzureRequestOptions,
ScmIntegrations,
} from '@backstage/integration';
@@ -26,7 +27,18 @@ import * as results from './results';
import { CatalogProcessor, CatalogProcessorEmit } from './types';
/**
* TODO
* Extracts repositories out of an Azure DevOps org.
*
* The following will create locations for all projects which have a catalog-info.yaml
* on the default branch. The first is shorthand for the second.
*
* target: "https://dev.azure.com/org/project"
* or
* target: https://dev.azure.com/org/project?path=/catalog-info.yaml
*
* You may also explicitly specify a single repo:
*
* target: https://dev.azure.com/org/project/_git/repo
**/
export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor {
private readonly integrations: ScmIntegrations;
@@ -62,13 +74,49 @@ export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor {
);
}
// TODO: extract this from configured URL
const { org, project } = { org: 'myOrg', project: 'myProject' };
const { baseUrl, org, project, repo, catalogPath } = parseUrl(
location.target,
);
this.logger.info(
`Reading Azure DevOps repositories from ${location.target}`,
);
// TODO: What's the search URL for self hosted DevOps?
const files = await this.search(
azureConfig,
org,
project,
repo,
catalogPath,
);
for (const file of files) {
emit(
results.location(
{
type: 'url',
target: `${baseUrl}/${org}/${project}/_git/${file.repository.name}?path=${file.path}`,
},
// Not all locations may actually exist, since the user defined them as a wildcard pattern.
// Thus, we emit them as optional and let the downstream processor find them while not outputting
// an error if it couldn't.
true,
),
);
}
return true;
}
// search returns all files that matches the given search path.
async search(
azureConfig: AzureIntegrationConfig,
org: string,
project: string,
repo: string,
path: string,
): Promise<CodeSearchResultItem[]> {
// TODO: What's the search URL for onpremises DevOps?
const searchUrl = `https://almsearch.dev.azure.com/${org}/${project}/_apis/search/codesearchresults?api-version=6.0-preview.1`;
const opts = getAzureRequestOptions(azureConfig);
const response = await fetch(searchUrl, {
method: 'POST',
headers: {
@@ -76,53 +124,81 @@ export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor {
'Content-Type': 'application/json',
},
body: JSON.stringify({
searchText: 'catalog-info.yaml',
searchText: `path:${path} repo:${repo || '*'}`,
$top: 1000,
}),
});
// TODO: more logging
if (response.status === 200) {
const responseBody: AzureDevOpsCodeSearchResults = await response.json();
// TODO: should we support different file names?
const matches = responseBody.results.filter(
r => r.fileName === 'catalog-info.yaml',
if (response.status !== 200) {
this.logger.warn(
`Azure DevOps search failed with response status ${response.status}`,
);
for (const match of matches) {
emit(
results.location(
{
type: 'url',
// TODO: Do we need to support non-default branches?
target: `${location.target}/_git/${match.repository.name}?path=${match.path}`,
},
// Not all locations may actually exist, since the user defined them as a wildcard pattern.
// Thus, we emit them as optional and let the downstream processor find them while not outputting
// an error if it couldn't.
true,
),
);
}
return [];
}
return true;
const responseBody: CodeSearchResponse = await response.json();
this.logger.info(`Azure DevOps search found ${responseBody.count} files`);
return responseBody.results;
}
}
interface AzureDevOpsCodeSearchResults {
count: number;
results: Array<{
fileName: string;
path: string;
project: {
id: string;
name: string;
/**
* parseUrl extracts segments from the Azure DevOps URL.
**/
export function parseUrl(urlString: string): {
baseUrl: string;
org: string;
project: string;
repo: string;
catalogPath: string;
} {
const url = new URL(urlString);
const path = url.pathname.substr(1).split('/');
const catalogPath = url.searchParams.get('path') || '/catalog-info.yaml';
if (path.length === 2 && path[0].length && path[1].length) {
return {
baseUrl: url.origin,
org: decodeURIComponent(path[0]),
project: decodeURIComponent(path[1]),
repo: '',
catalogPath,
};
repository: {
id: string;
name: string;
} else if (
path.length === 4 &&
path[0].length &&
path[1].length &&
path[2].length &&
path[3].length
) {
return {
baseUrl: url.origin,
org: decodeURIComponent(path[0]),
project: decodeURIComponent(path[1]),
repo: decodeURIComponent(path[3]),
catalogPath,
};
}>;
}
throw new Error(`Failed to parse ${urlString}`);
}
interface CodeSearchResponse {
count: number;
results: CodeSearchResultItem[];
}
interface CodeSearchResultItem {
fileName: string;
path: string;
project: {
id: string;
name: string;
};
repository: {
id: string;
name: string;
};
}