paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 41 Packages Projects Releases Wiki Activity

Merge pull request #6930 from RoadieHQ/default-branch-discovery-processor

Browse Source 
ability to discover catalog-infos from default branch
This commit is contained in:
Fredrik Adelöw
2021-08-24 19:05:01 +02:00
committed by GitHub
parent 32d5184f38 b000c79abf
commit 82df606aa2
5 changed files with 91 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/big-jars-repair.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-catalog-backend': patch
---
GitHub discovery processor adds support for discovering the default GitHub branch
plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.test.ts
+38 -3
View File
@@ -35,7 +35,8 @@ describe('GithubDiscoveryProcessor', () => {
org: 'foo',
host: 'github.com',
repoSearchPath: /^proj$/,
catalogPath: '/blob/master/catalog.yaml',
branch: 'master',
catalogPath: '/catalog.yaml',
});
expect(
parseUrl('https://github.com/foo/proj*/blob/master/catalog.yaml'),
@@ -43,14 +44,21 @@ describe('GithubDiscoveryProcessor', () => {
org: 'foo',
host: 'github.com',
repoSearchPath: /^proj.*$/,
catalogPath: '/blob/master/catalog.yaml',
branch: 'master',
catalogPath: '/catalog.yaml',
});
expect(parseUrl('https://github.com/foo')).toEqual({
org: 'foo',
host: 'github.com',
repoSearchPath: /^.*$/,
branch: '-',
catalogPath: '/catalog-info.yaml',
});
});
it('throws on incorrectly formed URLs', () => {
expect(() => parseUrl('https://github.com')).toThrow();
expect(() => parseUrl('https://github.com//')).toThrow();
expect(() => parseUrl('https://github.com/foo')).toThrow();
expect(() => parseUrl('https://github.com//foo')).toThrow();
expect(() => parseUrl('https://github.com/org/teams')).toThrow();
expect(() => parseUrl('https://github.com/org//teams')).toThrow();
@@ -125,11 +133,17 @@ describe('GithubDiscoveryProcessor', () => {
name: 'backstage',
url: 'https://github.com/backstage/backstage',
isArchived: false,
defaultBranchRef: {
name: 'master',
},
},
{
name: 'demo',
url: 'https://github.com/backstage/demo',
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
],
});
@@ -168,16 +182,25 @@ describe('GithubDiscoveryProcessor', () => {
name: 'backstage',
url: 'https://github.com/backstage/backstage',
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
{
name: 'techdocs-cli',
url: 'https://github.com/backstage/techdocs-cli',
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
{
name: 'techdocs-container',
url: 'https://github.com/backstage/techdocs-container',
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
],
});
@@ -215,21 +238,33 @@ describe('GithubDiscoveryProcessor', () => {
name: 'abstest',
url: 'https://github.com/backstage/abctest',
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
{
name: 'test',
url: 'https://github.com/backstage/test',
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
{
name: 'test-archived',
url: 'https://github.com/backstage/test',
isArchived: true,
defaultBranchRef: {
name: 'main',
},
},
{
name: 'testxyz',
url: 'https://github.com/backstage/testxyz',
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
],
});
plugins/catalog-backend/src/ingestion/processors/GithubDiscoveryProcessor.ts
+30 -4
View File
@@ -28,7 +28,18 @@ import { CatalogProcessor, CatalogProcessorEmit } from './types';
/**
* Extracts repositories out of a GitHub org.
*/
*
* The following will create locations for all projects which have a catalog-info.yaml
* on the default branch. The first is shorthand for the second.
*
* target: "https://github.com/backstage"
* or
* target: https://github.com/backstage/*\/blob/-/catalog-info.yaml
*
* You may also explicitly specify the source branch:
*
* target: https://github.com/backstage/*\/blob/main/catalog-info.yaml
**/
export class GithubDiscoveryProcessor implements CatalogProcessor {
private readonly integrations: ScmIntegrations;
private readonly logger: Logger;
@@ -65,7 +76,7 @@ export class GithubDiscoveryProcessor implements CatalogProcessor {
);
}
const { org, repoSearchPath, catalogPath, host } = parseUrl(
const { org, repoSearchPath, catalogPath, branch, host } = parseUrl(
location.target,
);
@@ -97,11 +108,14 @@ export class GithubDiscoveryProcessor implements CatalogProcessor {
);
for (const repository of matching) {
const path = `/blob/${
branch === '-' ? repository.defaultBranchRef.name : branch
}${catalogPath}`;
emit(
results.location(
{
type: 'url',
target: `${repository.url}${catalogPath}`,
target: `${repository.url}${path}`,
},
// Not all locations may actually exist, since the user defined them as a wildcard pattern.
// Thus, we emit them as optional and let the downstream processor find them while not outputting
@@ -123,19 +137,31 @@ export function parseUrl(urlString: string): {
org: string;
repoSearchPath: RegExp;
catalogPath: string;
branch: string;
host: string;
} {
const url = new URL(urlString);
const path = url.pathname.substr(1).split('/');
// /backstage/techdocs-*/blob/master/catalog-info.yaml
// can also be
// /backstage
if (path.length > 2 && path[0].length && path[1].length) {
return {
org: decodeURIComponent(path[0]),
repoSearchPath: escapeRegExp(decodeURIComponent(path[1])),
catalogPath: `/${decodeURIComponent(path.slice(2).join('/'))}`,
branch: decodeURIComponent(path[3]),
catalogPath: `/${decodeURIComponent(path.slice(4).join('/'))}`,
host: url.host,
};
} else if (path.length === 1 && path[0].length) {
return {
org: decodeURIComponent(path[0]),
host: url.host,
repoSearchPath: escapeRegExp('*'),
catalogPath: '/catalog-info.yaml',
branch: '-',
};
}
throw new Error(`Failed to parse ${urlString}`);
plugins/catalog-backend/src/ingestion/processors/github/github.test.ts
+12
View File
@@ -201,11 +201,17 @@ describe('github', () => {
name: 'backstage',
url: 'https://github.com/backstage/backstage',
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
{
name: 'demo',
url: 'https://github.com/backstage/demo',
isArchived: true,
defaultBranchRef: {
name: 'main',
},
},
],
pageInfo: {
@@ -221,11 +227,17 @@ describe('github', () => {
name: 'backstage',
url: 'https://github.com/backstage/backstage',
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
{
name: 'demo',
url: 'https://github.com/backstage/demo',
isArchived: true,
defaultBranchRef: {
name: 'main',
},
},
],
};
plugins/catalog-backend/src/ingestion/processors/github/github.ts
+6
View File
@@ -60,6 +60,9 @@ export type Repository = {
name: string;
url: string;
isArchived: boolean;
defaultBranchRef: {
name: string;
};
};
export type Connection<T> = {
@@ -252,6 +255,9 @@ export async function getOrganizationRepositories(
name
url
isArchived
defaultBranchRef {
name
}
}
pageInfo {
hasNextPage