paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 53 Packages Projects Releases Wiki Activity

Merge pull request #13387 from marcus-crane/github-catalog-topic-filter

Browse Source 
Add support to GithubEntityProvider for including (or excluding) Github repositories by topic
This commit is contained in:
Johan Haals
2022-09-05 11:13:21 +02:00
committed by GitHub
parent 78856a969c 4da5a2fc40
commit cd4fd0bec4
11 changed files with 549 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/catalog-backend-module-github/src/lib/github.test.ts
+8
View File
@@ -207,6 +207,9 @@ describe('github', () => {
name: 'backstage',
url: 'https://github.com/backstage/backstage',
isArchived: false,
repositoryTopics: {
nodes: [{ topic: { name: 'blah' } }],
},
defaultBranchRef: {
name: 'main',
},
@@ -215,6 +218,7 @@ describe('github', () => {
name: 'demo',
url: 'https://github.com/backstage/demo',
isArchived: true,
repositoryTopics: { nodes: [] },
defaultBranchRef: {
name: 'main',
},
@@ -233,6 +237,9 @@ describe('github', () => {
name: 'backstage',
url: 'https://github.com/backstage/backstage',
isArchived: false,
repositoryTopics: {
nodes: [{ topic: { name: 'blah' } }],
},
defaultBranchRef: {
name: 'main',
},
@@ -241,6 +248,7 @@ describe('github', () => {
name: 'demo',
url: 'https://github.com/backstage/demo',
isArchived: true,
repositoryTopics: { nodes: [] },
defaultBranchRef: {
name: 'main',
},
plugins/catalog-backend-module-github/src/lib/github.ts
+20
View File
@@ -61,11 +61,22 @@ export type Repository = {
name: string;
url: string;
isArchived: boolean;
repositoryTopics: RepositoryTopics;
defaultBranchRef: {
name: string;
} | null;
};
type RepositoryTopics = {
nodes: TopicNodes[];
};
type TopicNodes = {
topic: {
name: string;
};
};
export type Connection<T> = {
pageInfo: PageInfo;
nodes: T[];
@@ -265,6 +276,15 @@ export async function getOrganizationRepositories(
name
url
isArchived
repositoryTopics(first: 100) {
nodes {
... on RepositoryTopic {
topic {
name
}
}
}
}
defaultBranchRef {
name
}
plugins/catalog-backend-module-github/src/lib/util.test.ts
+60 -1
View File
@@ -14,7 +14,8 @@
* limitations under the License.
*/
import { parseGitHubOrgUrl } from './util';
import { GithubTopicFilters } from '../providers/GitHubEntityProviderConfig';
import { parseGitHubOrgUrl, satisfiesTopicFilter } from './util';
describe('parseGitHubOrgUrl', () => {
it('only supports clean org urls, and decodes them', () => {
@@ -28,3 +29,61 @@ describe('parseGitHubOrgUrl', () => {
});
});
});
describe('satisfiesTopicFilter', () => {
it('handles cases where the filter will never apply', () => {
expect(satisfiesTopicFilter([], undefined)).toEqual(true);
expect(
satisfiesTopicFilter([], { include: undefined, exclude: undefined }),
).toEqual(true);
expect(satisfiesTopicFilter([], { include: [], exclude: [] })).toEqual(
true,
);
});
it('handles cases where include filter is configured', () => {
const filter: GithubTopicFilters = {
include: ['backstage-include', 'open-source'],
exclude: undefined,
};
expect(satisfiesTopicFilter([], filter)).toEqual(false);
expect(satisfiesTopicFilter(['blah', 'something-else'], filter)).toEqual(
false,
);
expect(satisfiesTopicFilter(['backstage-include'], filter)).toEqual(true);
expect(
satisfiesTopicFilter(['backstage-include', 'open-source'], filter),
).toEqual(true);
});
it('handles cases where exclude filter is configured', () => {
const filter: GithubTopicFilters = {
include: undefined,
exclude: ['backstage-exclude', 'experiment'],
};
expect(satisfiesTopicFilter([], filter)).toEqual(true);
expect(satisfiesTopicFilter(['blah', 'bleh'], filter)).toEqual(true);
expect(
satisfiesTopicFilter(['backstage-include', 'backstage-exclude'], filter),
).toEqual(false);
expect(
satisfiesTopicFilter(['experiment', 'backstage-include'], filter),
).toEqual(false);
});
it('handles cases where both exclusion filters are configured', () => {
const filter: GithubTopicFilters = {
include: ['backstage-include', 'blah'],
exclude: ['backstage-exclude', 'bleh'],
};
expect(satisfiesTopicFilter([], filter)).toEqual(false);
expect(satisfiesTopicFilter(['backstage-include'], filter)).toEqual(true);
expect(satisfiesTopicFilter(['blah', 'nothing'], filter)).toEqual(true);
expect(satisfiesTopicFilter(['backstage-exclude'], filter)).toEqual(false);
expect(satisfiesTopicFilter(['bleh, nothing'], filter)).toEqual(false);
expect(satisfiesTopicFilter(['abc123'], filter)).toEqual(false);
expect(
satisfiesTopicFilter(['backstage-include', 'backstage-exclude'], filter),
).toEqual(false);
});
});
plugins/catalog-backend-module-github/src/lib/util.ts
+53
View File
@@ -14,6 +14,8 @@
* limitations under the License.
*/
import { GithubTopicFilters } from '../providers/GitHubEntityProviderConfig';
export function parseGitHubOrgUrl(urlString: string): { org: string } {
const path = new URL(urlString).pathname.substr(1).split('/');
@@ -24,3 +26,54 @@ export function parseGitHubOrgUrl(urlString: string): { org: string } {
throw new Error(`Expected a URL pointing to /<org>`);
}
export function satisfiesTopicFilter(
topics: string[],
topicFilter: GithubTopicFilters | undefined,
): Boolean {
// We don't want to do anything if a filter is not configured (or configured but empty)
if (!topicFilter) return true;
if (!topicFilter.include && !topicFilter.exclude) return true;
if (!topicFilter.include?.length && !topicFilter.exclude?.length) return true;
// If topic.include is in use, a topic MUST be set that matches the inclusion
// filter in order for a repository to be ingested
if (topicFilter.include?.length && !topicFilter.exclude) {
for (const topic of topics) {
if (topicFilter.include.includes(topic)) return true;
}
return false;
}
// If topic.exclude is in use, all topics are included by default
// with anything matching the filter being discarded. It is technically
// possible for the filter to be explicitly empty meaning all repositories
// are ingested although doing so would be pointless.
if (!topicFilter.include && topicFilter.exclude?.length) {
if (!topics.length) return true;
for (const topic of topics) {
if (topicFilter.exclude.includes(topic)) return false;
}
return true;
}
// Now the tricky part is where we have both include and exclude configured.
// In this case, exclude wins out so we take everything matching the initial
// inclusion filter and from that group, we further discard anything matching
// the exclusion filter. If an item were to have a topic set in both filters,
// we expect it to be discarded in the end. The use case here is that is that
// you may want to retrieve all repos with the topic of team-a while excluding
// matching repos that are marked as experiments.
if (topicFilter.include && topicFilter.exclude) {
const matchesInclude = satisfiesTopicFilter(topics, {
include: topicFilter.include,
});
const matchesExclude = !satisfiesTopicFilter(topics, {
exclude: topicFilter.exclude,
});
if (matchesExclude) return false;
return matchesInclude;
}
// If the topic filter is somehow ever in a state that is not covered here, we
// will fail "open" so that Backstage is still usable as if the filter was
// not configured at all.
return true;
}
plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts
+13
View File
@@ -148,6 +148,7 @@ describe('GithubDiscoveryProcessor', () => {
{
name: 'backstage',
url: 'https://github.com/backstage/backstage',
repositoryTopics: { nodes: [] },
isArchived: false,
defaultBranchRef: {
name: 'master',
@@ -156,6 +157,7 @@ describe('GithubDiscoveryProcessor', () => {
{
name: 'demo',
url: 'https://github.com/backstage/demo',
repositoryTopics: { nodes: [] },
isArchived: false,
defaultBranchRef: {
name: 'main',
@@ -196,6 +198,7 @@ describe('GithubDiscoveryProcessor', () => {
{
name: 'backstage',
url: 'https://github.com/backstage/tech-docs',
repositoryTopics: { nodes: [] },
isArchived: false,
defaultBranchRef: {
name: 'main',
@@ -228,6 +231,7 @@ describe('GithubDiscoveryProcessor', () => {
{
name: 'backstage',
url: 'https://github.com/backstage/tech-docs',
repositoryTopics: { nodes: [] },
isArchived: false,
defaultBranchRef: null,
},
@@ -250,6 +254,7 @@ describe('GithubDiscoveryProcessor', () => {
{
name: 'backstage',
url: 'https://github.com/backstage/backstage',
repositoryTopics: { nodes: [] },
isArchived: false,
defaultBranchRef: {
name: 'master',
@@ -283,6 +288,7 @@ describe('GithubDiscoveryProcessor', () => {
{
name: 'backstage',
url: 'https://github.com/backstage/backstage',
repositoryTopics: { nodes: [] },
isArchived: false,
defaultBranchRef: {
name: 'main',
@@ -291,6 +297,7 @@ describe('GithubDiscoveryProcessor', () => {
{
name: 'techdocs-cli',
url: 'https://github.com/backstage/techdocs-cli',
repositoryTopics: { nodes: [] },
isArchived: false,
defaultBranchRef: {
name: 'main',
@@ -299,6 +306,7 @@ describe('GithubDiscoveryProcessor', () => {
{
name: 'techdocs-container',
url: 'https://github.com/backstage/techdocs-container',
repositoryTopics: { nodes: [] },
isArchived: false,
defaultBranchRef: {
name: 'main',
@@ -307,6 +315,7 @@ describe('GithubDiscoveryProcessor', () => {
{
name: 'techdocs-durp',
url: 'https://github.com/backstage/techdocs-durp',
repositoryTopics: { nodes: [] },
isArchived: false,
defaultBranchRef: null,
},
@@ -347,6 +356,7 @@ describe('GithubDiscoveryProcessor', () => {
name: 'abstest',
url: 'https://github.com/backstage/abctest',
isArchived: false,
repositoryTopics: { nodes: [] },
defaultBranchRef: {
name: 'main',
},
@@ -355,6 +365,7 @@ describe('GithubDiscoveryProcessor', () => {
name: 'test',
url: 'https://github.com/backstage/test',
isArchived: false,
repositoryTopics: { nodes: [] },
defaultBranchRef: {
name: 'main',
},
@@ -362,6 +373,7 @@ describe('GithubDiscoveryProcessor', () => {
{
name: 'test-archived',
url: 'https://github.com/backstage/test',
repositoryTopics: { nodes: [] },
isArchived: true,
defaultBranchRef: {
name: 'main',
@@ -370,6 +382,7 @@ describe('GithubDiscoveryProcessor', () => {
{
name: 'testxyz',
url: 'https://github.com/backstage/testxyz',
repositoryTopics: { nodes: [] },
isArchived: false,
defaultBranchRef: {
name: 'main',
plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts
+298 -1
View File
@@ -126,7 +126,7 @@ describe('GitHubEntityProvider', () => {
);
});
it('apply full update on scheduled execution', async () => {
it('apply full update on scheduled execution with basic filters', async () => {
const config = new ConfigReader({
catalog: {
providers: {
@@ -164,6 +164,176 @@ describe('GitHubEntityProvider', () => {
{
name: 'test-repo',
url: 'https://github.com/test-org/test-repo',
repositoryTopics: { nodes: [] },
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
],
}),
);
await provider.connect(entityProviderConnection);
const taskDef = schedule.getTasks()[0];
expect(taskDef.id).toEqual('github-provider:myProvider:refresh');
await (taskDef.fn as () => Promise<void>)();
const url = `https://github.com/test-org/test-repo/blob/main/custom/path/catalog-custom.yaml`;
const expectedEntities = [
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
annotations: {
'backstage.io/managed-by-location': `url:${url}`,
'backstage.io/managed-by-origin-location': `url:${url}`,
},
name: 'generated-5e4b9498097f15434e88c477cfba6c079aa8ca7f',
},
spec: {
presence: 'optional',
target: `${url}`,
type: 'url',
},
},
locationKey: 'github-provider:myProvider',
},
];
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
type: 'full',
entities: expectedEntities,
});
});
it('apply full update on scheduled execution with topic exclusion', async () => {
const config = new ConfigReader({
catalog: {
providers: {
github: {
myProvider: {
organization: 'test-org',
catalogPath: 'custom/path/catalog-custom.yaml',
filters: {
branch: 'main',
repository: 'test-.*',
topic: {
exclude: ['backstage-exclude'],
},
},
},
},
},
},
});
const schedule = new PersistingTaskRunner();
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
};
const provider = GitHubEntityProvider.fromConfig(config, {
logger,
schedule,
})[0];
const mockGetOrganizationRepositories = jest.spyOn(
helpers,
'getOrganizationRepositories',
);
mockGetOrganizationRepositories.mockReturnValue(
Promise.resolve({
repositories: [
{
name: 'test-repo',
url: 'https://github.com/test-org/test-repo',
repositoryTopics: {
nodes: [
{
topic: { name: 'backstage-exclude' },
},
{
topic: { name: 'neat-repos' },
},
],
},
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
],
}),
);
await provider.connect(entityProviderConnection);
const taskDef = schedule.getTasks()[0];
expect(taskDef.id).toEqual('github-provider:myProvider:refresh');
await (taskDef.fn as () => Promise<void>)();
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
type: 'full',
entities: [],
});
});
it('apply full update on scheduled execution with topic inclusion', async () => {
const config = new ConfigReader({
catalog: {
providers: {
github: {
myProvider: {
organization: 'test-org',
catalogPath: 'custom/path/catalog-custom.yaml',
filters: {
branch: 'main',
repository: 'test-.*',
topic: {
include: ['backstage-include'],
},
},
},
},
},
},
});
const schedule = new PersistingTaskRunner();
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
};
const provider = GitHubEntityProvider.fromConfig(config, {
logger,
schedule,
})[0];
const mockGetOrganizationRepositories = jest.spyOn(
helpers,
'getOrganizationRepositories',
);
mockGetOrganizationRepositories.mockReturnValue(
Promise.resolve({
repositories: [
{
name: 'test-repo',
url: 'https://github.com/test-org/test-repo',
repositoryTopics: {
nodes: [
{
topic: { name: 'backstage-include' },
},
{
topic: { name: 'fruits' },
},
],
},
isArchived: false,
defaultBranchRef: {
name: 'main',
@@ -209,3 +379,130 @@ describe('GitHubEntityProvider', () => {
});
});
});
it('apply full update on scheduled execution with topic exclusion taking priority over topic inclusion', async () => {
const config = new ConfigReader({
catalog: {
providers: {
github: {
myProvider: {
organization: 'test-org',
catalogPath: 'custom/path/catalog-custom.yaml',
filters: {
branch: 'main',
repository: 'test-.*',
topic: {
exclude: ['backstage-exclude'],
include: ['backstage-include'],
},
},
},
},
},
},
});
const schedule = new PersistingTaskRunner();
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
};
const provider = GitHubEntityProvider.fromConfig(config, {
logger,
schedule,
})[0];
const mockGetOrganizationRepositories = jest.spyOn(
helpers,
'getOrganizationRepositories',
);
mockGetOrganizationRepositories.mockReturnValue(
Promise.resolve({
repositories: [
{
name: 'test-repo',
url: 'https://github.com/test-org/test-repo',
repositoryTopics: {
nodes: [
{
topic: { name: 'backstage-include' },
},
],
},
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
{
name: 'test-repo-2',
url: 'https://github.com/test-org/test-repo-2',
repositoryTopics: {
nodes: [
{
topic: { name: 'backstage-include' },
},
{
topic: { name: 'backstage-exclude' },
},
],
},
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
{
name: 'test-repo-3',
url: 'https://github.com/test-org/test-repo-3',
repositoryTopics: {
nodes: [
{
topic: { name: 'backstage-exclude' },
},
],
},
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
],
}),
);
await provider.connect(entityProviderConnection);
const taskDef = schedule.getTasks()[0];
expect(taskDef.id).toEqual('github-provider:myProvider:refresh');
await (taskDef.fn as () => Promise<void>)();
const url = `https://github.com/test-org/test-repo/blob/main/custom/path/catalog-custom.yaml`;
const expectedEntities = [
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
annotations: {
'backstage.io/managed-by-location': `url:${url}`,
'backstage.io/managed-by-origin-location': `url:${url}`,
},
name: 'generated-5e4b9498097f15434e88c477cfba6c079aa8ca7f',
},
spec: {
presence: 'optional',
target: `${url}`,
type: 'url',
},
},
locationKey: 'github-provider:myProvider',
},
];
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
type: 'full',
entities: expectedEntities,
});
});
plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts
+6
View File
@@ -38,6 +38,7 @@ import {
GitHubEntityProviderConfig,
} from './GitHubEntityProviderConfig';
import { getOrganizationRepositories, Repository } from '../lib/github';
import { satisfiesTopicFilter } from '../lib/util';
/**
* Discovers catalog files located in [GitHub](https://github.com).
@@ -184,11 +185,16 @@ export class GitHubEntityProvider implements EntityProvider {
private matchesFilters(repositories: Repository[]) {
const repositoryFilter = this.config.filters?.repository;
const topicFilters = this.config.filters?.topic;
const matchingRepositories = repositories.filter(r => {
const repoTopics: string[] = r.repositoryTopics.nodes.map(
node => node.topic.name,
);
return (
!r.isArchived &&
(!repositoryFilter || repositoryFilter.test(r.name)) &&
satisfiesTopicFilter(repoTopics, topicFilters) &&
r.defaultBranchRef?.name
);
});
plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts
+44 -1
View File
@@ -68,6 +68,15 @@ describe('readProviderConfigs', () => {
branch: 'branch-name',
},
},
providerWithTopicFilter: {
organization: 'test-org5',
filters: {
topic: {
exclude: ['backstage-exclude'],
include: ['backstage-include'],
},
},
},
providerWithHost: {
organization: 'test-org1',
host: 'ghe.internal.com',
@@ -78,7 +87,7 @@ describe('readProviderConfigs', () => {
});
const providerConfigs = readProviderConfigs(config);
expect(providerConfigs).toHaveLength(5);
expect(providerConfigs).toHaveLength(6);
expect(providerConfigs[0]).toEqual({
id: 'providerOrganizationOnly',
organization: 'test-org1',
@@ -87,6 +96,10 @@ describe('readProviderConfigs', () => {
filters: {
repository: undefined,
branch: undefined,
topic: {
include: undefined,
exclude: undefined,
},
},
});
expect(providerConfigs[1]).toEqual({
@@ -97,6 +110,10 @@ describe('readProviderConfigs', () => {
filters: {
repository: undefined,
branch: undefined,
topic: {
include: undefined,
exclude: undefined,
},
},
});
expect(providerConfigs[2]).toEqual({
@@ -107,6 +124,10 @@ describe('readProviderConfigs', () => {
filters: {
repository: /^repository.*filter$/, // repo
branch: undefined, // branch
topic: {
include: undefined,
exclude: undefined,
},
},
});
expect(providerConfigs[3]).toEqual({
@@ -117,9 +138,27 @@ describe('readProviderConfigs', () => {
filters: {
repository: undefined,
branch: 'branch-name',
topic: {
include: undefined,
exclude: undefined,
},
},
});
expect(providerConfigs[4]).toEqual({
id: 'providerWithTopicFilter',
organization: 'test-org5',
catalogPath: '/catalog-info.yaml',
host: 'github.com',
filters: {
repository: undefined,
branch: undefined,
topic: {
include: ['backstage-include'],
exclude: ['backstage-exclude'],
},
},
});
expect(providerConfigs[5]).toEqual({
id: 'providerWithHost',
organization: 'test-org1',
catalogPath: '/catalog-info.yaml',
@@ -127,6 +166,10 @@ describe('readProviderConfigs', () => {
filters: {
repository: undefined,
branch: undefined,
topic: {
include: undefined,
exclude: undefined,
},
},
});
});
plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts
+16
View File
@@ -27,9 +27,15 @@ export type GitHubEntityProviderConfig = {
filters?: {
repository?: RegExp;
branch?: string;
topic?: GithubTopicFilters;
};
};
export type GithubTopicFilters = {
exclude?: string[];
include?: string[];
};
export function readProviderConfigs(
config: Config,
): GitHubEntityProviderConfig[] {
@@ -60,6 +66,12 @@ function readProviderConfig(
const host = config.getOptionalString('host') ?? 'github.com';
const repositoryPattern = config.getOptionalString('filters.repository');
const branchPattern = config.getOptionalString('filters.branch');
const topicFilterInclude = config?.getOptionalStringArray(
'filters.topic.include',
);
const topicFilterExclude = config?.getOptionalStringArray(
'filters.topic.exclude',
);
return {
id,
@@ -71,6 +83,10 @@ function readProviderConfig(
? compileRegExp(repositoryPattern)
: undefined,
branch: branchPattern || undefined,
topic: {
include: topicFilterInclude,
exclude: topicFilterExclude,
},
},
};
}