paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 28 Packages Projects Releases Wiki Activity

Merge pull request #13387 from marcus-crane/github-catalog-topic-filter

Browse Source 
Add support to GithubEntityProvider for including (or excluding) Github repositories by topic
This commit is contained in:
Johan Haals
2022-09-05 11:13:21 +02:00
committed by GitHub
parent 78856a969c 4da5a2fc40
commit cd4fd0bec4
11 changed files with 549 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/shiny-ties-leave.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-catalog-backend-module-github': patch
---
Add support for including (or excluding) Github repositories by topic
docs/integrations/github/discovery.md
+26
View File
@@ -80,6 +80,22 @@ catalog:
filters: # optional filters
branch: 'develop' # optional string
repository: '.*' # optional Regex
topicProviderId:
organization: 'backstage' # string
catalogPath: '/catalog-info.yaml' # string
filters:
branch: 'main' # string
repository: '.*' # Regex
topic: 'backstage-exclude' # optional string
topicFilterProviderId:
organization: 'backstage' # string
catalogPath: '/catalog-info.yaml' # string
filters:
branch: 'main' # string
repository: '.*' # Regex
topic:
include: ['backstage-include'] # optional array of strings
exclude: ['experiments'] # optional array of strings
```
This provider supports multiple organizations via unique provider IDs.
@@ -96,6 +112,16 @@ This provider supports multiple organizations via unique provider IDs.
String used to filter results based on the branch name.
- **repository** _(optional)_:
Regular expression used to filter results based on the repository name.
- **topic** _(optional)_:
Both of the filters below may be used at the same time but the exclusion filter has the highest priority.
In the example above, a repository with the `backstage-include` topic would still be excluded
if it were also carrying the `experiments` topic.
- **include** _(optional)_:
An array of strings used to filter in results based on their associated Github topics.
If configured, only repositories with one (or more) topic(s) present in the inclusion filter will be ingested
- **exclude** _(optional)_:
An array of strings used to filter out results based on their associated Github topics.
If configured, all repositories _except_ those with one (or more) topics(s) present in the exclusion filter will be ingested.
- **organization**:
Name of your organization account/workspace.
If you want to add multiple organizations, you need to add one provider config each.
plugins/catalog-backend-module-github/src/lib/github.test.ts
+8
View File
@@ -207,6 +207,9 @@ describe('github', () => {
name: 'backstage',
url: 'https://github.com/backstage/backstage',
isArchived: false,
repositoryTopics: {
nodes: [{ topic: { name: 'blah' } }],
},
defaultBranchRef: {
name: 'main',
},
@@ -215,6 +218,7 @@ describe('github', () => {
name: 'demo',
url: 'https://github.com/backstage/demo',
isArchived: true,
repositoryTopics: { nodes: [] },
defaultBranchRef: {
name: 'main',
},
@@ -233,6 +237,9 @@ describe('github', () => {
name: 'backstage',
url: 'https://github.com/backstage/backstage',
isArchived: false,
repositoryTopics: {
nodes: [{ topic: { name: 'blah' } }],
},
defaultBranchRef: {
name: 'main',
},
@@ -241,6 +248,7 @@ describe('github', () => {
name: 'demo',
url: 'https://github.com/backstage/demo',
isArchived: true,
repositoryTopics: { nodes: [] },
defaultBranchRef: {
name: 'main',
},
plugins/catalog-backend-module-github/src/lib/github.ts
+20
View File
@@ -61,11 +61,22 @@ export type Repository = {
name: string;
url: string;
isArchived: boolean;
repositoryTopics: RepositoryTopics;
defaultBranchRef: {
name: string;
} | null;
};
type RepositoryTopics = {
nodes: TopicNodes[];
};
type TopicNodes = {
topic: {
name: string;
};
};
export type Connection<T> = {
pageInfo: PageInfo;
nodes: T[];
@@ -265,6 +276,15 @@ export async function getOrganizationRepositories(
name
url
isArchived
repositoryTopics(first: 100) {
nodes {
... on RepositoryTopic {
topic {
name
}
}
}
}
defaultBranchRef {
name
}
plugins/catalog-backend-module-github/src/lib/util.test.ts
+60 -1
View File
@@ -14,7 +14,8 @@
* limitations under the License.
*/
import { parseGitHubOrgUrl } from './util';
import { GithubTopicFilters } from '../providers/GitHubEntityProviderConfig';
import { parseGitHubOrgUrl, satisfiesTopicFilter } from './util';
describe('parseGitHubOrgUrl', () => {
it('only supports clean org urls, and decodes them', () => {
@@ -28,3 +29,61 @@ describe('parseGitHubOrgUrl', () => {
});
});
});
describe('satisfiesTopicFilter', () => {
it('handles cases where the filter will never apply', () => {
expect(satisfiesTopicFilter([], undefined)).toEqual(true);
expect(
satisfiesTopicFilter([], { include: undefined, exclude: undefined }),
).toEqual(true);
expect(satisfiesTopicFilter([], { include: [], exclude: [] })).toEqual(
true,
);
});
it('handles cases where include filter is configured', () => {
const filter: GithubTopicFilters = {
include: ['backstage-include', 'open-source'],
exclude: undefined,
};
expect(satisfiesTopicFilter([], filter)).toEqual(false);
expect(satisfiesTopicFilter(['blah', 'something-else'], filter)).toEqual(
false,
);
expect(satisfiesTopicFilter(['backstage-include'], filter)).toEqual(true);
expect(
satisfiesTopicFilter(['backstage-include', 'open-source'], filter),
).toEqual(true);
});
it('handles cases where exclude filter is configured', () => {
const filter: GithubTopicFilters = {
include: undefined,
exclude: ['backstage-exclude', 'experiment'],
};
expect(satisfiesTopicFilter([], filter)).toEqual(true);
expect(satisfiesTopicFilter(['blah', 'bleh'], filter)).toEqual(true);
expect(
satisfiesTopicFilter(['backstage-include', 'backstage-exclude'], filter),
).toEqual(false);
expect(
satisfiesTopicFilter(['experiment', 'backstage-include'], filter),
).toEqual(false);
});
it('handles cases where both exclusion filters are configured', () => {
const filter: GithubTopicFilters = {
include: ['backstage-include', 'blah'],
exclude: ['backstage-exclude', 'bleh'],
};
expect(satisfiesTopicFilter([], filter)).toEqual(false);
expect(satisfiesTopicFilter(['backstage-include'], filter)).toEqual(true);
expect(satisfiesTopicFilter(['blah', 'nothing'], filter)).toEqual(true);
expect(satisfiesTopicFilter(['backstage-exclude'], filter)).toEqual(false);
expect(satisfiesTopicFilter(['bleh, nothing'], filter)).toEqual(false);
expect(satisfiesTopicFilter(['abc123'], filter)).toEqual(false);
expect(
satisfiesTopicFilter(['backstage-include', 'backstage-exclude'], filter),
).toEqual(false);
});
});
plugins/catalog-backend-module-github/src/lib/util.ts
+53
View File
@@ -14,6 +14,8 @@
* limitations under the License.
*/
import { GithubTopicFilters } from '../providers/GitHubEntityProviderConfig';
export function parseGitHubOrgUrl(urlString: string): { org: string } {
const path = new URL(urlString).pathname.substr(1).split('/');
@@ -24,3 +26,54 @@ export function parseGitHubOrgUrl(urlString: string): { org: string } {
throw new Error(`Expected a URL pointing to /<org>`);
}
export function satisfiesTopicFilter(
topics: string[],
topicFilter: GithubTopicFilters | undefined,
): Boolean {
// We don't want to do anything if a filter is not configured (or configured but empty)
if (!topicFilter) return true;
if (!topicFilter.include && !topicFilter.exclude) return true;
if (!topicFilter.include?.length && !topicFilter.exclude?.length) return true;
// If topic.include is in use, a topic MUST be set that matches the inclusion
// filter in order for a repository to be ingested
if (topicFilter.include?.length && !topicFilter.exclude) {
for (const topic of topics) {
if (topicFilter.include.includes(topic)) return true;
}
return false;
}
// If topic.exclude is in use, all topics are included by default
// with anything matching the filter being discarded. It is technically
// possible for the filter to be explicitly empty meaning all repositories
// are ingested although doing so would be pointless.
if (!topicFilter.include && topicFilter.exclude?.length) {
if (!topics.length) return true;
for (const topic of topics) {
if (topicFilter.exclude.includes(topic)) return false;
}
return true;
}
// Now the tricky part is where we have both include and exclude configured.
// In this case, exclude wins out so we take everything matching the initial
// inclusion filter and from that group, we further discard anything matching
// the exclusion filter. If an item were to have a topic set in both filters,
// we expect it to be discarded in the end. The use case here is that is that
// you may want to retrieve all repos with the topic of team-a while excluding
// matching repos that are marked as experiments.
if (topicFilter.include && topicFilter.exclude) {
const matchesInclude = satisfiesTopicFilter(topics, {
include: topicFilter.include,
});
const matchesExclude = !satisfiesTopicFilter(topics, {
exclude: topicFilter.exclude,
});
if (matchesExclude) return false;
return matchesInclude;
}
// If the topic filter is somehow ever in a state that is not covered here, we
// will fail "open" so that Backstage is still usable as if the filter was
// not configured at all.
return true;
}
plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts
+13
View File
@@ -148,6 +148,7 @@ describe('GithubDiscoveryProcessor', () => {
{
name: 'backstage',
url: 'https://github.com/backstage/backstage',
repositoryTopics: { nodes: [] },
isArchived: false,
defaultBranchRef: {
name: 'master',
@@ -156,6 +157,7 @@ describe('GithubDiscoveryProcessor', () => {
{
name: 'demo',
url: 'https://github.com/backstage/demo',
repositoryTopics: { nodes: [] },
isArchived: false,
defaultBranchRef: {
name: 'main',
@@ -196,6 +198,7 @@ describe('GithubDiscoveryProcessor', () => {
{
name: 'backstage',
url: 'https://github.com/backstage/tech-docs',
repositoryTopics: { nodes: [] },
isArchived: false,
defaultBranchRef: {
name: 'main',
@@ -228,6 +231,7 @@ describe('GithubDiscoveryProcessor', () => {
{
name: 'backstage',
url: 'https://github.com/backstage/tech-docs',
repositoryTopics: { nodes: [] },
isArchived: false,
defaultBranchRef: null,
},
@@ -250,6 +254,7 @@ describe('GithubDiscoveryProcessor', () => {
{
name: 'backstage',
url: 'https://github.com/backstage/backstage',
repositoryTopics: { nodes: [] },
isArchived: false,
defaultBranchRef: {
name: 'master',
@@ -283,6 +288,7 @@ describe('GithubDiscoveryProcessor', () => {
{
name: 'backstage',
url: 'https://github.com/backstage/backstage',
repositoryTopics: { nodes: [] },
isArchived: false,
defaultBranchRef: {
name: 'main',
@@ -291,6 +297,7 @@ describe('GithubDiscoveryProcessor', () => {
{
name: 'techdocs-cli',
url: 'https://github.com/backstage/techdocs-cli',
repositoryTopics: { nodes: [] },
isArchived: false,
defaultBranchRef: {
name: 'main',
@@ -299,6 +306,7 @@ describe('GithubDiscoveryProcessor', () => {
{
name: 'techdocs-container',
url: 'https://github.com/backstage/techdocs-container',
repositoryTopics: { nodes: [] },
isArchived: false,
defaultBranchRef: {
name: 'main',
@@ -307,6 +315,7 @@ describe('GithubDiscoveryProcessor', () => {
{
name: 'techdocs-durp',
url: 'https://github.com/backstage/techdocs-durp',
repositoryTopics: { nodes: [] },
isArchived: false,
defaultBranchRef: null,
},
@@ -347,6 +356,7 @@ describe('GithubDiscoveryProcessor', () => {
name: 'abstest',
url: 'https://github.com/backstage/abctest',
isArchived: false,
repositoryTopics: { nodes: [] },
defaultBranchRef: {
name: 'main',
},
@@ -355,6 +365,7 @@ describe('GithubDiscoveryProcessor', () => {
name: 'test',
url: 'https://github.com/backstage/test',
isArchived: false,
repositoryTopics: { nodes: [] },
defaultBranchRef: {
name: 'main',
},
@@ -362,6 +373,7 @@ describe('GithubDiscoveryProcessor', () => {
{
name: 'test-archived',
url: 'https://github.com/backstage/test',
repositoryTopics: { nodes: [] },
isArchived: true,
defaultBranchRef: {
name: 'main',
@@ -370,6 +382,7 @@ describe('GithubDiscoveryProcessor', () => {
{
name: 'testxyz',
url: 'https://github.com/backstage/testxyz',
repositoryTopics: { nodes: [] },
isArchived: false,
defaultBranchRef: {
name: 'main',
plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts
+298 -1
View File
@@ -126,7 +126,7 @@ describe('GitHubEntityProvider', () => {
);
});
it('apply full update on scheduled execution', async () => {
it('apply full update on scheduled execution with basic filters', async () => {
const config = new ConfigReader({
catalog: {
providers: {
@@ -164,6 +164,176 @@ describe('GitHubEntityProvider', () => {
{
name: 'test-repo',
url: 'https://github.com/test-org/test-repo',
repositoryTopics: { nodes: [] },
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
],
}),
);
await provider.connect(entityProviderConnection);
const taskDef = schedule.getTasks()[0];
expect(taskDef.id).toEqual('github-provider:myProvider:refresh');
await (taskDef.fn as () => Promise<void>)();
const url = `https://github.com/test-org/test-repo/blob/main/custom/path/catalog-custom.yaml`;
const expectedEntities = [
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
annotations: {
'backstage.io/managed-by-location': `url:${url}`,
'backstage.io/managed-by-origin-location': `url:${url}`,
},
name: 'generated-5e4b9498097f15434e88c477cfba6c079aa8ca7f',
},
spec: {
presence: 'optional',
target: `${url}`,
type: 'url',
},
},
locationKey: 'github-provider:myProvider',
},
];
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
type: 'full',
entities: expectedEntities,
});
});
it('apply full update on scheduled execution with topic exclusion', async () => {
const config = new ConfigReader({
catalog: {
providers: {
github: {
myProvider: {
organization: 'test-org',
catalogPath: 'custom/path/catalog-custom.yaml',
filters: {
branch: 'main',
repository: 'test-.*',
topic: {
exclude: ['backstage-exclude'],
},
},
},
},
},
},
});
const schedule = new PersistingTaskRunner();
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
};
const provider = GitHubEntityProvider.fromConfig(config, {
logger,
schedule,
})[0];
const mockGetOrganizationRepositories = jest.spyOn(
helpers,
'getOrganizationRepositories',
);
mockGetOrganizationRepositories.mockReturnValue(
Promise.resolve({
repositories: [
{
name: 'test-repo',
url: 'https://github.com/test-org/test-repo',
repositoryTopics: {
nodes: [
{
topic: { name: 'backstage-exclude' },
},
{
topic: { name: 'neat-repos' },
},
],
},
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
],
}),
);
await provider.connect(entityProviderConnection);
const taskDef = schedule.getTasks()[0];
expect(taskDef.id).toEqual('github-provider:myProvider:refresh');
await (taskDef.fn as () => Promise<void>)();
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
type: 'full',
entities: [],
});
});
it('apply full update on scheduled execution with topic inclusion', async () => {
const config = new ConfigReader({
catalog: {
providers: {
github: {
myProvider: {
organization: 'test-org',
catalogPath: 'custom/path/catalog-custom.yaml',
filters: {
branch: 'main',
repository: 'test-.*',
topic: {
include: ['backstage-include'],
},
},
},
},
},
},
});
const schedule = new PersistingTaskRunner();
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
};
const provider = GitHubEntityProvider.fromConfig(config, {
logger,
schedule,
})[0];
const mockGetOrganizationRepositories = jest.spyOn(
helpers,
'getOrganizationRepositories',
);
mockGetOrganizationRepositories.mockReturnValue(
Promise.resolve({
repositories: [
{
name: 'test-repo',
url: 'https://github.com/test-org/test-repo',
repositoryTopics: {
nodes: [
{
topic: { name: 'backstage-include' },
},
{
topic: { name: 'fruits' },
},
],
},
isArchived: false,
defaultBranchRef: {
name: 'main',
@@ -209,3 +379,130 @@ describe('GitHubEntityProvider', () => {
});
});
});
it('apply full update on scheduled execution with topic exclusion taking priority over topic inclusion', async () => {
const config = new ConfigReader({
catalog: {
providers: {
github: {
myProvider: {
organization: 'test-org',
catalogPath: 'custom/path/catalog-custom.yaml',
filters: {
branch: 'main',
repository: 'test-.*',
topic: {
exclude: ['backstage-exclude'],
include: ['backstage-include'],
},
},
},
},
},
},
});
const schedule = new PersistingTaskRunner();
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
};
const provider = GitHubEntityProvider.fromConfig(config, {
logger,
schedule,
})[0];
const mockGetOrganizationRepositories = jest.spyOn(
helpers,
'getOrganizationRepositories',
);
mockGetOrganizationRepositories.mockReturnValue(
Promise.resolve({
repositories: [
{
name: 'test-repo',
url: 'https://github.com/test-org/test-repo',
repositoryTopics: {
nodes: [
{
topic: { name: 'backstage-include' },
},
],
},
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
{
name: 'test-repo-2',
url: 'https://github.com/test-org/test-repo-2',
repositoryTopics: {
nodes: [
{
topic: { name: 'backstage-include' },
},
{
topic: { name: 'backstage-exclude' },
},
],
},
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
{
name: 'test-repo-3',
url: 'https://github.com/test-org/test-repo-3',
repositoryTopics: {
nodes: [
{
topic: { name: 'backstage-exclude' },
},
],
},
isArchived: false,
defaultBranchRef: {
name: 'main',
},
},
],
}),
);
await provider.connect(entityProviderConnection);
const taskDef = schedule.getTasks()[0];
expect(taskDef.id).toEqual('github-provider:myProvider:refresh');
await (taskDef.fn as () => Promise<void>)();
const url = `https://github.com/test-org/test-repo/blob/main/custom/path/catalog-custom.yaml`;
const expectedEntities = [
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
annotations: {
'backstage.io/managed-by-location': `url:${url}`,
'backstage.io/managed-by-origin-location': `url:${url}`,
},
name: 'generated-5e4b9498097f15434e88c477cfba6c079aa8ca7f',
},
spec: {
presence: 'optional',
target: `${url}`,
type: 'url',
},
},
locationKey: 'github-provider:myProvider',
},
];
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
type: 'full',
entities: expectedEntities,
});
});
plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts
+6
View File
@@ -38,6 +38,7 @@ import {
GitHubEntityProviderConfig,
} from './GitHubEntityProviderConfig';
import { getOrganizationRepositories, Repository } from '../lib/github';
import { satisfiesTopicFilter } from '../lib/util';
/**
* Discovers catalog files located in [GitHub](https://github.com).
@@ -184,11 +185,16 @@ export class GitHubEntityProvider implements EntityProvider {
private matchesFilters(repositories: Repository[]) {
const repositoryFilter = this.config.filters?.repository;
const topicFilters = this.config.filters?.topic;
const matchingRepositories = repositories.filter(r => {
const repoTopics: string[] = r.repositoryTopics.nodes.map(
node => node.topic.name,
);
return (
!r.isArchived &&
(!repositoryFilter || repositoryFilter.test(r.name)) &&
satisfiesTopicFilter(repoTopics, topicFilters) &&
r.defaultBranchRef?.name
);
});
plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts
+44 -1
View File
@@ -68,6 +68,15 @@ describe('readProviderConfigs', () => {
branch: 'branch-name',
},
},
providerWithTopicFilter: {
organization: 'test-org5',
filters: {
topic: {
exclude: ['backstage-exclude'],
include: ['backstage-include'],
},
},
},
providerWithHost: {
organization: 'test-org1',
host: 'ghe.internal.com',
@@ -78,7 +87,7 @@ describe('readProviderConfigs', () => {
});
const providerConfigs = readProviderConfigs(config);
expect(providerConfigs).toHaveLength(5);
expect(providerConfigs).toHaveLength(6);
expect(providerConfigs[0]).toEqual({
id: 'providerOrganizationOnly',
organization: 'test-org1',
@@ -87,6 +96,10 @@ describe('readProviderConfigs', () => {
filters: {
repository: undefined,
branch: undefined,
topic: {
include: undefined,
exclude: undefined,
},
},
});
expect(providerConfigs[1]).toEqual({
@@ -97,6 +110,10 @@ describe('readProviderConfigs', () => {
filters: {
repository: undefined,
branch: undefined,
topic: {
include: undefined,
exclude: undefined,
},
},
});
expect(providerConfigs[2]).toEqual({
@@ -107,6 +124,10 @@ describe('readProviderConfigs', () => {
filters: {
repository: /^repository.*filter$/, // repo
branch: undefined, // branch
topic: {
include: undefined,
exclude: undefined,
},
},
});
expect(providerConfigs[3]).toEqual({
@@ -117,9 +138,27 @@ describe('readProviderConfigs', () => {
filters: {
repository: undefined,
branch: 'branch-name',
topic: {
include: undefined,
exclude: undefined,
},
},
});
expect(providerConfigs[4]).toEqual({
id: 'providerWithTopicFilter',
organization: 'test-org5',
catalogPath: '/catalog-info.yaml',
host: 'github.com',
filters: {
repository: undefined,
branch: undefined,
topic: {
include: ['backstage-include'],
exclude: ['backstage-exclude'],
},
},
});
expect(providerConfigs[5]).toEqual({
id: 'providerWithHost',
organization: 'test-org1',
catalogPath: '/catalog-info.yaml',
@@ -127,6 +166,10 @@ describe('readProviderConfigs', () => {
filters: {
repository: undefined,
branch: undefined,
topic: {
include: undefined,
exclude: undefined,
},
},
});
});
plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts
+16
View File
@@ -27,9 +27,15 @@ export type GitHubEntityProviderConfig = {
filters?: {
repository?: RegExp;
branch?: string;
topic?: GithubTopicFilters;
};
};
export type GithubTopicFilters = {
exclude?: string[];
include?: string[];
};
export function readProviderConfigs(
config: Config,
): GitHubEntityProviderConfig[] {
@@ -60,6 +66,12 @@ function readProviderConfig(
const host = config.getOptionalString('host') ?? 'github.com';
const repositoryPattern = config.getOptionalString('filters.repository');
const branchPattern = config.getOptionalString('filters.branch');
const topicFilterInclude = config?.getOptionalStringArray(
'filters.topic.include',
);
const topicFilterExclude = config?.getOptionalStringArray(
'filters.topic.exclude',
);
return {
id,
@@ -71,6 +83,10 @@ function readProviderConfig(
? compileRegExp(repositoryPattern)
: undefined,
branch: branchPattern || undefined,
topic: {
include: topicFilterInclude,
exclude: topicFilterExclude,
},
},
};
}