paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 35 Packages Projects Releases Wiki Activity

feat(catalog-backend-module-github): add visibility filter to github plugin

Browse Source 
Signed-off-by: TANGUY Antoine <tanguyantoine@users.noreply.github.com>
This commit is contained in:
TANGUY Antoine
2023-03-26 20:57:37 +02:00
parent ac7529636f
commit 3756dd98fe
11 changed files with 111 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/eleven-balloons-occur.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/plugin-catalog-backend-module-github': patch
---
Add visibility filter on GithubEntityProvider
docs/integrations/github/discovery.md
+7
View File
@@ -163,6 +163,13 @@ catalog:
branch: 'main' # string
repository: '.*' # Regex
validateLocationsExist: true # optional boolean
visibilityProviderId:
organization: 'backstage' # string
catalogPath: '/catalog-info.yaml' # string
filters:
visibilities:
- public
- internal
enterpriseProviderId:
host: ghe.example.net
organization: 'backstage' # string
plugins/catalog-backend-module-github/src/lib/github.test.ts
+4
View File
@@ -493,6 +493,7 @@ describe('github', () => {
name: 'main',
},
catalogInfoFile: null,
visibility: 'public',
},
{
name: 'demo',
@@ -508,6 +509,7 @@ describe('github', () => {
id: 'acb123',
text: 'some yaml',
},
visibility: 'private',
},
],
pageInfo: {
@@ -531,6 +533,7 @@ describe('github', () => {
name: 'main',
},
catalogInfoFile: null,
visibility: 'public',
},
{
name: 'demo',
@@ -546,6 +549,7 @@ describe('github', () => {
id: 'acb123',
text: 'some yaml',
},
visibility: 'private',
},
],
};
plugins/catalog-backend-module-github/src/lib/github.ts
+2
View File
@@ -99,6 +99,7 @@ export type RepositoryResponse = {
id: string;
text: string;
} | null;
visibility: string;
};
type RepositoryTopics = {
@@ -437,6 +438,7 @@ export async function getOrganizationRepositories(
url
isArchived
isFork
visibility
repositoryTopics(first: 100) {
nodes {
... on RepositoryTopic {
plugins/catalog-backend-module-github/src/lib/util.test.ts
+12
View File
@@ -19,6 +19,7 @@ import {
parseGithubOrgUrl,
satisfiesTopicFilter,
satisfiesForkFilter,
satisfiesVisibilityFilter,
} from './util';
describe('parseGithubOrgUrl', () => {
@@ -109,3 +110,14 @@ describe('satisfiesForkFilter', () => {
expect(satisfiesForkFilter(true, false)).toEqual(true);
});
});
describe('satisfiesVisibilityFilter', () => {
it('satisfies if no visibilities are given', () => {
expect(satisfiesVisibilityFilter([], 'public')).toEqual(true);
});
it('handles visibility presence into the visibilities filter', () => {
expect(satisfiesVisibilityFilter(['public'], 'public')).toEqual(true);
expect(satisfiesVisibilityFilter(['private'], 'public')).toEqual(false);
});
});
plugins/catalog-backend-module-github/src/lib/util.ts
+10
View File
@@ -99,3 +99,13 @@ export function splitTeamSlug(slug: string): [string, string] {
}
return [parts[0], parts[1]];
}
export function satisfiesVisibilityFilter(
visibilities: string[],
visibility: string,
): Boolean {
if (!visibilities.length) {
return true;
}
return visibilities.indexOf(visibility) > -1;
}
plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts
+13
View File
@@ -155,6 +155,7 @@ describe('GithubDiscoveryProcessor', () => {
name: 'master',
},
catalogInfoFile: null,
visibility: 'public',
},
{
name: 'demo',
@@ -166,6 +167,7 @@ describe('GithubDiscoveryProcessor', () => {
name: 'main',
},
catalogInfoFile: null,
visibility: 'public',
},
],
});
@@ -209,6 +211,7 @@ describe('GithubDiscoveryProcessor', () => {
name: 'main',
},
catalogInfoFile: null,
visibility: 'public',
},
],
});
@@ -242,6 +245,7 @@ describe('GithubDiscoveryProcessor', () => {
isFork: false,
defaultBranchRef: null,
catalogInfoFile: null,
visibility: 'public',
},
],
});
@@ -269,6 +273,7 @@ describe('GithubDiscoveryProcessor', () => {
name: 'master',
},
catalogInfoFile: null,
visibility: 'public',
},
],
});
@@ -305,6 +310,7 @@ describe('GithubDiscoveryProcessor', () => {
name: 'main',
},
catalogInfoFile: null,
visibility: 'public',
},
{
name: 'techdocs-cli',
@@ -316,6 +322,7 @@ describe('GithubDiscoveryProcessor', () => {
name: 'main',
},
catalogInfoFile: null,
visibility: 'public',
},
{
name: 'techdocs-container',
@@ -327,6 +334,7 @@ describe('GithubDiscoveryProcessor', () => {
name: 'main',
},
catalogInfoFile: null,
visibility: 'public',
},
{
name: 'techdocs-durp',
@@ -336,6 +344,7 @@ describe('GithubDiscoveryProcessor', () => {
isFork: false,
defaultBranchRef: null,
catalogInfoFile: null,
visibility: 'public',
},
],
});
@@ -380,6 +389,7 @@ describe('GithubDiscoveryProcessor', () => {
name: 'main',
},
catalogInfoFile: null,
visibility: 'public',
},
{
name: 'test',
@@ -391,6 +401,7 @@ describe('GithubDiscoveryProcessor', () => {
name: 'main',
},
catalogInfoFile: null,
visibility: 'public',
},
{
name: 'test-archived',
@@ -402,6 +413,7 @@ describe('GithubDiscoveryProcessor', () => {
name: 'main',
},
catalogInfoFile: null,
visibility: 'public',
},
{
name: 'testxyz',
@@ -413,6 +425,7 @@ describe('GithubDiscoveryProcessor', () => {
name: 'main',
},
catalogInfoFile: null,
visibility: 'public',
},
],
});
plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts
+8
View File
@@ -181,6 +181,7 @@ describe('GithubEntityProvider', () => {
id: 'abc123',
text: 'some yaml',
},
visibility: 'public',
},
],
}),
@@ -284,6 +285,7 @@ describe('GithubEntityProvider', () => {
id: 'abc123',
text: 'some yaml',
},
visibility: 'public',
},
],
}),
@@ -364,6 +366,7 @@ describe('GithubEntityProvider', () => {
id: 'abc123',
text: 'some yaml',
},
visibility: 'public',
},
],
}),
@@ -453,6 +456,7 @@ describe('GithubEntityProvider', () => {
name: 'main',
},
catalogInfoFile: null,
visibility: 'public',
},
{
name: 'another-repo',
@@ -470,6 +474,7 @@ describe('GithubEntityProvider', () => {
id: 'abc123',
text: 'some yaml',
},
visibility: 'public',
},
],
}),
@@ -571,6 +576,7 @@ describe('GithubEntityProvider', () => {
id: 'abc123',
text: 'some yaml',
},
visibility: 'public',
},
{
name: 'test-repo-2',
@@ -595,6 +601,7 @@ describe('GithubEntityProvider', () => {
id: 'abc123',
text: 'some yaml',
},
visibility: 'public',
},
{
name: 'test-repo-3',
@@ -616,6 +623,7 @@ describe('GithubEntityProvider', () => {
id: 'abc123',
text: 'some yaml',
},
visibility: 'public',
},
],
}),
plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts
+10 -1
View File
@@ -40,7 +40,11 @@ import {
GithubEntityProviderConfig,
} from './GithubEntityProviderConfig';
import { getOrganizationRepositories } from '../lib/github';
import { satisfiesTopicFilter, satisfiesForkFilter } from '../lib/util';
import {
satisfiesTopicFilter,
satisfiesForkFilter,
satisfiesVisibilityFilter,
} from '../lib/util';
import { EventParams, EventSubscriber } from '@backstage/plugin-events-node';
import { PushEvent, Commit } from '@octokit/webhooks-types';
@@ -56,6 +60,7 @@ type Repository = {
repositoryTopics: string[];
defaultBranchRef?: string;
isCatalogInfoFilePresent: boolean;
visibility: string;
};
/**
@@ -221,6 +226,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
isCatalogInfoFilePresent:
r.catalogInfoFile?.__typename === 'Blob' &&
r.catalogInfoFile.text !== '',
visibility: r.visibility,
};
});
@@ -237,6 +243,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
const repositoryFilter = this.config.filters?.repository;
const topicFilters = this.config.filters?.topic;
const allowForks = this.config.filters?.allowForks ?? true;
const visibilities = this.config.filters?.visibilities ?? [];
const matchingRepositories = repositories.filter(r => {
const repoTopics: string[] = r.repositoryTopics;
@@ -245,6 +252,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
(!repositoryFilter || repositoryFilter.test(r.name)) &&
satisfiesTopicFilter(repoTopics, topicFilters) &&
satisfiesForkFilter(allowForks, r.isFork) &&
satisfiesVisibilityFilter(visibilities, r.visibility) &&
r.defaultBranchRef
);
});
@@ -310,6 +318,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
// we can consider this file present because
// only the catalog file will be recovered from the commits
isCatalogInfoFilePresent: true,
visibility: event.repository.visibility,
};
const matchingTargets = this.matchesFilters([repository]);
plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts
+34 -2
View File
@@ -84,6 +84,12 @@ describe('readProviderConfigs', () => {
allowForks: false,
},
},
providerWithVisibilitiesFilter: {
organization: 'test-org6',
filters: {
visibilities: ['public', 'internal'],
},
},
providerWithHost: {
organization: 'test-org1',
host: 'ghe.internal.com',
@@ -103,7 +109,7 @@ describe('readProviderConfigs', () => {
});
const providerConfigs = readProviderConfigs(config);
expect(providerConfigs).toHaveLength(8);
expect(providerConfigs).toHaveLength(9);
expect(providerConfigs[0]).toEqual({
id: 'providerOrganizationOnly',
organization: 'test-org1',
@@ -117,6 +123,7 @@ describe('readProviderConfigs', () => {
include: undefined,
exclude: undefined,
},
visibilities: undefined,
},
schedule: undefined,
validateLocationsExist: false,
@@ -134,6 +141,7 @@ describe('readProviderConfigs', () => {
include: undefined,
exclude: undefined,
},
visibilities: undefined,
},
schedule: undefined,
validateLocationsExist: false,
@@ -151,6 +159,7 @@ describe('readProviderConfigs', () => {
include: undefined,
exclude: undefined,
},
visibilities: undefined,
},
schedule: undefined,
validateLocationsExist: false,
@@ -168,6 +177,7 @@ describe('readProviderConfigs', () => {
include: undefined,
exclude: undefined,
},
visibilities: undefined,
},
schedule: undefined,
validateLocationsExist: false,
@@ -185,6 +195,7 @@ describe('readProviderConfigs', () => {
include: ['backstage-include'],
exclude: ['backstage-exclude'],
},
visibilities: undefined,
},
schedule: undefined,
validateLocationsExist: false,
@@ -202,11 +213,30 @@ describe('readProviderConfigs', () => {
include: undefined,
exclude: undefined,
},
visibilities: undefined,
},
schedule: undefined,
validateLocationsExist: false,
});
expect(providerConfigs[6]).toEqual({
id: 'providerWithVisibilitiesFilter',
organization: 'test-org6',
catalogPath: '/catalog-info.yaml',
host: 'github.com',
filters: {
repository: undefined,
branch: undefined,
allowForks: true,
topic: {
include: undefined,
exclude: undefined,
},
visibilities: ['public', 'internal'],
},
schedule: undefined,
validateLocationsExist: false,
});
expect(providerConfigs[7]).toEqual({
id: 'providerWithHost',
organization: 'test-org1',
catalogPath: '/catalog-info.yaml',
@@ -219,11 +249,12 @@ describe('readProviderConfigs', () => {
include: undefined,
exclude: undefined,
},
visibilities: undefined,
},
validateLocationsExist: false,
schedule: undefined,
});
expect(providerConfigs[7]).toEqual({
expect(providerConfigs[8]).toEqual({
id: 'providerWithSchedule',
organization: 'test-org1',
catalogPath: '/catalog-info.yaml',
@@ -236,6 +267,7 @@ describe('readProviderConfigs', () => {
include: undefined,
exclude: undefined,
},
visibilities: undefined,
},
schedule: {
frequency: Duration.fromISO('PT30M'),
plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts
+6
View File
@@ -33,6 +33,7 @@ export type GithubEntityProviderConfig = {
branch?: string;
topic?: GithubTopicFilters;
allowForks?: boolean;
visibilities?: string[];
};
validateLocationsExist: boolean;
schedule?: TaskScheduleDefinition;
@@ -85,6 +86,10 @@ function readProviderConfig(
const catalogPathContainsWildcard = catalogPath.includes('*');
const visibilitiesInclude = config?.getOptionalStringArray(
'filters.visibilities',
);
if (validateLocationsExist && catalogPathContainsWildcard) {
throw Error(
`Error while processing GitHub provider config. The catalog path ${catalogPath} contains a wildcard, which is incompatible with validation of locations existing before emitting them. Ensure that validateLocationsExist is set to false.`,
@@ -110,6 +115,7 @@ function readProviderConfig(
include: topicFilterInclude,
exclude: topicFilterExclude,
},
visibilities: visibilitiesInclude,
},
schedule,
validateLocationsExist,