paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 27 Packages Projects Releases Wiki Activity

chore(events,catalog/github): update with review comments

Browse Source 
Signed-off-by: Rogerio Angeliski <angeliski@hotmail.com>
This commit is contained in:
Rogerio Angeliski
2022-11-23 15:34:39 +00:00
parent a0fd4af94a
commit e90b092d2a
8 changed files with 196 additions and 319 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/catalog-backend-module-github/api-report.md
-3
View File
@@ -5,7 +5,6 @@
```ts
import { AnalyzeOptions } from '@backstage/plugin-catalog-backend';
import { BackendFeature } from '@backstage/backend-plugin-api';
import { CatalogApi } from '@backstage/catalog-client';
import { CatalogProcessor } from '@backstage/plugin-catalog-backend';
import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend';
import { Config } from '@backstage/config';
@@ -86,11 +85,9 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
static fromConfig(
config: Config,
options: {
catalogApi?: CatalogApi;
logger: Logger;
schedule?: TaskRunner;
scheduler?: PluginTaskScheduler;
tokenManager?: TokenManager;
},
): GithubEntityProvider[];
// (undocumented)
plugins/catalog-backend-module-github/src/lib/github.ts
-55
View File
@@ -34,7 +34,6 @@ export type QueryResponse = {
type RepositoryOwnerResponse = {
repositories?: Connection<RepositoryResponse>;
repository?: RepositoryResponse;
};
export type OrganizationResponse = {
@@ -301,60 +300,6 @@ export async function getOrganizationRepositories(
return { repositories };
}
export async function getOrganizationRepository(
client: typeof graphql,
org: string,
catalogPath: string,
repoName: string,
): Promise<{ repository: RepositoryResponse | undefined }> {
let relativeCatalogPathRef: string;
// We must strip the leading slash or the query for objects does not work
if (catalogPath.startsWith('/')) {
relativeCatalogPathRef = catalogPath.substring(1);
} else {
relativeCatalogPathRef = catalogPath;
}
const catalogPathRef = `HEAD:${relativeCatalogPathRef}`;
const query = `
query repositories($org: String!, $catalogPathRef: String!, $repoName: String!) {
repositoryOwner(login: $org) {
login
repository(name: $repoName) {
name
catalogInfoFile: object(expression: $catalogPathRef) {
__typename
... on Blob {
id
text
}
}
url
isArchived
repositoryTopics(first: 100) {
nodes {
... on RepositoryTopic {
topic {
name
}
}
}
}
defaultBranchRef {
name
}
}
}
}`;
const response: QueryResponse = await client(query, {
org,
catalogPathRef,
repoName,
});
return { repository: response.repositoryOwner?.repository };
}
/**
* Gets all the users out of a Github organization.
*
plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts
+86 -127
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { getVoidLogger, TokenManager } from '@backstage/backend-common';
import { getVoidLogger } from '@backstage/backend-common';
import {
PluginTaskScheduler,
TaskInvocationDefinition,
@@ -25,12 +25,10 @@ import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
import { GithubEntityProvider } from './GithubEntityProvider';
import * as helpers from '../lib/github';
import { EventParams } from '@backstage/plugin-events-node';
import { CatalogApi } from '@backstage/catalog-client';
jest.mock('../lib/github', () => {
return {
getOrganizationRepositories: jest.fn(),
getOrganizationRepository: jest.fn(),
};
});
class PersistingTaskRunner implements TaskRunner {
@@ -48,15 +46,6 @@ class PersistingTaskRunner implements TaskRunner {
const logger = getVoidLogger();
const mockCatalogApi: Partial<CatalogApi> = {
refreshEntity: jest.fn(),
};
const mockTokenManager: jest.Mocked<TokenManager> = {
getToken: jest.fn(),
authenticate: jest.fn(),
};
describe('GithubEntityProvider', () => {
afterEach(() => jest.resetAllMocks());
@@ -752,30 +741,6 @@ describe('GithubEntityProvider', () => {
};
await provider.connect(entityProviderConnection);
const mockGetOrganizationRepository = jest.spyOn(
helpers,
'getOrganizationRepository',
);
mockGetOrganizationRepository.mockReturnValue(
Promise.resolve({
repository: {
name: 'test-repo',
url: 'https://github.com/test-org/test-repo',
repositoryTopics: { nodes: [] },
isArchived: false,
defaultBranchRef: {
name: 'main',
},
catalogInfoFile: {
__typename: 'Blob',
id: 'abc123',
text: 'some yaml',
},
},
}),
);
const event: EventParams = {
topic: 'github.push',
metadata: {
@@ -790,6 +755,7 @@ describe('GithubEntityProvider', () => {
stargazers: 0,
master_branch: 'main',
organization: 'test-org',
topics: [],
},
created: true,
deleted: false,
@@ -865,30 +831,6 @@ describe('GithubEntityProvider', () => {
};
await provider.connect(entityProviderConnection);
const mockGetOrganizationRepository = jest.spyOn(
helpers,
'getOrganizationRepository',
);
mockGetOrganizationRepository.mockReturnValue(
Promise.resolve({
repository: {
name: 'test-repo',
url: 'https://github.com/test-org/test-repo',
repositoryTopics: { nodes: [] },
isArchived: false,
defaultBranchRef: {
name: 'main',
},
catalogInfoFile: {
__typename: 'Blob',
id: 'abc123',
text: 'some yaml',
},
},
}),
);
const event: EventParams = {
topic: 'github.push',
metadata: {
@@ -903,6 +845,7 @@ describe('GithubEntityProvider', () => {
stargazers: 0,
master_branch: 'main',
organization: 'test-org',
topics: [],
},
created: true,
deleted: false,
@@ -967,12 +910,9 @@ describe('GithubEntityProvider', () => {
},
});
mockTokenManager.getToken.mockResolvedValue({ token: '' });
const provider = GithubEntityProvider.fromConfig(config, {
logger,
schedule,
tokenManager: mockTokenManager,
catalogApi: mockCatalogApi as unknown as CatalogApi,
})[0];
const entityProviderConnection: EntityProviderConnection = {
@@ -981,30 +921,6 @@ describe('GithubEntityProvider', () => {
};
await provider.connect(entityProviderConnection);
const mockGetOrganizationRepository = jest.spyOn(
helpers,
'getOrganizationRepository',
);
mockGetOrganizationRepository.mockReturnValue(
Promise.resolve({
repository: {
name: 'test-repo',
url: 'https://github.com/test-org/test-repo',
repositoryTopics: { nodes: [] },
isArchived: false,
defaultBranchRef: {
name: 'main',
},
catalogInfoFile: {
__typename: 'Blob',
id: 'abc123',
text: 'some yaml',
},
},
}),
);
const event: EventParams = {
topic: 'github.push',
metadata: {
@@ -1019,6 +935,7 @@ describe('GithubEntityProvider', () => {
stargazers: 0,
master_branch: 'main',
organization: 'test-org',
topics: [],
},
created: true,
deleted: false,
@@ -1040,36 +957,34 @@ describe('GithubEntityProvider', () => {
await provider.onEvent(event);
expect(mockCatalogApi.refreshEntity).toHaveBeenCalledTimes(1);
expect(mockCatalogApi.refreshEntity).toHaveBeenCalledWith(
'location:default/generated-8688630f57e421bc85f12b9828ed7dad6aff3bb3',
{ token: '' },
);
expect(entityProviderConnection.refresh).toHaveBeenCalledTimes(1);
expect(entityProviderConnection.refresh).toHaveBeenCalledWith({
keys: [
'url:https://github.com/test-org/test-repo/tree/main/catalog-info.yaml',
],
});
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0);
});
it('should recover repository information when match filters from push event', async () => {
it('should process repository when match filters from push event', async () => {
const schedule = new PersistingTaskRunner();
const config = new ConfigReader({
catalog: {
providers: {
github: {
organization: 'test-org',
checkRepositoryFiltersForWebhook: true,
filters: {
branch: 'my-special-branch',
repository: 'test-repo',
},
},
},
},
});
mockTokenManager.getToken.mockResolvedValue({ token: '' });
const provider = GithubEntityProvider.fromConfig(config, {
logger,
schedule,
tokenManager: mockTokenManager,
catalogApi: mockCatalogApi as unknown as CatalogApi,
})[0];
const entityProviderConnection: EntityProviderConnection = {
@@ -1078,30 +993,6 @@ describe('GithubEntityProvider', () => {
};
await provider.connect(entityProviderConnection);
const mockGetOrganizationRepository = jest.spyOn(
helpers,
'getOrganizationRepository',
);
mockGetOrganizationRepository.mockReturnValue(
Promise.resolve({
repository: {
name: 'test-repo',
url: 'https://github.com/test-org/test-repo',
repositoryTopics: { nodes: [] },
isArchived: false,
defaultBranchRef: {
name: 'main',
},
catalogInfoFile: {
__typename: 'Blob',
id: 'abc123',
text: 'some yaml',
},
},
}),
);
const event: EventParams = {
topic: 'github.push',
metadata: {
@@ -1110,12 +1001,13 @@ describe('GithubEntityProvider', () => {
eventPayload: {
ref: 'refs/heads/my-special-branch',
repository: {
name: 'teste-1',
name: 'test-repo',
url: 'https://github.com/test-org/test-repo',
default_branch: 'main',
stargazers: 0,
master_branch: 'main',
organization: 'test-org',
topics: [],
},
created: true,
deleted: false,
@@ -1137,11 +1029,78 @@ describe('GithubEntityProvider', () => {
await provider.onEvent(event);
expect(mockCatalogApi.refreshEntity).toHaveBeenCalledTimes(1);
expect(mockCatalogApi.refreshEntity).toHaveBeenCalledWith(
'location:default/generated-6ffd478ea33caf9af61fa75cc09b5aa7770470f0de',
{ token: '' },
);
expect(entityProviderConnection.refresh).toHaveBeenCalledTimes(1);
expect(entityProviderConnection.refresh).toHaveBeenCalledWith({
keys: [
'url:https://github.com/test-org/test-repo/tree/my-special-branch/catalog-info.yaml',
],
});
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0);
});
it("should skip process when didn't match filters from push event", async () => {
const schedule = new PersistingTaskRunner();
const config = new ConfigReader({
catalog: {
providers: {
github: {
organization: 'test-org',
filters: {
repository: 'only-special-repository',
},
},
},
},
});
const provider = GithubEntityProvider.fromConfig(config, {
logger,
schedule,
})[0];
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
await provider.connect(entityProviderConnection);
const event: EventParams = {
topic: 'github.push',
metadata: {
'x-github-event': 'push',
},
eventPayload: {
ref: 'refs/heads/main',
repository: {
name: 'teste-1',
url: 'https://github.com/test-org/test-repo',
default_branch: 'main',
stargazers: 0,
master_branch: 'main',
organization: 'test-org',
topics: [],
},
created: true,
deleted: false,
forced: false,
commits: [
{
added: ['new-file.yaml'],
removed: [],
modified: [],
},
{
added: [],
removed: [],
modified: ['catalog-info.yaml'],
},
],
},
};
await provider.onEvent(event);
expect(entityProviderConnection.refresh).toHaveBeenCalledTimes(0);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(0);
});
});
plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.ts
+62 -107
View File
@@ -40,20 +40,23 @@ import {
readProviderConfigs,
GithubEntityProviderConfig,
} from './GithubEntityProviderConfig';
import {
getOrganizationRepositories,
getOrganizationRepository,
RepositoryResponse,
} from '../lib/github';
import { getOrganizationRepositories } from '../lib/github';
import { satisfiesTopicFilter } from '../lib/util';
import { EventParams, EventSubscriber } from '@backstage/plugin-events-node';
import { PushEvent, Commit } from '@octokit/webhooks-types';
import { CatalogApi } from '@backstage/catalog-client';
import { TokenManager } from '@backstage/backend-common';
import { stringifyEntityRef } from '@backstage/catalog-model';
const TOPIC_REPO_PUSH = 'github.push';
type Repository = {
name: string;
url: string;
isArchived: boolean;
repositoryTopics: string[];
defaultBranchRef: string | null;
isCatalogInfoFilePresent: boolean;
};
/**
* Discovers catalog files located in [GitHub](https://github.com).
* The provider will search your GitHub account and register catalog files matching the configured path
@@ -68,18 +71,14 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
private readonly integration: GithubIntegrationConfig;
private readonly scheduleFn: () => Promise<void>;
private connection?: EntityProviderConnection;
private readonly catalogApi?: CatalogApi;
private readonly tokenManager?: TokenManager;
private readonly githubCredentialsProvider: GithubCredentialsProvider;
static fromConfig(
config: Config,
options: {
catalogApi?: CatalogApi;
logger: Logger;
schedule?: TaskRunner;
scheduler?: PluginTaskScheduler;
tokenManager?: TokenManager;
},
): GithubEntityProvider[] {
if (!options.schedule && !options.scheduler) {
@@ -113,8 +112,6 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
integration,
options.logger,
taskRunner,
options.catalogApi,
options.tokenManager,
);
});
}
@@ -124,8 +121,6 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
integration: GithubIntegration,
logger: Logger,
taskRunner: TaskRunner,
catalogApi?: CatalogApi,
tokenManager?: TokenManager,
) {
this.config = config;
this.integration = integration.config;
@@ -135,9 +130,6 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
this.scheduleFn = this.createScheduleFn(taskRunner);
this.githubCredentialsProvider =
SingleInstanceGithubCredentialsProvider.create(integration.config);
this.catalogApi = catalogApi;
this.tokenManager = tokenManager;
}
/** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */
@@ -200,7 +192,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
}
// go to the server and get all of the repositories
private async findCatalogFiles(): Promise<RepositoryResponse[]> {
private async findCatalogFiles(): Promise<Repository[]> {
const organization = this.config.organization;
const host = this.integration.host;
const catalogPath = this.config.catalogPath;
@@ -215,45 +207,49 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
headers,
});
const { repositories } = await getOrganizationRepositories(
client,
organization,
catalogPath,
);
const { repositories: repositoriesFromGithub } =
await getOrganizationRepositories(client, organization, catalogPath);
const repositories = repositoriesFromGithub.map(r => {
return {
url: r.url,
name: r.name,
defaultBranchRef: r.defaultBranchRef?.name || null,
repositoryTopics: r.repositoryTopics.nodes.map(t => t.topic.name),
isArchived: r.isArchived,
isCatalogInfoFilePresent:
r.catalogInfoFile?.__typename === 'Blob' &&
r.catalogInfoFile.text !== '',
};
});
if (this.config.validateLocationsExist) {
return repositories.filter(repository => {
return (
repository.catalogInfoFile?.__typename === 'Blob' &&
repository.catalogInfoFile.text !== ''
);
});
return repositories.filter(
repository => repository.isCatalogInfoFilePresent,
);
}
return repositories;
}
private matchesFilters(repositories: RepositoryResponse[]) {
private matchesFilters(repositories: Repository[]) {
const repositoryFilter = this.config.filters?.repository;
const topicFilters = this.config.filters?.topic;
const matchingRepositories = repositories.filter(r => {
const repoTopics: string[] = r.repositoryTopics.nodes.map(
node => node.topic.name,
);
const repoTopics: string[] = r.repositoryTopics;
return (
!r.isArchived &&
(!repositoryFilter || repositoryFilter.test(r.name)) &&
satisfiesTopicFilter(repoTopics, topicFilters) &&
r.defaultBranchRef?.name
r.defaultBranchRef
);
});
return matchingRepositories;
}
private createLocationUrl(repository: RepositoryResponse): string {
private createLocationUrl(repository: Repository): string {
const branch =
this.config.filters?.branch || repository.defaultBranchRef?.name || '-';
this.config.filters?.branch || repository.defaultBranchRef || '-';
const catalogFile = this.config.catalogPath.startsWith('/')
? this.config.catalogPath.substring(1)
: this.config.catalogPath;
@@ -275,11 +271,7 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
return;
}
if (params.metadata?.['x-github-event'] === 'push') {
await this.onRepoPush(params.eventPayload as PushEvent);
}
return;
await this.onRepoPush(params.eventPayload as PushEvent);
}
/** {@inheritdoc @backstage/plugin-events-node#EventSubscriber.supportsEventTopics} */
@@ -304,23 +296,23 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
return;
}
if (this.config.checkRepositoryFiltersForWebhook) {
const repository = await this.getRepositoryInfo(repoName);
const repository: Repository = {
url: event.repository.url,
name: event.repository.name,
defaultBranchRef: event.repository.default_branch,
repositoryTopics: event.repository.topics,
isArchived: event.repository.archived,
// we can consider this file present because
// only the catalog file will be recovered from the commits
isCatalogInfoFilePresent: true,
};
if (!repository) {
this.logger.debug(
`skipping push event from repository ${repoName} because didn't find information in Github`,
);
return;
}
const matchingTargets = this.matchesFilters([repository]);
if (matchingTargets.length === 0) {
this.logger.debug(
`skipping push event from repository ${repoName} because didn't match provider filters`,
);
return;
}
const matchingTargets = this.matchesFilters([repository]);
if (matchingTargets.length === 0) {
this.logger.debug(
`skipping push event from repository ${repoName} because didn't match provider filters`,
);
return;
}
// the commit has information about the files (added,removed,modified)
@@ -338,36 +330,26 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
event.commits,
(commit: Commit) => [...commit.removed],
);
const modifiedCatalogFiles = this.collectFilesFromCommit(
const modified = this.collectFilesFromCommit(
event.commits,
(commit: Commit) => [...commit.modified],
);
const limiter = limiterFactory(10);
const promises: Promise<void>[] = [];
let modified = [];
let promises: Promise<void>[] = [];
if (this.catalogApi && this.tokenManager) {
modified = modifiedCatalogFiles
.map(filePath => `${event.repository.url}/blob/${branch}/${filePath}`)
.map(url => {
const location = GithubEntityProvider.toLocationSpec(url);
return locationSpecToLocationEntity({ location });
});
const { token } = await this.tokenManager.getToken();
promises = modified.map(entity =>
if (modified.length > 0) {
const connection = this.connection;
promises.push(
limiter(async () =>
this.catalogApi!.refreshEntity(stringifyEntityRef(entity), { token }),
connection.refresh({
keys: modified.map(
filePath =>
`url:${event.repository.url}/tree/${branch}/${filePath}`,
),
}),
),
);
} else {
this.logger.debug(
`skipping modified operation because is missing CatalogApi and/or TokenManager.`,
);
}
if (added.length > 0 || removed.length > 0) {
@@ -420,33 +402,6 @@ export class GithubEntityProvider implements EntityProvider, EventSubscriber {
.filter(file => catalogFile.includes(file));
}
private async getRepositoryInfo(
repoName: string,
): Promise<RepositoryResponse | undefined> {
const organization = this.config.organization;
const host = this.integration.host;
const catalogPath = this.config.catalogPath;
const orgUrl = `https://${host}/${organization}`;
const { headers } = await this.githubCredentialsProvider.getCredentials({
url: orgUrl,
});
const client = graphql.defaults({
baseUrl: this.integration.apiBaseUrl,
headers,
});
const { repository } = await getOrganizationRepository(
client,
organization,
catalogPath,
repoName,
);
return repository;
}
private toDeferredEntities(targets: string[]): DeferredEntity[] {
return targets
.map(target => {
plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts
-7
View File
@@ -102,7 +102,6 @@ describe('readProviderConfigs', () => {
id: 'providerOrganizationOnly',
organization: 'test-org1',
catalogPath: '/catalog-info.yaml',
checkRepositoryFiltersForWebhook: false,
host: 'github.com',
filters: {
repository: undefined,
@@ -120,7 +119,6 @@ describe('readProviderConfigs', () => {
organization: 'test-org2',
catalogPath: 'custom/path/catalog-info.yaml',
host: 'github.com',
checkRepositoryFiltersForWebhook: false,
filters: {
repository: undefined,
branch: undefined,
@@ -136,7 +134,6 @@ describe('readProviderConfigs', () => {
id: 'providerWithRepositoryFilter',
organization: 'test-org3', // organization
catalogPath: '/catalog-info.yaml', // file
checkRepositoryFiltersForWebhook: false,
host: 'github.com',
filters: {
repository: /^repository.*filter$/, // repo
@@ -153,7 +150,6 @@ describe('readProviderConfigs', () => {
id: 'providerWithBranchFilter',
organization: 'test-org4',
catalogPath: '/catalog-info.yaml',
checkRepositoryFiltersForWebhook: false,
host: 'github.com',
filters: {
repository: undefined,
@@ -170,7 +166,6 @@ describe('readProviderConfigs', () => {
id: 'providerWithTopicFilter',
organization: 'test-org5',
catalogPath: '/catalog-info.yaml',
checkRepositoryFiltersForWebhook: false,
host: 'github.com',
filters: {
repository: undefined,
@@ -187,7 +182,6 @@ describe('readProviderConfigs', () => {
id: 'providerWithHost',
organization: 'test-org1',
catalogPath: '/catalog-info.yaml',
checkRepositoryFiltersForWebhook: false,
host: 'ghe.internal.com',
filters: {
repository: undefined,
@@ -204,7 +198,6 @@ describe('readProviderConfigs', () => {
id: 'providerWithSchedule',
organization: 'test-org1',
catalogPath: '/catalog-info.yaml',
checkRepositoryFiltersForWebhook: false,
host: 'github.com',
filters: {
repository: undefined,
plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.ts
-5
View File
@@ -34,7 +34,6 @@ export type GithubEntityProviderConfig = {
topic?: GithubTopicFilters;
};
validateLocationsExist: boolean;
checkRepositoryFiltersForWebhook: boolean;
schedule?: TaskScheduleDefinition;
};
@@ -82,9 +81,6 @@ function readProviderConfig(
const validateLocationsExist =
config?.getOptionalBoolean('validateLocationsExist') ?? false;
const checkRepositoryFiltersForWebhook =
config?.getOptionalBoolean('checkRepositoryFiltersForWebhook') ?? false;
const catalogPathContainsWildcard = catalogPath.includes('*');
if (validateLocationsExist && catalogPathContainsWildcard) {
@@ -114,7 +110,6 @@ function readProviderConfig(
},
schedule,
validateLocationsExist,
checkRepositoryFiltersForWebhook,
};
}