Merge pull request #12822 from brentg-telus/github-entity-provider-3
feat: github entity provider
This commit is contained in:
Ben Lambert
@@ -0,0 +1,9 @@
|
||||
---
|
||||
'@backstage/plugin-catalog-backend-module-github': patch
|
||||
---
|
||||
|
||||
Github Entity Provider functionality for adding entities to the catalog.
|
||||
|
||||
This provider replaces the GithubDiscoveryProcessor functionality as providers offer more flexibility with scheduling ingestion, removing and preventing orphaned entities.
|
||||
|
||||
More information can be found on the [GitHub Discovery](https://backstage.io/docs/integrations/github/discovery) page.
|
||||
@@ -6,6 +6,119 @@ sidebar_label: Discovery
|
||||
description: Automatically discovering catalog entities from repositories in a GitHub organization
|
||||
---
|
||||
|
||||
## GitHub Provider
|
||||
|
||||
The GitHub integration has a discovery provider for discovering catalog
|
||||
entities within a GitHub organization. The provider will crawl the GitHub
|
||||
organization and register entities matching the configured path. This can be
|
||||
useful as an alternative to static locations or manually adding things to the
|
||||
catalog. This is the prefered method for ingesting entities into the catalog.
|
||||
|
||||
## Installation
|
||||
|
||||
You will have to add the provider in the catalog initialization code of your
|
||||
backend. They are not installed by default, therefore you have to add a
|
||||
dependency on `@backstage/plugin-catalog-backend-module-github` to your backend
|
||||
package.
|
||||
|
||||
```bash
|
||||
# From your Backstage root directory
|
||||
yarn add --cwd packages/backend @backstage/plugin-catalog-backend-module-github
|
||||
```
|
||||
|
||||
And then add the entity provider to your catalog builder:
|
||||
|
||||
```diff
|
||||
// In packages/backend/src/plugins/catalog.ts
|
||||
+ import { GitHubEntityProvider } from '@backstage/plugin-catalog-backend-module-github';
|
||||
|
||||
export default async function createPlugin(
|
||||
env: PluginEnvironment,
|
||||
): Promise<Router> {
|
||||
const builder = await CatalogBuilder.create(env);
|
||||
+ builder.addEntityProvider(
|
||||
+ GitHubEntityProvider.fromConfig(env.config, {
|
||||
+ logger: env.logger,
|
||||
+ schedule: env.scheduler.createScheduledTaskRunner({
|
||||
+ frequency: { minutes: 30 },
|
||||
+ timeout: { minutes: 3 },
|
||||
+ }),
|
||||
+ }),
|
||||
+ );
|
||||
|
||||
// [...]
|
||||
}
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
To use the discovery provider, you'll need a GitHub integration
|
||||
[set up](locations.md) with either a [Personal Access Token](../../getting-started/configuration.md#setting-up-a-github-integration) or [GitHub Apps](./github-apps.md).
|
||||
|
||||
Then you can add a github config to the catalog providers configuration:
|
||||
|
||||
```yaml
|
||||
catalog:
|
||||
providers:
|
||||
github:
|
||||
# the provider ID can be any camelCase string
|
||||
providerId:
|
||||
organization: 'backstage' # string
|
||||
catalogPath: '/catalog-info.yaml' # string
|
||||
filters:
|
||||
branch: 'main' # string
|
||||
repository: '.*' # Regex
|
||||
customProviderId:
|
||||
organization: 'new-org' # string
|
||||
catalogPath: '/custom/path/catalog-info.yaml' # string
|
||||
filters: # optional filters
|
||||
branch: 'develop' # optional string
|
||||
repository: '.*' # optional Regex
|
||||
```
|
||||
|
||||
This provider supports multiple organizations via unique provider IDs.
|
||||
|
||||
> **Note:** It is possible but certainly not recommended to skip the provider ID level.
|
||||
> If you do so, `default` will be used as provider ID.
|
||||
|
||||
- **`catalogPath`** _(optional)_:
|
||||
Default: `/catalog-info.yaml`.
|
||||
Path where to look for `catalog-info.yaml` files.
|
||||
When started with `/`, it is an absolute path from the repo root.
|
||||
- **filters** _(optional)_:
|
||||
- **branch** _(optional)_:
|
||||
String used to filter results based on the branch name.
|
||||
- **repository** _(optional)_:
|
||||
Regular expression used to filter results based on the repository name.
|
||||
- **organization**:
|
||||
Name of your organization account/workspace.
|
||||
If you want to add multiple organizations, you need to add one provider config each.
|
||||
|
||||
## GitHub API Rate Limits
|
||||
|
||||
GitHub [rate limits](https://docs.github.com/en/rest/overview/resources-in-the-rest-api#rate-limiting) API requests to 5,000 per hour (or more for Enterprise
|
||||
accounts). The snippet below refreshes the Backstage catalog data every 35 minutes, which issues an API request for each discovered location.
|
||||
|
||||
If your requests are too frequent then you may get throttled by
|
||||
rate limiting. You can change the refresh frequency of the catalog in your `packages/backend/src/plugins/catalog.ts` file:
|
||||
|
||||
```typescript
|
||||
schedule: env.scheduler.createScheduledTaskRunner({
|
||||
frequency: { minutes: 35 },
|
||||
timeout: { minutes: 30 },
|
||||
}),
|
||||
```
|
||||
|
||||
More information about scheduling can be found on the [TaskScheduleDefinition](https://backstage.io/docs/reference/backend-tasks.taskscheduledefinition) page.
|
||||
|
||||
Alternatively, or additionally, you can configure [github-apps](github-apps.md) authentication
|
||||
which carries a much higher rate limit at GitHub.
|
||||
|
||||
This is true for any method of adding GitHub entities to the catalog, but
|
||||
especially easy to hit with automatic discovery.
|
||||
|
||||
## GitHub Processor (To Be Deprecated)
|
||||
|
||||
The GitHub integration has a special discovery processor for discovering catalog
|
||||
entities within a GitHub organization. The processor will crawl the GitHub
|
||||
organization and register entities matching the configured path. This can be
|
||||
|
||||
@@ -40,6 +40,24 @@ export class GithubDiscoveryProcessor implements CatalogProcessor {
|
||||
): Promise<boolean>;
|
||||
}
|
||||
|
||||
// @public
|
||||
export class GitHubEntityProvider implements EntityProvider {
|
||||
// (undocumented)
|
||||
connect(connection: EntityProviderConnection): Promise<void>;
|
||||
// (undocumented)
|
||||
static fromConfig(
|
||||
config: Config,
|
||||
options: {
|
||||
logger: Logger;
|
||||
schedule: TaskRunner;
|
||||
},
|
||||
): GitHubEntityProvider[];
|
||||
// (undocumented)
|
||||
getProviderName(): string;
|
||||
// (undocumented)
|
||||
refresh(logger: Logger): Promise<void>;
|
||||
}
|
||||
|
||||
// @public
|
||||
export type GithubMultiOrgConfig = Array<{
|
||||
name: string;
|
||||
|
||||
@@ -20,9 +20,10 @@
|
||||
* @packageDocumentation
|
||||
*/
|
||||
|
||||
export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor';
|
||||
export { GithubMultiOrgReaderProcessor } from './GithubMultiOrgReaderProcessor';
|
||||
export { GitHubOrgEntityProvider } from './GitHubOrgEntityProvider';
|
||||
export type { GitHubOrgEntityProviderOptions } from './GitHubOrgEntityProvider';
|
||||
export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor';
|
||||
export { GithubDiscoveryProcessor } from './processors/GithubDiscoveryProcessor';
|
||||
export { GithubMultiOrgReaderProcessor } from './processors/GithubMultiOrgReaderProcessor';
|
||||
export { GithubOrgReaderProcessor } from './processors/GithubOrgReaderProcessor';
|
||||
export { GitHubEntityProvider } from './providers/GitHubEntityProvider';
|
||||
export { GitHubOrgEntityProvider } from './providers/GitHubOrgEntityProvider';
|
||||
export type { GitHubOrgEntityProviderOptions } from './providers/GitHubOrgEntityProvider';
|
||||
export type { GithubMultiOrgConfig } from './lib';
|
||||
|
||||
+2
-2
@@ -22,9 +22,9 @@ import {
|
||||
} from '@backstage/integration';
|
||||
import { LocationSpec } from '@backstage/plugin-catalog-backend';
|
||||
import { GithubDiscoveryProcessor, parseUrl } from './GithubDiscoveryProcessor';
|
||||
import { getOrganizationRepositories } from './lib';
|
||||
import { getOrganizationRepositories } from '../lib';
|
||||
|
||||
jest.mock('./lib');
|
||||
jest.mock('../lib');
|
||||
const mockGetOrganizationRepositories =
|
||||
getOrganizationRepositories as jest.MockedFunction<
|
||||
typeof getOrganizationRepositories
|
||||
+1
-1
@@ -29,7 +29,7 @@ import {
|
||||
} from '@backstage/plugin-catalog-backend';
|
||||
import { graphql } from '@octokit/graphql';
|
||||
import { Logger } from 'winston';
|
||||
import { getOrganizationRepositories } from './lib';
|
||||
import { getOrganizationRepositories } from '../lib';
|
||||
|
||||
/**
|
||||
* Extracts repositories out of a GitHub org.
|
||||
+1
-1
@@ -37,7 +37,7 @@ import {
|
||||
getOrganizationUsers,
|
||||
GithubMultiOrgConfig,
|
||||
readGithubMultiOrgConfig,
|
||||
} from './lib';
|
||||
} from '../lib';
|
||||
|
||||
/**
|
||||
* Extracts teams and users out of a multiple GitHub orgs namespaced per org.
|
||||
+1
-1
@@ -36,7 +36,7 @@ import {
|
||||
getOrganizationTeams,
|
||||
getOrganizationUsers,
|
||||
parseGitHubOrgUrl,
|
||||
} from './lib';
|
||||
} from '../lib';
|
||||
|
||||
type GraphQL = typeof graphql;
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { getVoidLogger } from '@backstage/backend-common';
|
||||
import { TaskInvocationDefinition, TaskRunner } from '@backstage/backend-tasks';
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
|
||||
import { GitHubEntityProvider } from './GitHubEntityProvider';
|
||||
import * as helpers from '../lib/github';
|
||||
|
||||
class PersistingTaskRunner implements TaskRunner {
|
||||
private tasks: TaskInvocationDefinition[] = [];
|
||||
|
||||
getTasks() {
|
||||
return this.tasks;
|
||||
}
|
||||
|
||||
run(task: TaskInvocationDefinition): Promise<void> {
|
||||
this.tasks.push(task);
|
||||
return Promise.resolve(undefined);
|
||||
}
|
||||
}
|
||||
|
||||
const logger = getVoidLogger();
|
||||
|
||||
describe('GitHubEntityProvider', () => {
|
||||
afterEach(() => jest.resetAllMocks());
|
||||
|
||||
it('no provider config', () => {
|
||||
const schedule = new PersistingTaskRunner();
|
||||
const config = new ConfigReader({});
|
||||
const providers = GitHubEntityProvider.fromConfig(config, {
|
||||
logger,
|
||||
schedule,
|
||||
});
|
||||
|
||||
expect(providers).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('single simple provider config', () => {
|
||||
const schedule = new PersistingTaskRunner();
|
||||
const config = new ConfigReader({
|
||||
catalog: {
|
||||
providers: {
|
||||
github: {
|
||||
organization: 'test-org',
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const providers = GitHubEntityProvider.fromConfig(config, {
|
||||
logger,
|
||||
schedule,
|
||||
});
|
||||
|
||||
expect(providers).toHaveLength(1);
|
||||
expect(providers[0].getProviderName()).toEqual('github-provider:default');
|
||||
});
|
||||
|
||||
it('multiple provider configs', () => {
|
||||
const schedule = new PersistingTaskRunner();
|
||||
const config = new ConfigReader({
|
||||
catalog: {
|
||||
providers: {
|
||||
github: {
|
||||
myProvider: {
|
||||
organization: 'test-org1',
|
||||
},
|
||||
anotherProvider: {
|
||||
organization: 'test-org2',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const providers = GitHubEntityProvider.fromConfig(config, {
|
||||
logger,
|
||||
schedule,
|
||||
});
|
||||
|
||||
expect(providers).toHaveLength(2);
|
||||
expect(providers[0].getProviderName()).toEqual(
|
||||
'github-provider:myProvider',
|
||||
);
|
||||
expect(providers[1].getProviderName()).toEqual(
|
||||
'github-provider:anotherProvider',
|
||||
);
|
||||
});
|
||||
|
||||
it('apply full update on scheduled execution', async () => {
|
||||
const config = new ConfigReader({
|
||||
catalog: {
|
||||
providers: {
|
||||
github: {
|
||||
myProvider: {
|
||||
organization: 'test-org',
|
||||
catalogPath: 'custom/path/catalog-custom.yaml',
|
||||
filters: {
|
||||
branch: 'main',
|
||||
repository: 'test-.*',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const schedule = new PersistingTaskRunner();
|
||||
const entityProviderConnection: EntityProviderConnection = {
|
||||
applyMutation: jest.fn(),
|
||||
};
|
||||
|
||||
const provider = GitHubEntityProvider.fromConfig(config, {
|
||||
logger,
|
||||
schedule,
|
||||
})[0];
|
||||
|
||||
const mockGetOrganizationRepositories = jest.spyOn(
|
||||
helpers,
|
||||
'getOrganizationRepositories',
|
||||
);
|
||||
|
||||
mockGetOrganizationRepositories.mockReturnValue(
|
||||
Promise.resolve({
|
||||
repositories: [
|
||||
{
|
||||
name: 'test-repo',
|
||||
url: 'https://github.com/test-org/test-repo',
|
||||
isArchived: false,
|
||||
defaultBranchRef: {
|
||||
name: 'main',
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
await provider.connect(entityProviderConnection);
|
||||
|
||||
const taskDef = schedule.getTasks()[0];
|
||||
expect(taskDef.id).toEqual('github-provider:myProvider:refresh');
|
||||
await (taskDef.fn as () => Promise<void>)();
|
||||
|
||||
const url = `https://github.com/test-org/test-repo/blob/main/catalog-custom.yaml`;
|
||||
const expectedEntities = [
|
||||
{
|
||||
entity: {
|
||||
apiVersion: 'backstage.io/v1alpha1',
|
||||
kind: 'Location',
|
||||
metadata: {
|
||||
annotations: {
|
||||
'backstage.io/managed-by-location': `url:${url}`,
|
||||
'backstage.io/managed-by-origin-location': `url:${url}`,
|
||||
},
|
||||
name: 'generated-21936a3d1e926b8bb3b00ac4398dc9a8dbb90b45',
|
||||
},
|
||||
spec: {
|
||||
presence: 'optional',
|
||||
target: `${url}`,
|
||||
type: 'url',
|
||||
},
|
||||
},
|
||||
locationKey: 'github-provider:myProvider',
|
||||
},
|
||||
];
|
||||
|
||||
expect(entityProviderConnection.applyMutation).toBeCalledTimes(1);
|
||||
expect(entityProviderConnection.applyMutation).toBeCalledWith({
|
||||
type: 'full',
|
||||
entities: expectedEntities,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,253 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { TaskRunner } from '@backstage/backend-tasks';
|
||||
import { Config } from '@backstage/config';
|
||||
import {
|
||||
GithubCredentialsProvider,
|
||||
ScmIntegrations,
|
||||
GitHubIntegrationConfig,
|
||||
GitHubIntegration,
|
||||
SingleInstanceGithubCredentialsProvider,
|
||||
} from '@backstage/integration';
|
||||
import {
|
||||
EntityProvider,
|
||||
EntityProviderConnection,
|
||||
LocationSpec,
|
||||
locationSpecToLocationEntity,
|
||||
} from '@backstage/plugin-catalog-backend';
|
||||
|
||||
import { graphql } from '@octokit/graphql';
|
||||
import * as uuid from 'uuid';
|
||||
import { Logger } from 'winston';
|
||||
import {
|
||||
readProviderConfigs,
|
||||
GitHubEntityProviderConfig,
|
||||
} from './GitHubEntityProviderConfig';
|
||||
import { getOrganizationRepositories, Repository } from '../lib/github';
|
||||
|
||||
/**
|
||||
* Discovers catalog files located in [GitHub](https://github.com).
|
||||
* The provider will search your GitHub account and register catalog files matching the configured path
|
||||
* as Location entity and via following processing steps add all contained catalog entities.
|
||||
* This can be useful as an alternative to static locations or manually adding things to the catalog.
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export class GitHubEntityProvider implements EntityProvider {
|
||||
private readonly config: GitHubEntityProviderConfig;
|
||||
private readonly logger: Logger;
|
||||
private readonly integration: GitHubIntegrationConfig;
|
||||
private readonly scheduleFn: () => Promise<void>;
|
||||
private connection?: EntityProviderConnection;
|
||||
private readonly githubCredentialsProvider: GithubCredentialsProvider;
|
||||
|
||||
static fromConfig(
|
||||
config: Config,
|
||||
options: {
|
||||
logger: Logger;
|
||||
schedule: TaskRunner;
|
||||
},
|
||||
): GitHubEntityProvider[] {
|
||||
const integrations = ScmIntegrations.fromConfig(config);
|
||||
const integration = integrations.github.byHost('github.com');
|
||||
|
||||
if (!integration) {
|
||||
throw new Error(
|
||||
`There is no GitHub config that matches github. Please add a configuration entry for it under integrations.github`,
|
||||
);
|
||||
}
|
||||
|
||||
return readProviderConfigs(config).map(
|
||||
providerConfig =>
|
||||
new GitHubEntityProvider(
|
||||
providerConfig,
|
||||
integration,
|
||||
options.logger,
|
||||
options.schedule,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
private constructor(
|
||||
config: GitHubEntityProviderConfig,
|
||||
integration: GitHubIntegration,
|
||||
logger: Logger,
|
||||
schedule: TaskRunner,
|
||||
) {
|
||||
this.config = config;
|
||||
this.integration = integration.config;
|
||||
this.logger = logger.child({
|
||||
target: this.getProviderName(),
|
||||
});
|
||||
this.scheduleFn = this.createScheduleFn(schedule);
|
||||
this.githubCredentialsProvider =
|
||||
SingleInstanceGithubCredentialsProvider.create(integration.config);
|
||||
this.scheduleFn = this.createScheduleFn(schedule);
|
||||
}
|
||||
|
||||
/** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.getProviderName} */
|
||||
getProviderName(): string {
|
||||
return `github-provider:${this.config.id}`;
|
||||
}
|
||||
|
||||
/** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */
|
||||
async connect(connection: EntityProviderConnection): Promise<void> {
|
||||
this.connection = connection;
|
||||
return await this.scheduleFn();
|
||||
}
|
||||
|
||||
private createScheduleFn(schedule: TaskRunner): () => Promise<void> {
|
||||
return async () => {
|
||||
const taskId = `${this.getProviderName()}:refresh`;
|
||||
return schedule.run({
|
||||
id: taskId,
|
||||
fn: async () => {
|
||||
const logger = this.logger.child({
|
||||
class: GitHubEntityProvider.prototype.constructor.name,
|
||||
taskId,
|
||||
taskInstanceId: uuid.v4(),
|
||||
});
|
||||
try {
|
||||
await this.refresh(logger);
|
||||
} catch (error) {
|
||||
logger.error(error);
|
||||
}
|
||||
},
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
async refresh(logger: Logger) {
|
||||
if (!this.connection) {
|
||||
throw new Error('Not initialized');
|
||||
}
|
||||
|
||||
const targets = await this.findCatalogFiles();
|
||||
const matchingTargets = this.matchesFilters(targets);
|
||||
const entities = matchingTargets
|
||||
.map(repository => this.createLocationUrl(repository))
|
||||
.map(GitHubEntityProvider.toLocationSpec)
|
||||
.map(location => {
|
||||
return {
|
||||
locationKey: this.getProviderName(),
|
||||
entity: locationSpecToLocationEntity({ location }),
|
||||
};
|
||||
});
|
||||
|
||||
await this.connection.applyMutation({
|
||||
type: 'full',
|
||||
entities,
|
||||
});
|
||||
|
||||
logger.info(
|
||||
`Read ${targets.length} GitHub repositories (${entities.length} matching the pattern)`,
|
||||
);
|
||||
}
|
||||
|
||||
// go to the server and get all of the repositories
|
||||
private async findCatalogFiles(): Promise<Repository[]> {
|
||||
const organization = this.config.organization;
|
||||
const host = this.integration.host;
|
||||
const orgUrl = `https://${host}/${organization}`;
|
||||
|
||||
const { headers } = await this.githubCredentialsProvider.getCredentials({
|
||||
url: orgUrl,
|
||||
});
|
||||
|
||||
const client = graphql.defaults({
|
||||
baseUrl: this.integration.apiBaseUrl,
|
||||
headers,
|
||||
});
|
||||
|
||||
const { repositories } = await getOrganizationRepositories(
|
||||
client,
|
||||
organization,
|
||||
);
|
||||
|
||||
return repositories;
|
||||
}
|
||||
|
||||
private matchesFilters(repositories: Repository[]) {
|
||||
const repositoryFilter = this.config.filters?.repository;
|
||||
|
||||
const matchingRepositories = repositories.filter(r => {
|
||||
return !r.isArchived && repositoryFilter
|
||||
? repositoryFilter?.test(r.name)
|
||||
: true && r.defaultBranchRef?.name;
|
||||
});
|
||||
return matchingRepositories;
|
||||
}
|
||||
|
||||
private createLocationUrl(repository: Repository): string {
|
||||
const branch =
|
||||
this.config.filters?.branch || repository.defaultBranchRef?.name || '-';
|
||||
const catalogFile = this.config.catalogPath.substring(
|
||||
this.config.catalogPath.lastIndexOf('/') + 1,
|
||||
);
|
||||
return `${repository.url}/blob/${branch}/${catalogFile}`;
|
||||
}
|
||||
|
||||
private static toLocationSpec(target: string): LocationSpec {
|
||||
return {
|
||||
type: 'url',
|
||||
target: target,
|
||||
presence: 'optional',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Helpers
|
||||
*/
|
||||
|
||||
export function parseUrl(urlString: string): {
|
||||
org: string;
|
||||
repoSearchPath: RegExp;
|
||||
catalogPath: string;
|
||||
branch: string;
|
||||
host: string;
|
||||
} {
|
||||
const url = new URL(urlString);
|
||||
const path = url.pathname.substr(1).split('/');
|
||||
|
||||
// /backstage/techdocs-*/blob/master/catalog-info.yaml
|
||||
// can also be
|
||||
// /backstage
|
||||
if (path.length > 2 && path[0].length && path[1].length) {
|
||||
return {
|
||||
org: decodeURIComponent(path[0]),
|
||||
repoSearchPath: escapeRegExp(decodeURIComponent(path[1])),
|
||||
catalogPath: `/${decodeURIComponent(path.slice(4).join('/'))}`,
|
||||
branch: decodeURIComponent(path[3]),
|
||||
host: url.host,
|
||||
};
|
||||
} else if (path.length === 1 && path[0].length) {
|
||||
return {
|
||||
org: decodeURIComponent(path[0]),
|
||||
repoSearchPath: escapeRegExp('*'),
|
||||
catalogPath: '/catalog-info.yaml',
|
||||
branch: '-',
|
||||
host: url.host,
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error(`Failed to parse ${urlString}`);
|
||||
}
|
||||
|
||||
export function escapeRegExp(str: string): RegExp {
|
||||
return new RegExp(`^${str.replace(/\*/g, '.*')}$`);
|
||||
}
|
||||
+115
@@ -0,0 +1,115 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { ConfigReader } from '@backstage/config';
|
||||
import { readProviderConfigs } from './GitHubEntityProviderConfig';
|
||||
|
||||
describe('readProviderConfigs', () => {
|
||||
afterEach(() => jest.resetAllMocks());
|
||||
|
||||
it('no provider config', () => {
|
||||
const config = new ConfigReader({});
|
||||
const providerConfigs = readProviderConfigs(config);
|
||||
|
||||
expect(providerConfigs).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('single simple provider config', () => {
|
||||
const config = new ConfigReader({
|
||||
catalog: {
|
||||
providers: {
|
||||
github: {
|
||||
organization: 'test-org',
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const providerConfigs = readProviderConfigs(config);
|
||||
|
||||
expect(providerConfigs).toHaveLength(1);
|
||||
expect(providerConfigs[0].id).toEqual('default');
|
||||
expect(providerConfigs[0].organization).toEqual('test-org');
|
||||
});
|
||||
|
||||
it('multiple provider configs', () => {
|
||||
const config = new ConfigReader({
|
||||
catalog: {
|
||||
providers: {
|
||||
github: {
|
||||
providerOrganizationOnly: {
|
||||
organization: 'test-org1',
|
||||
},
|
||||
providerCustomCatalogPath: {
|
||||
organization: 'test-org2',
|
||||
catalogPath: 'custom/path/catalog-info.yaml',
|
||||
},
|
||||
providerWithRepositoryFilter: {
|
||||
organization: 'test-org3',
|
||||
filters: {
|
||||
repository: 'repository.*filter',
|
||||
},
|
||||
},
|
||||
providerWithBranchFilter: {
|
||||
organization: 'test-org4',
|
||||
filters: {
|
||||
branch: 'branch-name',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const providerConfigs = readProviderConfigs(config);
|
||||
|
||||
expect(providerConfigs).toHaveLength(4);
|
||||
expect(providerConfigs[0]).toEqual({
|
||||
id: 'providerOrganizationOnly',
|
||||
organization: 'test-org1',
|
||||
catalogPath: '/catalog-info.yaml',
|
||||
filters: {
|
||||
repository: undefined,
|
||||
branch: undefined,
|
||||
},
|
||||
});
|
||||
expect(providerConfigs[1]).toEqual({
|
||||
id: 'providerCustomCatalogPath',
|
||||
organization: 'test-org2',
|
||||
catalogPath: 'custom/path/catalog-info.yaml',
|
||||
filters: {
|
||||
repository: undefined,
|
||||
branch: undefined,
|
||||
},
|
||||
});
|
||||
expect(providerConfigs[2]).toEqual({
|
||||
id: 'providerWithRepositoryFilter',
|
||||
organization: 'test-org3', // organization
|
||||
catalogPath: '/catalog-info.yaml', // file
|
||||
filters: {
|
||||
repository: /^repository.*filter$/, // repo
|
||||
branch: undefined, // branch
|
||||
},
|
||||
});
|
||||
expect(providerConfigs[3]).toEqual({
|
||||
id: 'providerWithBranchFilter',
|
||||
organization: 'test-org4',
|
||||
catalogPath: '/catalog-info.yaml',
|
||||
filters: {
|
||||
repository: undefined,
|
||||
branch: 'branch-name',
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,90 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { Config } from '@backstage/config';
|
||||
|
||||
const DEFAULT_CATALOG_PATH = '/catalog-info.yaml';
|
||||
const DEFAULT_PROVIDER_ID = 'default';
|
||||
|
||||
export type GitHubEntityProviderConfig = {
|
||||
id: string;
|
||||
catalogPath: string;
|
||||
organization: string;
|
||||
filters?: {
|
||||
repository?: RegExp;
|
||||
branch?: string;
|
||||
};
|
||||
};
|
||||
|
||||
export function readProviderConfigs(
|
||||
config: Config,
|
||||
): GitHubEntityProviderConfig[] {
|
||||
const providersConfig = config.getOptionalConfig('catalog.providers.github');
|
||||
if (!providersConfig) {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (providersConfig.has('organization')) {
|
||||
// simple/single config variant
|
||||
return [readProviderConfig(DEFAULT_PROVIDER_ID, providersConfig)];
|
||||
}
|
||||
|
||||
return providersConfig.keys().map(id => {
|
||||
const providerConfig = providersConfig.getConfig(id);
|
||||
|
||||
return readProviderConfig(id, providerConfig);
|
||||
});
|
||||
}
|
||||
|
||||
function readProviderConfig(
|
||||
id: string,
|
||||
config: Config,
|
||||
): GitHubEntityProviderConfig {
|
||||
const organization = config.getString('organization');
|
||||
const catalogPath =
|
||||
config.getOptionalString('catalogPath') ?? DEFAULT_CATALOG_PATH;
|
||||
const repositoryPattern = config.getOptionalString('filters.repository');
|
||||
const branchPattern = config.getOptionalString('filters.branch');
|
||||
|
||||
return {
|
||||
id,
|
||||
catalogPath,
|
||||
organization,
|
||||
filters: {
|
||||
repository: repositoryPattern
|
||||
? compileRegExp(repositoryPattern)
|
||||
: undefined,
|
||||
branch: branchPattern || undefined,
|
||||
},
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Compiles a RegExp while enforcing the pattern to contain
|
||||
* the start-of-line and end-of-line anchors.
|
||||
*
|
||||
* @param pattern
|
||||
*/
|
||||
function compileRegExp(pattern: string): RegExp {
|
||||
let fullLinePattern = pattern;
|
||||
if (!fullLinePattern.startsWith('^')) {
|
||||
fullLinePattern = `^${fullLinePattern}`;
|
||||
}
|
||||
if (!fullLinePattern.endsWith('$')) {
|
||||
fullLinePattern = `${fullLinePattern}$`;
|
||||
}
|
||||
|
||||
return new RegExp(fullLinePattern);
|
||||
}
|
||||
+1
-1
@@ -42,7 +42,7 @@ import {
|
||||
getOrganizationTeams,
|
||||
getOrganizationUsers,
|
||||
parseGitHubOrgUrl,
|
||||
} from './lib';
|
||||
} from '../lib';
|
||||
|
||||
/**
|
||||
* Options for {@link GitHubOrgEntityProvider}.
|
||||
Reference in New Issue
Block a user