paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 37 Packages Projects Releases Wiki Activity

refactor: remove level config

Browse Source 
Signed-off-by: Benjamin Janssens <benji.janssens@gmail.com>
This commit is contained in:
Benjamin Janssens
2024-10-15 17:08:16 +02:00
parent a8ba324217
commit 7cc909a2c2
7 changed files with 27 additions and 246 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/wise-snakes-sleep.md
+1 -1
View File
@@ -2,4 +2,4 @@
'@backstage/plugin-catalog-backend-module-bitbucket-cloud': patch
---
Added discovery level configuration to shift Bitbucket Cloud API limits
Implemented discovery on project-level to shift Bitbucket Cloud API limits
docs/integrations/bitbucketCloud/discovery.md
-5
View File
@@ -146,7 +146,6 @@ catalog:
filters: # optional
projectKey: '^apis-.*$' # optional; RegExp
repoSlug: '^service-.*$' # optional; RegExp
level: workspace # default value
schedule: # same options as in SchedulerServiceTaskScheduleDefinition
# supports cron, ISO duration, "human duration" as used in code
frequency: { minutes: 30 }
@@ -169,8 +168,6 @@ catalog:
Regular expression used to filter results based on the project key.
- **`repoSlug`** _(optional)_:
Regular expression used to filter results based on the repo slug.
- **`level`** _(optional)_:
`'workspace'` (default) or `'project'`. At what level discovery should take place, affecting Bitbucket Cloud API limits.
- **`schedule`**:
- **`frequency`**:
How often you want the task to run. The system does its best to avoid overlapping invocations.
@@ -183,5 +180,3 @@ catalog:
- **`workspace`**:
Name of your organization account/workspace.
If you want to add multiple workspaces, you need to add one provider config each.
> **Note:** By default, discovery will take place at the `workspace` level. While being the most efficient in terms of API calls to Bitbucket Cloud, discovery at the workspace level is limited to 900 repositories per workspace. If your workspace consists of more than 900 repositories, you should switch to discovery at the `project` level, shifting the limit to 900 repositories per project.
plugins/catalog-backend-module-bitbucket-cloud/config.d.ts
Vendored
-16
View File
@@ -55,14 +55,6 @@ export interface Config {
*/
projectKey?: string;
};
/**
* (Optional) At what level discovery should take place, affecting Bitbucket Cloud API limits.
*
* Possible values:
* - `workspace` (default): 1 API call per workspace, limited to 900 repositories per workspace.
* - `project`: 1 API call per project, limited to 900 repositories per project.
*/
level?: 'workspace' | 'project';
/**
* (Optional) TaskScheduleDefinition for the discovery.
*/
@@ -96,14 +88,6 @@ export interface Config {
*/
projectKey?: string;
};
/**
* (Optional) At what level discovery should take place, affecting Bitbucket Cloud API limits.
*
* Possible values:
* - `workspace` (default): 1 API call per workspace, limited to 900 repositories per workspace.
* - `project`: 1 API call per project, limited to 900 repositories per project.
*/
level?: 'workspace' | 'project';
/**
* (Optional) TaskScheduleDefinition for the discovery.
*/
plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.test.ts
+14 -180
View File
@@ -87,23 +87,6 @@ describe('BitbucketCloudEntityProvider', () => {
},
},
});
const projectLevelConfig = new ConfigReader({
catalog: {
providers: {
bitbucketCloud: {
myProvider: {
workspace: 'test-ws',
catalogPath: 'catalog-custom.yaml',
filters: {
projectKey: 'test-.*',
repoSlug: 'test-.*',
},
level: 'project',
},
},
},
},
});
const schedule = new PersistingTaskRunner();
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
@@ -291,163 +274,6 @@ describe('BitbucketCloudEntityProvider', () => {
'bitbucketCloud-provider:myProvider',
);
server.use(
rest.get(
`https://api.bitbucket.org/2.0/workspaces/test-ws/search/code`,
(_req, res, ctx) => {
const response = {
values: [
{
// skipped as empty
path_matches: [],
file: {
type: 'commit_file',
path: 'path/to/ignored/file',
},
},
{
path_matches: [
{
match: true,
text: 'catalog-custom.yaml',
},
],
file: {
type: 'commit_file',
path: 'custom/path/catalog-custom.yaml',
commit: {
repository: {
// skipped as no match with filter
slug: 'repo',
project: {
key: 'test-project',
},
mainbranch: {
name: 'main',
},
links: {
html: {
href: 'https://bitbucket.org/test-ws/repo',
},
},
},
},
},
},
{
path_matches: [
{
match: true,
text: 'catalog-custom.yaml',
},
],
file: {
type: 'commit_file',
path: 'custom/path/catalog-custom.yaml',
commit: {
repository: {
slug: 'test-repo1',
project: {
// skipped as no match with filter
key: 'project',
},
mainbranch: {
name: 'main',
},
links: {
html: {
href: 'https://bitbucket.org/test-ws/test-repo1',
},
},
},
},
},
},
{
path_matches: [
{
match: true,
text: 'catalog-custom.yaml',
},
],
file: {
type: 'commit_file',
path: 'custom/path/catalog-custom.yaml',
commit: {
repository: {
slug: 'test-repo2',
project: {
key: 'test-project',
},
mainbranch: {
name: 'main',
},
links: {
html: {
href: 'https://bitbucket.org/test-ws/test-repo2',
},
},
},
},
},
},
],
};
return res(ctx.json(response));
},
),
);
await provider.connect(entityProviderConnection);
const taskDef = schedule.getTasks()[0];
expect(taskDef.id).toEqual('bitbucketCloud-provider:myProvider:refresh');
await (taskDef.fn as () => Promise<void>)();
const url = `https://bitbucket.org/test-ws/test-repo2/src/main/custom/path/catalog-custom.yaml`;
const expectedEntities = [
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Location',
metadata: {
annotations: {
'backstage.io/managed-by-location': `url:${url}`,
'backstage.io/managed-by-origin-location': `url:${url}`,
'bitbucket.org/repo-url':
'https://bitbucket.org/test-ws/test-repo2',
},
name: 'generated-7c2e6263b6cc2d14e69fd4d029afba601ad6dc3b',
},
spec: {
presence: 'required',
target: `${url}`,
type: 'url',
},
},
locationKey: 'bitbucketCloud-provider:myProvider',
},
];
expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1);
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
type: 'full',
entities: expectedEntities,
});
});
it('apply full update on scheduled execution on project level', async () => {
const provider = BitbucketCloudEntityProvider.fromConfig(
projectLevelConfig,
{
logger,
schedule,
},
)[0];
expect(provider.getProviderName()).toEqual(
'bitbucketCloud-provider:myProvider',
);
server.use(
rest.get(
`https://api.bitbucket.org/2.0/workspaces/test-ws/projects`,
@@ -464,12 +290,7 @@ describe('BitbucketCloudEntityProvider', () => {
),
rest.get(
`https://api.bitbucket.org/2.0/workspaces/test-ws/search/code`,
(req, res, ctx) => {
const query = req.url.searchParams.get('search_query');
if (!query || !query.includes('project:TEST')) {
return res(ctx.json({ values: [] }));
}
(_req, res, ctx) => {
const response = {
values: [
{
@@ -657,6 +478,19 @@ describe('BitbucketCloudEntityProvider', () => {
})[0];
server.use(
rest.get(
`https://api.bitbucket.org/2.0/workspaces/test-ws/projects`,
(_req, res, ctx) => {
const response = {
values: [
{
key: 'TEST',
},
],
};
return res(ctx.json(response));
},
),
rest.get(
`https://api.bitbucket.org/2.0/workspaces/test-ws/search/code`,
(req, res, ctx) => {
plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProvider.ts
+11 -16
View File
@@ -207,7 +207,7 @@ export class BitbucketCloudEntityProvider implements EntityProvider {
logger.info('Discovering catalog files in Bitbucket Cloud repositories');
const targets = await this.findCatalogFiles(this.config.level);
const targets = await this.findCatalogFiles();
const entities = this.toDeferredEntities(targets);
await this.connection.applyMutation({
@@ -271,7 +271,7 @@ export class BitbucketCloudEntityProvider implements EntityProvider {
// Hence, we will just trigger a refresh for catalog file(s) within the repository
// if we get notified about changes there.
const targets = await this.findCatalogFiles('workspace', repoSlug);
const targets = await this.findCatalogFiles(repoSlug);
const { token } = await this.tokenManager!.getToken();
const existing = await this.findExistingLocations(repoUrl, token);
@@ -334,7 +334,6 @@ export class BitbucketCloudEntityProvider implements EntityProvider {
}
private async findCatalogFiles(
level: 'workspace' | 'project',
repoSlug?: string,
): Promise<IngestionTarget[]> {
const workspace = this.config.workspace;
@@ -347,23 +346,19 @@ export class BitbucketCloudEntityProvider implements EntityProvider {
const optRepoFilter = repoSlug ? ` repo:${repoSlug}` : '';
const query = `"${catalogFilename}" path:${catalogPath}${optRepoFilter}`;
if (level === 'project') {
const projects = this.client
.listProjectsByWorkspace(workspace)
.iterateResults();
const projects = this.client
.listProjectsByWorkspace(workspace)
.iterateResults();
let results: IngestionTarget[] = [];
let results: IngestionTarget[] = [];
for await (const project of projects) {
const projectQuery = `${query} project:${project.key}`;
const result = await this.processQuery(workspace, projectQuery);
results = results.concat(result);
}
return results;
for await (const project of projects) {
const projectQuery = `${query} project:${project.key}`;
const result = await this.processQuery(workspace, projectQuery);
results = results.concat(result);
}
return this.processQuery(workspace, query);
return results;
}
private async processQuery(
plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.test.ts
+1 -20
View File
@@ -77,17 +77,13 @@ describe('readProviderConfigs', () => {
},
},
},
providerWithProjectLevel: {
workspace: 'test-ws6',
level: 'project',
},
},
},
},
});
const providerConfigs = readProviderConfigs(config);
expect(providerConfigs).toHaveLength(6);
expect(providerConfigs).toHaveLength(5);
expect(providerConfigs[0]).toEqual({
id: 'providerWorkspaceOnly',
workspace: 'test-ws1',
@@ -96,7 +92,6 @@ describe('readProviderConfigs', () => {
projectKey: undefined,
repoSlug: undefined,
},
level: 'workspace',
});
expect(providerConfigs[1]).toEqual({
id: 'providerCustomCatalogPath',
@@ -106,7 +101,6 @@ describe('readProviderConfigs', () => {
projectKey: undefined,
repoSlug: undefined,
},
level: 'workspace',
});
expect(providerConfigs[2]).toEqual({
id: 'providerWithProjectKeyFilter',
@@ -116,7 +110,6 @@ describe('readProviderConfigs', () => {
projectKey: /^projectKey.*filter$/,
repoSlug: undefined,
},
level: 'workspace',
});
expect(providerConfigs[3]).toEqual({
id: 'providerWithRepoSlugFilter',
@@ -126,7 +119,6 @@ describe('readProviderConfigs', () => {
projectKey: undefined,
repoSlug: /^repoSlug.*filter$/,
},
level: 'workspace',
});
expect(providerConfigs[4]).toEqual({
id: 'providerWithSchedule',
@@ -142,17 +134,6 @@ describe('readProviderConfigs', () => {
minutes: 3,
},
},
level: 'workspace',
});
expect(providerConfigs[5]).toEqual({
id: 'providerWithProjectLevel',
workspace: 'test-ws6',
catalogPath: '/catalog-info.yaml',
filters: {
projectKey: undefined,
repoSlug: undefined,
},
level: 'project',
});
});
});
plugins/catalog-backend-module-bitbucket-cloud/src/providers/BitbucketCloudEntityProviderConfig.ts
-8
View File
@@ -32,7 +32,6 @@ export type BitbucketCloudEntityProviderConfig = {
repoSlug?: RegExp;
};
schedule?: SchedulerServiceTaskScheduleDefinition;
level: 'workspace' | 'project';
};
export function readProviderConfigs(
@@ -73,12 +72,6 @@ function readProviderConfig(
)
: undefined;
const level =
(config.getOptionalString('level') as
| 'workspace'
| 'project'
| undefined) ?? 'workspace';
return {
id,
catalogPath,
@@ -90,7 +83,6 @@ function readProviderConfig(
repoSlug: repoSlugPattern ? compileRegExp(repoSlugPattern) : undefined,
},
schedule,
level,
};
}