From 1416e69de9f73601094bf0a84631036f7ee04a46 Mon Sep 17 00:00:00 2001 From: Nikolas Skoufis Date: Mon, 10 Oct 2022 16:32:19 +1100 Subject: [PATCH 1/9] Add a config field to configure entity validation This adds a new config field that can be used to configure entity validation. It defaults to false to ensure backwards compatibility. It is not permitted when catalog paths contain wildcards, due to limitations with how the GraphQL query will work. Signed-off-by: Nikolas Skoufis --- .../GitHubEntityProviderConfig.test.ts | 37 +++++++++++++++++++ .../providers/GitHubEntityProviderConfig.ts | 12 ++++++ 2 files changed, 49 insertions(+) diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts index 66a5463bf5..16f9d96010 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.test.ts @@ -101,6 +101,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, }, + validateLocationsExist: false, }); expect(providerConfigs[1]).toEqual({ id: 'providerCustomCatalogPath', @@ -115,6 +116,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, }, + validateLocationsExist: false, }); expect(providerConfigs[2]).toEqual({ id: 'providerWithRepositoryFilter', @@ -129,6 +131,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, }, + validateLocationsExist: false, }); expect(providerConfigs[3]).toEqual({ id: 'providerWithBranchFilter', @@ -143,6 +146,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, }, + validateLocationsExist: false, }); expect(providerConfigs[4]).toEqual({ id: 'providerWithTopicFilter', @@ -157,6 +161,7 @@ describe('readProviderConfigs', () => { exclude: ['backstage-exclude'], }, }, + validateLocationsExist: false, }); expect(providerConfigs[5]).toEqual({ id: 'providerWithHost', @@ -171,6 +176,38 @@ describe('readProviderConfigs', () => { exclude: undefined, }, }, + validateLocationsExist: false, }); }); + + it('defaults validateLocationsExist to false', () => { + const config = new ConfigReader({ + catalog: { + providers: { + github: { + organization: 'test-org', + }, + }, + }, + }); + const providerConfigs = readProviderConfigs(config); + + expect(providerConfigs[0].validateLocationsExist).toEqual(false); + }); + + it('throws an error when a wildcard catalog path is configured with validation of locations', () => { + const config = new ConfigReader({ + catalog: { + providers: { + github: { + organization: 'test-org', + validateLocationsExist: true, + catalogPath: '/*/catalog-info.yaml', + }, + }, + }, + }); + + expect(() => readProviderConfigs(config)).toThrow(); + }); }); diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts index d40b206751..fd5482d1b6 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProviderConfig.ts @@ -29,6 +29,7 @@ export type GitHubEntityProviderConfig = { branch?: string; topic?: GithubTopicFilters; }; + validateLocationsExist: boolean; }; export type GithubTopicFilters = { @@ -72,6 +73,16 @@ function readProviderConfig( const topicFilterExclude = config?.getOptionalStringArray( 'filters.topic.exclude', ); + const validateLocationsExist = + config?.getOptionalBoolean('validateLocationsExist') ?? false; + + const catalogPathContainsWildcard = catalogPath.includes('*'); + + if (validateLocationsExist && catalogPathContainsWildcard) { + throw Error( + `Error while processing GitHub provider config. The catalog path ${catalogPath} contains a wildcard, which is incompatible with validation of locations existing before emitting them. Ensure that validateLocationsExist is set to false.`, + ); + } return { id, @@ -88,6 +99,7 @@ function readProviderConfig( exclude: topicFilterExclude, }, }, + validateLocationsExist, }; } /** From 143cae25da50e978cff03053c9e0f3ce1f7797af Mon Sep 17 00:00:00 2001 From: Nikolas Skoufis Date: Mon, 10 Oct 2022 17:15:40 +1100 Subject: [PATCH 2/9] Implement filtering of github entities based on presence This implements the config from the previous commit. The getRepositories graphql query is updated to include information about the specified catalog info file. When validation is turned on, if the file is not present or empty, the location will not be emitted. Signed-off-by: Nikolas Skoufis --- .../src/lib/github.test.ts | 18 ++- .../src/lib/github.ts | 17 ++- .../providers/GitHubEntityProvider.test.ts | 134 ++++++++++++++++++ .../src/providers/GitHubEntityProvider.ts | 11 ++ 4 files changed, 175 insertions(+), 5 deletions(-) diff --git a/plugins/catalog-backend-module-github/src/lib/github.test.ts b/plugins/catalog-backend-module-github/src/lib/github.test.ts index e2303c16e6..25878c2005 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.test.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.test.ts @@ -213,6 +213,7 @@ describe('github', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: null, }, { name: 'demo', @@ -222,6 +223,11 @@ describe('github', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: { + __typename: 'Blob', + id: 'acb123', + text: 'some yaml', + }, }, ], pageInfo: { @@ -243,6 +249,7 @@ describe('github', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: null, }, { name: 'demo', @@ -252,6 +259,11 @@ describe('github', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: { + __typename: 'Blob', + id: 'acb123', + text: 'some yaml', + }, }, ], }; @@ -262,9 +274,9 @@ describe('github', () => { ), ); - await expect(getOrganizationRepositories(graphql, 'a')).resolves.toEqual( - output, - ); + await expect( + getOrganizationRepositories(graphql, 'a', 'catalog-info.yaml'), + ).resolves.toEqual(output); }); }); }); diff --git a/plugins/catalog-backend-module-github/src/lib/github.ts b/plugins/catalog-backend-module-github/src/lib/github.ts index 6a944af330..d854cc9043 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.ts @@ -65,6 +65,11 @@ export type Repository = { defaultBranchRef: { name: string; } | null; + catalogInfoFile: { + __typename: string; + id: string; + text: string; + } | null; }; type RepositoryTopics = { @@ -266,14 +271,22 @@ export async function getOrganizationTeams( export async function getOrganizationRepositories( client: typeof graphql, org: string, + catalogPath: string, ): Promise<{ repositories: Repository[] }> { + const catalogPathRef = `HEAD:${catalogPath}`; const query = ` - query repositories($org: String!, $cursor: String) { + query repositories($org: String!, $catalogPathRef: String!, $cursor: String) { repositoryOwner(login: $org) { login repositories(first: 100, after: $cursor) { nodes { name + catalogInfoFile: object(expression: $catalogPathRef) { + __typename + ... on Blob { + id + } + } url isArchived repositoryTopics(first: 100) { @@ -302,7 +315,7 @@ export async function getOrganizationRepositories( query, r => r.repositoryOwner?.repositories, x => x, - { org }, + { org, catalogPathRef }, ); return { repositories }; diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts index 63fb77288b..8b315a8a1e 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.test.ts @@ -170,6 +170,11 @@ describe('GitHubEntityProvider', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: { + __typename: 'Blob', + id: 'abc123', + text: 'some yaml', + }, }, ], }), @@ -267,6 +272,11 @@ describe('GitHubEntityProvider', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: { + __typename: 'Blob', + id: 'abc123', + text: 'some yaml', + }, }, ], }), @@ -341,6 +351,11 @@ describe('GitHubEntityProvider', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: { + __typename: 'Blob', + id: 'abc123', + text: 'some yaml', + }, }, ], }), @@ -381,6 +396,110 @@ describe('GitHubEntityProvider', () => { entities: expectedEntities, }); }); + + it('should filter out invalid locations when validateLocationsExist is set to true', async () => { + const config = new ConfigReader({ + catalog: { + providers: { + github: { + myProvider: { + organization: 'test-org', + catalogPath: 'catalog-custom.yaml', + filters: { + branch: 'main', + }, + validateLocationsExist: true, + }, + }, + }, + }, + }); + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + const provider = GitHubEntityProvider.fromConfig(config, { + logger, + schedule, + })[0]; + + const mockGetOrganizationRepositories = jest.spyOn( + helpers, + 'getOrganizationRepositories', + ); + + mockGetOrganizationRepositories.mockReturnValue( + Promise.resolve({ + repositories: [ + { + name: 'test-repo', + url: 'https://github.com/test-org/test-repo', + repositoryTopics: { + nodes: [], + }, + isArchived: false, + defaultBranchRef: { + name: 'main', + }, + catalogInfoFile: null, + }, + { + name: 'another-repo', + url: 'https://github.com/test-org/another-repo', + repositoryTopics: { + nodes: [], + }, + isArchived: false, + defaultBranchRef: { + name: 'main', + }, + catalogInfoFile: { + __typename: 'Blob', + id: 'abc123', + text: 'some yaml', + }, + }, + ], + }), + ); + + await provider.connect(entityProviderConnection); + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual('github-provider:myProvider:refresh'); + await (taskDef.fn as () => Promise)(); + + const url = `https://github.com/test-org/another-repo/blob/main/catalog-custom.yaml`; + const expectedEntities = [ + { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${url}`, + 'backstage.io/managed-by-origin-location': `url:${url}`, + }, + name: 'generated-934f500db2ba2e8ea3524567926f45a73bb0b532', + }, + spec: { + presence: 'optional', + target: `${url}`, + type: 'url', + }, + }, + locationKey: 'github-provider:myProvider', + }, + ]; + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledTimes(1); + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: expectedEntities, + }); + }); }); it('apply full update on scheduled execution with topic exclusion taking priority over topic inclusion', async () => { @@ -437,6 +556,11 @@ it('apply full update on scheduled execution with topic exclusion taking priorit defaultBranchRef: { name: 'main', }, + catalogInfoFile: { + __typename: 'Blob', + id: 'abc123', + text: 'some yaml', + }, }, { name: 'test-repo-2', @@ -455,6 +579,11 @@ it('apply full update on scheduled execution with topic exclusion taking priorit defaultBranchRef: { name: 'main', }, + catalogInfoFile: { + __typename: 'Blob', + id: 'abc123', + text: 'some yaml', + }, }, { name: 'test-repo-3', @@ -470,6 +599,11 @@ it('apply full update on scheduled execution with topic exclusion taking priorit defaultBranchRef: { name: 'main', }, + catalogInfoFile: { + __typename: 'Blob', + id: 'abc123', + text: 'some yaml', + }, }, ], }), diff --git a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts index 3909b5a3c2..42dd6afa5d 100644 --- a/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts +++ b/plugins/catalog-backend-module-github/src/providers/GitHubEntityProvider.ts @@ -163,6 +163,7 @@ export class GitHubEntityProvider implements EntityProvider { private async findCatalogFiles(): Promise { const organization = this.config.organization; const host = this.integration.host; + const catalogPath = this.config.catalogPath; const orgUrl = `https://${host}/${organization}`; const { headers } = await this.githubCredentialsProvider.getCredentials({ @@ -177,8 +178,18 @@ export class GitHubEntityProvider implements EntityProvider { const { repositories } = await getOrganizationRepositories( client, organization, + catalogPath, ); + if (this.config.validateLocationsExist) { + return repositories.filter(repository => { + return ( + repository.catalogInfoFile?.__typename === 'Blob' && + repository.catalogInfoFile.text !== '' + ); + }); + } + return repositories; } From abab3ce38de8f4f2ed3ff09709a87c7b43a9a17a Mon Sep 17 00:00:00 2001 From: Nikolas Skoufis Date: Mon, 10 Oct 2022 17:23:00 +1100 Subject: [PATCH 3/9] Add docs for the new validateLocationsExist option Signed-off-by: Nikolas Skoufis --- docs/integrations/github/discovery.md | 16 +++++++++++++++- .../processors/GithubDiscoveryProcessor.test.ts | 13 +++++++++++++ .../src/processors/GithubDiscoveryProcessor.ts | 6 +++++- 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/docs/integrations/github/discovery.md b/docs/integrations/github/discovery.md index 9b60b41682..49ee712c6e 100644 --- a/docs/integrations/github/discovery.md +++ b/docs/integrations/github/discovery.md @@ -96,6 +96,13 @@ catalog: topic: include: ['backstage-include'] # optional array of strings exclude: ['experiments'] # optional array of strings + validateLocationsExist: + organization: 'backstage' # string + catalogPath: '/catalog-info.yaml' # string + filters: + branch: 'main' # string + repository: '.*' # Regex + validateLocationsExist: true # optional boolean enterpriseProviderId: host: ghe.example.net organization: 'backstage' # string @@ -110,7 +117,8 @@ This provider supports multiple organizations via unique provider IDs. - **`catalogPath`** _(optional)_: Default: `/catalog-info.yaml`. Path where to look for `catalog-info.yaml` files. - You can use wildcards - `*` or `**` - to search the path and/or the filename + You can use wildcards - `*` or `**` - to search the path and/or the filename. + Wildcards cannot be used if the `validateLocationsExist` option is set to `true`. - **filters** _(optional)_: - **branch** _(optional)_: String used to filter results based on the branch name. @@ -131,6 +139,12 @@ This provider supports multiple organizations via unique provider IDs. If you want to add multiple organizations, you need to add one provider config each. - **host** _(optional)_: The hostname of your GitHub Enterprise instance. It must match a host defined in [integrations.github](locations.md). +- **validateLocationsExist** _(optional)_: + Whether to validate locations that exist before emitting them. + This option avoids generating locations for catalog info files that do not exist in the source repository. + Defaults to `false`. + Due to limitations in the GitHub API's ability to query for repository objects, this option cannot be used in + conjunction with wildcards in the `catalogPath`. ## GitHub API Rate Limits diff --git a/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts b/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts index c915e8b136..b69728cc62 100644 --- a/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.test.ts @@ -153,6 +153,7 @@ describe('GithubDiscoveryProcessor', () => { defaultBranchRef: { name: 'master', }, + catalogInfoFile: null, }, { name: 'demo', @@ -162,6 +163,7 @@ describe('GithubDiscoveryProcessor', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: null, }, ], }); @@ -203,6 +205,7 @@ describe('GithubDiscoveryProcessor', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: null, }, ], }); @@ -234,6 +237,7 @@ describe('GithubDiscoveryProcessor', () => { repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: null, + catalogInfoFile: null, }, ], }); @@ -259,6 +263,7 @@ describe('GithubDiscoveryProcessor', () => { defaultBranchRef: { name: 'master', }, + catalogInfoFile: null, }, ], }); @@ -293,6 +298,7 @@ describe('GithubDiscoveryProcessor', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: null, }, { name: 'techdocs-cli', @@ -302,6 +308,7 @@ describe('GithubDiscoveryProcessor', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: null, }, { name: 'techdocs-container', @@ -311,6 +318,7 @@ describe('GithubDiscoveryProcessor', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: null, }, { name: 'techdocs-durp', @@ -318,6 +326,7 @@ describe('GithubDiscoveryProcessor', () => { repositoryTopics: { nodes: [] }, isArchived: false, defaultBranchRef: null, + catalogInfoFile: null, }, ], }); @@ -360,6 +369,7 @@ describe('GithubDiscoveryProcessor', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: null, }, { name: 'test', @@ -369,6 +379,7 @@ describe('GithubDiscoveryProcessor', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: null, }, { name: 'test-archived', @@ -378,6 +389,7 @@ describe('GithubDiscoveryProcessor', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: null, }, { name: 'testxyz', @@ -387,6 +399,7 @@ describe('GithubDiscoveryProcessor', () => { defaultBranchRef: { name: 'main', }, + catalogInfoFile: null, }, ], }); diff --git a/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.ts b/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.ts index 97c77bf033..593f5f72f1 100644 --- a/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.ts +++ b/plugins/catalog-backend-module-github/src/processors/GithubDiscoveryProcessor.ts @@ -121,7 +121,11 @@ export class GithubDiscoveryProcessor implements CatalogProcessor { const startTimestamp = Date.now(); this.logger.info(`Reading GitHub repositories from ${location.target}`); - const { repositories } = await getOrganizationRepositories(client, org); + const { repositories } = await getOrganizationRepositories( + client, + org, + catalogPath, + ); const matching = repositories.filter( r => !r.isArchived && repoSearchPath.test(r.name), ); From f64d66a45c2ffa9c29b4e161aecc6cc9c6a81c81 Mon Sep 17 00:00:00 2001 From: Nikolas Skoufis Date: Mon, 10 Oct 2022 17:32:39 +1100 Subject: [PATCH 4/9] Add a changeset for my changes Signed-off-by: Nikolas Skoufis --- .changeset/three-poems-think.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .changeset/three-poems-think.md diff --git a/.changeset/three-poems-think.md b/.changeset/three-poems-think.md new file mode 100644 index 0000000000..13fc9cd328 --- /dev/null +++ b/.changeset/three-poems-think.md @@ -0,0 +1,14 @@ +--- +'@backstage/plugin-catalog-backend-module-github': minor +--- + +Added the ability for the GitHub discovery processor to validate that catalog files exist before emitting them. + +Users can now set the `validateLocationsExist` property to `true` in their GitHub discovery configuration to opt in to this feature. +This feature only works with `catalogPath`s that do not contain wildcards. + +When `validateLocationsExist` is set to `true`, the GitHub discovery processor will retrieve the object from the +repository at the provided `catalogPath`. +If this file exists and is non-empty, then it will be emitted as a location for further processing. +If this file does not exist or is empty, then it will not be emitted. +Not emitting locations that do not exist allows for far fewer calls to the GitHub API to validate locations that do not exist. From f9f2bcaa121d84bf02b7357480b86e3fcd1ac58d Mon Sep 17 00:00:00 2001 From: Nikolas Skoufis Date: Mon, 10 Oct 2022 18:00:13 +1100 Subject: [PATCH 5/9] Fix referring to the processor when I meant provider Signed-off-by: Nikolas Skoufis --- .changeset/three-poems-think.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.changeset/three-poems-think.md b/.changeset/three-poems-think.md index 13fc9cd328..aad3cae117 100644 --- a/.changeset/three-poems-think.md +++ b/.changeset/three-poems-think.md @@ -2,12 +2,12 @@ '@backstage/plugin-catalog-backend-module-github': minor --- -Added the ability for the GitHub discovery processor to validate that catalog files exist before emitting them. +Added the ability for the GitHub discovery provider to validate that catalog files exist before emitting them. Users can now set the `validateLocationsExist` property to `true` in their GitHub discovery configuration to opt in to this feature. This feature only works with `catalogPath`s that do not contain wildcards. -When `validateLocationsExist` is set to `true`, the GitHub discovery processor will retrieve the object from the +When `validateLocationsExist` is set to `true`, the GitHub discovery provider will retrieve the object from the repository at the provided `catalogPath`. If this file exists and is non-empty, then it will be emitted as a location for further processing. If this file does not exist or is empty, then it will not be emitted. From 0f0bbd70fe5898e9453ee8080ee6faca4598ae3e Mon Sep 17 00:00:00 2001 From: Nikolas Skoufis Date: Mon, 10 Oct 2022 18:19:05 +1100 Subject: [PATCH 6/9] Add back text field to query Signed-off-by: Nikolas Skoufis --- plugins/catalog-backend-module-github/src/lib/github.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/catalog-backend-module-github/src/lib/github.ts b/plugins/catalog-backend-module-github/src/lib/github.ts index d854cc9043..7fc69d0f86 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.ts @@ -285,6 +285,7 @@ export async function getOrganizationRepositories( __typename ... on Blob { id + text } } url From 4a5fd284ee536ecde94967f24596b99a0b965945 Mon Sep 17 00:00:00 2001 From: Nikolas Skoufis Date: Mon, 10 Oct 2022 18:31:00 +1100 Subject: [PATCH 7/9] Strip leading slash if present in catalog path ref Without this, the graphql query fails to return matching catalog paths Signed-off-by: Nikolas Skoufis --- plugins/catalog-backend-module-github/src/lib/github.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-github/src/lib/github.ts b/plugins/catalog-backend-module-github/src/lib/github.ts index 7fc69d0f86..ef6a7f817a 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.ts @@ -273,7 +273,14 @@ export async function getOrganizationRepositories( org: string, catalogPath: string, ): Promise<{ repositories: Repository[] }> { - const catalogPathRef = `HEAD:${catalogPath}`; + let relativeCatalogPathRef: string; + // We must strip the leading slash or the query for objects does not work + if (catalogPath.startsWith('/')) { + relativeCatalogPathRef = catalogPath.substring(1); + } else { + relativeCatalogPathRef = catalogPath; + } + const catalogPathRef = `HEAD:${relativeCatalogPathRef}`; const query = ` query repositories($org: String!, $catalogPathRef: String!, $cursor: String) { repositoryOwner(login: $org) { From 334dd9042c47a66442c044e2d4184eb45cce1520 Mon Sep 17 00:00:00 2001 From: Nikolas Skoufis Date: Mon, 10 Oct 2022 18:33:50 +1100 Subject: [PATCH 8/9] Fix linting issue in docs Signed-off-by: Nikolas Skoufis --- docs/integrations/github/discovery.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/github/discovery.md b/docs/integrations/github/discovery.md index 49ee712c6e..742f40f8af 100644 --- a/docs/integrations/github/discovery.md +++ b/docs/integrations/github/discovery.md @@ -143,7 +143,7 @@ This provider supports multiple organizations via unique provider IDs. Whether to validate locations that exist before emitting them. This option avoids generating locations for catalog info files that do not exist in the source repository. Defaults to `false`. - Due to limitations in the GitHub API's ability to query for repository objects, this option cannot be used in + Due to limitations in the GitHub API's ability to query for repository objects, this option cannot be used in conjunction with wildcards in the `catalogPath`. ## GitHub API Rate Limits From e2d088426bddfdd1b733f80f068631583e1a69a3 Mon Sep 17 00:00:00 2001 From: Nikolas Skoufis Date: Thu, 20 Oct 2022 11:13:07 +1100 Subject: [PATCH 9/9] Fix failing tests from merge Signed-off-by: Nikolas Skoufis --- .../providers/GithubEntityProvider.test.ts | 126 +++++++++--------- .../GithubEntityProviderConfig.test.ts | 1 + 2 files changed, 64 insertions(+), 63 deletions(-) diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts index 3dfea8ed67..cc9a9bfee3 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProvider.test.ts @@ -424,7 +424,7 @@ describe('GithubEntityProvider', () => { refresh: jest.fn(), }; - const provider = GitHubEntityProvider.fromConfig(config, { + const provider = GithubEntityProvider.fromConfig(config, { logger, schedule, })[0]; @@ -542,70 +542,70 @@ describe('GithubEntityProvider', () => { 'getOrganizationRepositories', ); - mockGetOrganizationRepositories.mockReturnValue( - Promise.resolve({ - repositories: [ - { - name: 'test-repo', - url: 'https://github.com/test-org/test-repo', - repositoryTopics: { - nodes: [ - { - topic: { name: 'backstage-include' }, - }, - ], + mockGetOrganizationRepositories.mockReturnValue( + Promise.resolve({ + repositories: [ + { + name: 'test-repo', + url: 'https://github.com/test-org/test-repo', + repositoryTopics: { + nodes: [ + { + topic: { name: 'backstage-include' }, + }, + ], + }, + isArchived: false, + defaultBranchRef: { + name: 'main', + }, + catalogInfoFile: { + __typename: 'Blob', + id: 'abc123', + text: 'some yaml', + }, }, - isArchived: false, - defaultBranchRef: { - name: 'main', + { + name: 'test-repo-2', + url: 'https://github.com/test-org/test-repo-2', + repositoryTopics: { + nodes: [ + { + topic: { name: 'backstage-include' }, + }, + { + topic: { name: 'backstage-exclude' }, + }, + ], + }, + isArchived: false, + defaultBranchRef: { + name: 'main', + }, + catalogInfoFile: { + __typename: 'Blob', + id: 'abc123', + text: 'some yaml', + }, }, - catalogInfoFile: { - __typename: 'Blob', - id: 'abc123', - text: 'some yaml', - }, - }, - { - name: 'test-repo-2', - url: 'https://github.com/test-org/test-repo-2', - repositoryTopics: { - nodes: [ - { - topic: { name: 'backstage-include' }, - }, - { - topic: { name: 'backstage-exclude' }, - }, - ], - }, - isArchived: false, - defaultBranchRef: { - name: 'main', - }, - catalogInfoFile: { - __typename: 'Blob', - id: 'abc123', - text: 'some yaml', - }, - }, - { - name: 'test-repo-3', - url: 'https://github.com/test-org/test-repo-3', - repositoryTopics: { - nodes: [ - { - topic: { name: 'backstage-exclude' }, - }, - ], - }, - isArchived: false, - defaultBranchRef: { - name: 'main', - }, - catalogInfoFile: { - __typename: 'Blob', - id: 'abc123', - text: 'some yaml', + { + name: 'test-repo-3', + url: 'https://github.com/test-org/test-repo-3', + repositoryTopics: { + nodes: [ + { + topic: { name: 'backstage-exclude' }, + }, + ], + }, + isArchived: false, + defaultBranchRef: { + name: 'main', + }, + catalogInfoFile: { + __typename: 'Blob', + id: 'abc123', + text: 'some yaml', }, }, ], diff --git a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts index 8ca9e76aca..0d9b5ce65b 100644 --- a/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts +++ b/plugins/catalog-backend-module-github/src/providers/GithubEntityProviderConfig.test.ts @@ -191,6 +191,7 @@ describe('readProviderConfigs', () => { exclude: undefined, }, }, + validateLocationsExist: false, schedule: undefined, }); expect(providerConfigs[6]).toEqual({