From 382f8f8f84db1e3e95ba448a5f0c3a8cd6748c53 Mon Sep 17 00:00:00 2001 From: Simon Knittel Date: Fri, 30 Apr 2021 15:19:25 +0200 Subject: [PATCH 1/3] Add support for non-organization accounts in GitHub Discovery Signed-off-by: Simon Knittel --- .../src/ingestion/processors/github/github.ts | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/github/github.ts b/plugins/catalog-backend/src/ingestion/processors/github/github.ts index e07ea8917b..0b7f3d831d 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/github.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/github.ts @@ -21,6 +21,7 @@ import { graphql } from '@octokit/graphql'; export type QueryResponse = { organization: Organization; + user: User; }; export type Organization = { @@ -41,6 +42,7 @@ export type User = { avatarUrl?: string; email?: string; name?: string; + repositories?: Connection; }; export type Team = { @@ -243,13 +245,27 @@ export async function getOrganizationRepositories( } } } + user(login: $org) { + name + repositories(first: 100, after: $cursor) { + nodes { + name + url + isArchived + } + pageInfo { + hasNextPage + endCursor + } + } + } } `; const repositories = await queryWithPaging( client, query, - r => r.organization?.repositories, + r => r.organization?.repositories || r.user?.repositories, x => x, { org }, ); @@ -327,10 +343,16 @@ export async function queryWithPaging< let cursor: string | undefined = undefined; for (let j = 0; j < 1000 /* just for sanity */; ++j) { - const response: Response = await client(query, { - ...variables, - cursor, - }); + let response: Response; + + try { + response = await client(query, { + ...variables, + cursor, + }); + } catch (e) { + response = e.data; + } const conn = connection(response); if (!conn) { From c1238cc022fdfe9467ef23833770629cd0c65846 Mon Sep 17 00:00:00 2001 From: Simon Knittel Date: Sat, 1 May 2021 11:07:17 +0200 Subject: [PATCH 2/3] Use different query for GitHub Discovery depending on the repository owner Signed-off-by: Simon Knittel --- .../src/ingestion/processors/github/github.ts | 97 +++++++++++++------ 1 file changed, 66 insertions(+), 31 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/github/github.ts b/plugins/catalog-backend/src/ingestion/processors/github/github.ts index 0b7f3d831d..f5230ce044 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/github.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/github.ts @@ -66,6 +66,14 @@ export type Connection = { nodes: T[]; }; +export type RepositoryOwnerType = { + __typename: 'Organization' | 'User'; +}; + +export type RepositoryOwnerResponse = { + repositoryOwner: RepositoryOwnerType | null; +}; + /** * Gets all the users out of a GitHub organization. * @@ -229,38 +237,49 @@ export async function getOrganizationRepositories( client: typeof graphql, org: string, ): Promise<{ repositories: Repository[] }> { - const query = ` - query repositories($org: String!, $cursor: String) { - organization(login: $org) { - name - repositories(first: 100, after: $cursor) { - nodes { - name - url - isArchived - } - pageInfo { - hasNextPage - endCursor - } - } - } - user(login: $org) { - name - repositories(first: 100, after: $cursor) { - nodes { - name - url - isArchived - } - pageInfo { - hasNextPage - endCursor - } - } - } + let query = ``; + + switch (await getLoginType(client, org)) { + case 'Organization': + query = ` + query repositories($org: String!, $cursor: String) { + organization(login: $org) { + name + repositories(first: 100, after: $cursor) { + nodes { + name + url + isArchived + } + pageInfo { + hasNextPage + endCursor + } + } + } + }`; + break; + + default: + query = ` + query repositories($org: String!, $cursor: String) { + user(login: $org) { + name + repositories(first: 100, after: $cursor) { + nodes { + name + url + isArchived + } + pageInfo { + hasNextPage + endCursor + } + } + } + }`; + break; } - `; const repositories = await queryWithPaging( client, @@ -372,3 +391,19 @@ export async function queryWithPaging< return result; } + +export async function getLoginType(client: typeof graphql, login: string) { + const query = ` + query repositoryOwner($login: String!) { + repositoryOwner(login: $login) { + __typename + } + }`; + + const response: RepositoryOwnerResponse = await client(query, { login }); + if (response.repositoryOwner === null) { + throw new Error(`Unknown repository owner for "${login}"`); + } + + return response.repositoryOwner.__typename; +} From 227439a7237aa084c82fb2b7a6cfd1aa528675fe Mon Sep 17 00:00:00 2001 From: Simon Knittel Date: Sat, 1 May 2021 13:40:11 +0200 Subject: [PATCH 3/3] Switch back to a single query again; Update tests and add changeset Signed-off-by: Simon Knittel --- .changeset/stupid-feet-reply.md | 5 + .../processors/github/github.test.ts | 2 +- .../src/ingestion/processors/github/github.ts | 98 ++++--------------- 3 files changed, 27 insertions(+), 78 deletions(-) create mode 100644 .changeset/stupid-feet-reply.md diff --git a/.changeset/stupid-feet-reply.md b/.changeset/stupid-feet-reply.md new file mode 100644 index 0000000000..d7213b1bdc --- /dev/null +++ b/.changeset/stupid-feet-reply.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Add support for non-organization accounts in GitHub Discovery diff --git a/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts b/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts index 15280b96b8..88bddc93fb 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts @@ -156,7 +156,7 @@ describe('github', () => { describe('getOrganizationRepositories', () => { it('read repositories', async () => { const input: QueryResponse = { - organization: { + repositoryOwner: { repositories: { nodes: [ { diff --git a/plugins/catalog-backend/src/ingestion/processors/github/github.ts b/plugins/catalog-backend/src/ingestion/processors/github/github.ts index f5230ce044..f6cfcb8901 100644 --- a/plugins/catalog-backend/src/ingestion/processors/github/github.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/github.ts @@ -20,8 +20,8 @@ import { graphql } from '@octokit/graphql'; // Graphql types export type QueryResponse = { - organization: Organization; - user: User; + organization?: Organization; + repositoryOwner?: Organization | User; }; export type Organization = { @@ -66,14 +66,6 @@ export type Connection = { nodes: T[]; }; -export type RepositoryOwnerType = { - __typename: 'Organization' | 'User'; -}; - -export type RepositoryOwnerResponse = { - repositoryOwner: RepositoryOwnerType | null; -}; - /** * Gets all the users out of a GitHub organization. * @@ -237,54 +229,28 @@ export async function getOrganizationRepositories( client: typeof graphql, org: string, ): Promise<{ repositories: Repository[] }> { - let query = ``; - - switch (await getLoginType(client, org)) { - case 'Organization': - query = ` - query repositories($org: String!, $cursor: String) { - organization(login: $org) { + const query = ` + query repositories($org: String!, $cursor: String) { + repositoryOwner(login: $org) { + login + repositories(first: 100, after: $cursor) { + nodes { name - repositories(first: 100, after: $cursor) { - nodes { - name - url - isArchived - } - pageInfo { - hasNextPage - endCursor - } - } + url + isArchived } - }`; - break; - - default: - query = ` - query repositories($org: String!, $cursor: String) { - user(login: $org) { - name - repositories(first: 100, after: $cursor) { - nodes { - name - url - isArchived - } - pageInfo { - hasNextPage - endCursor - } - } + pageInfo { + hasNextPage + endCursor } - }`; - break; - } + } + } + }`; const repositories = await queryWithPaging( client, query, - r => r.organization?.repositories || r.user?.repositories, + r => r.repositoryOwner?.repositories, x => x, { org }, ); @@ -362,16 +328,10 @@ export async function queryWithPaging< let cursor: string | undefined = undefined; for (let j = 0; j < 1000 /* just for sanity */; ++j) { - let response: Response; - - try { - response = await client(query, { - ...variables, - cursor, - }); - } catch (e) { - response = e.data; - } + const response: Response = await client(query, { + ...variables, + cursor, + }); const conn = connection(response); if (!conn) { @@ -391,19 +351,3 @@ export async function queryWithPaging< return result; } - -export async function getLoginType(client: typeof graphql, login: string) { - const query = ` - query repositoryOwner($login: String!) { - repositoryOwner(login: $login) { - __typename - } - }`; - - const response: RepositoryOwnerResponse = await client(query, { login }); - if (response.repositoryOwner === null) { - throw new Error(`Unknown repository owner for "${login}"`); - } - - return response.repositoryOwner.__typename; -}