From 76306d3e4b2cae48606d1172b9490cb612f17c90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mattias=20Frinnstr=C3=B6m?= Date: Wed, 18 Nov 2020 07:38:18 +0100 Subject: [PATCH 1/5] Extract types --- .../src/scaffolder/stages/publish/types.ts | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts index 6e9c45ba43..6dfc29dbde 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts @@ -29,11 +29,18 @@ export type PublisherBase = { * catalog, plus the values from the form and the directory that has * been templated */ - publish(opts: { - entity: TemplateEntityV1alpha1; - values: RequiredTemplateValues & Record; - directory: string; - }): Promise<{ remoteUrl: string }>; + publish(opts: PublisherOptions): Promise; +}; + +export type PublisherOptions = { + entity: TemplateEntityV1alpha1; + values: RequiredTemplateValues & Record; + directory: string; +}; + +export type PublisherResult = { + remoteUrl: string; + catalogInfoUrl?: string; }; export type PublisherBuilder = { From e0a910bc6dcaeef3a0790a7324893562dbe956f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mattias=20Frinnstr=C3=B6m?= Date: Thu, 19 Nov 2020 12:54:31 +0100 Subject: [PATCH 2/5] Start using the extracted types --- .../src/scaffolder/stages/publish/azure.ts | 7 ++----- .../src/scaffolder/stages/publish/github.ts | 7 ++----- .../src/scaffolder/stages/publish/gitlab.ts | 7 ++----- .../src/scaffolder/stages/publish/types.ts | 1 - plugins/scaffolder-backend/src/service/router.ts | 1 - 5 files changed, 6 insertions(+), 17 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts index 33f7f89199..8f3ed33423 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { PublisherBase } from './types'; +import { PublisherBase, PublisherOptions, PublisherResult } from './types'; import { GitApi } from 'azure-devops-node-api/GitApi'; import { GitRepositoryCreateOptions } from 'azure-devops-node-api/interfaces/GitInterfaces'; import { pushToRemoteUserPass } from './helpers'; @@ -33,10 +33,7 @@ export class AzurePublisher implements PublisherBase { async publish({ values, directory, - }: { - values: RequiredTemplateValues & Record; - directory: string; - }): Promise<{ remoteUrl: string }> { + }: PublisherOptions): Promise { const remoteUrl = await this.createRemote(values); await pushToRemoteUserPass(directory, remoteUrl, 'notempty', this.token); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts index d5542e8800..2e835ba3d4 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { PublisherBase } from './types'; +import { PublisherBase, PublisherOptions, PublisherResult } from './types'; import { Octokit } from '@octokit/rest'; import { pushToRemoteUserPass } from './helpers'; import { JsonValue } from '@backstage/config'; @@ -46,10 +46,7 @@ export class GithubPublisher implements PublisherBase { async publish({ values, directory, - }: { - values: RequiredTemplateValues & Record; - directory: string; - }): Promise<{ remoteUrl: string }> { + }: PublisherOptions): Promise { const remoteUrl = await this.createRemote(values); await pushToRemoteUserPass( directory, diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.ts index e748dc53dc..4fae8e8a5c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/gitlab.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { PublisherBase } from './types'; +import { PublisherBase, PublisherOptions, PublisherResult } from './types'; import { Gitlab } from '@gitbeaker/core'; import { pushToRemoteUserPass } from './helpers'; import { JsonValue } from '@backstage/config'; @@ -32,10 +32,7 @@ export class GitlabPublisher implements PublisherBase { async publish({ values, directory, - }: { - values: RequiredTemplateValues & Record; - directory: string; - }): Promise<{ remoteUrl: string }> { + }: PublisherOptions): Promise { const remoteUrl = await this.createRemote(values); await pushToRemoteUserPass(directory, remoteUrl, 'oauth2', this.token); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts index 6dfc29dbde..f3bc59e19d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts @@ -33,7 +33,6 @@ export type PublisherBase = { }; export type PublisherOptions = { - entity: TemplateEntityV1alpha1; values: RequiredTemplateValues & Record; directory: string; }; diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 2dc6a794fb..fd25f2d845 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -157,7 +157,6 @@ export async function createRouter( const publisher = publishers.get(ctx.entity); ctx.logger.info('Will now store the template'); const { remoteUrl } = await publisher.publish({ - entity: ctx.entity, values: ctx.values, directory: ctx.resultDir, }); From 67b6320e839e504f7c997e19dd3b9171ecc55188 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mattias=20Frinnstr=C3=B6m?= Date: Thu, 19 Nov 2020 13:51:40 +0100 Subject: [PATCH 3/5] Generate catatalog-info.yaml URL in the publishers --- .../scaffolder/stages/publish/azure.test.ts | 10 ++-- .../src/scaffolder/stages/publish/azure.ts | 3 +- .../scaffolder/stages/publish/github.test.ts | 50 +++++++++++++------ .../src/scaffolder/stages/publish/github.ts | 6 ++- .../scaffolder-backend/src/service/router.ts | 4 +- 5 files changed, 51 insertions(+), 22 deletions(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts index 7ab6f81ae5..9ea7d2de5f 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.test.ts @@ -41,7 +41,7 @@ describe('Azure Publisher', () => { describe('publish: createRemoteInAzure', () => { it('should use azure-devops-node-api to create a repo in the given project', async () => { mockGitApi.createRepository.mockResolvedValue({ - remoteUrl: 'mockclone', + remoteUrl: 'https://dev.azure.com/organization/project/_git/repo', } as { remoteUrl: string }); const result = await publisher.publish({ @@ -52,7 +52,11 @@ describe('Azure Publisher', () => { directory: '/tmp/test', }); - expect(result).toEqual({ remoteUrl: 'mockclone' }); + expect(result).toEqual({ + remoteUrl: 'https://dev.azure.com/organization/project/_git/repo', + catalogInfoUrl: + 'https://dev.azure.com/organization/project/_git/repo?path=%2Fcatalog-info.yaml', + }); expect(mockGitApi.createRepository).toHaveBeenCalledWith( { name: 'repo', @@ -61,7 +65,7 @@ describe('Azure Publisher', () => { ); expect(pushToRemoteUserPass).toHaveBeenCalledWith( '/tmp/test', - 'mockclone', + 'https://dev.azure.com/organization/project/_git/repo', 'notempty', 'fake-token', ); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts index 8f3ed33423..1e962bf223 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/azure.ts @@ -36,8 +36,9 @@ export class AzurePublisher implements PublisherBase { }: PublisherOptions): Promise { const remoteUrl = await this.createRemote(values); await pushToRemoteUserPass(directory, remoteUrl, 'notempty', this.token); + const catalogInfoUrl = `${remoteUrl}?path=%2Fcatalog-info.yaml`; - return { remoteUrl }; + return { remoteUrl, catalogInfoUrl }; } private async createRemote( diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts index c85b4acba3..cda64faf25 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts @@ -53,7 +53,7 @@ describe('GitHub Publisher', () => { it('should use octokit to create a repo in an organisation if the organisation property is set', async () => { mockGithubClient.repos.createInOrg.mockResolvedValue({ data: { - clone_url: 'mockclone', + clone_url: 'https://github.com/backstage/backstage.git', }, } as OctokitResponse); mockGithubClient.users.getByUsername.mockResolvedValue({ @@ -71,7 +71,11 @@ describe('GitHub Publisher', () => { directory: '/tmp/test', }); - expect(result).toEqual({ remoteUrl: 'mockclone' }); + expect(result).toEqual({ + remoteUrl: 'https://github.com/backstage/backstage.git', + catalogInfoUrl: + 'https://github.com/backstage/backstage/blob/master/catalog-info.yaml', + }); expect(mockGithubClient.repos.createInOrg).toHaveBeenCalledWith({ org: 'blam', name: 'test', @@ -89,7 +93,7 @@ describe('GitHub Publisher', () => { }); expect(pushToRemoteUserPass).toHaveBeenCalledWith( '/tmp/test', - 'mockclone', + 'https://github.com/backstage/backstage.git', 'abc', 'x-oauth-basic', ); @@ -98,7 +102,7 @@ describe('GitHub Publisher', () => { it('should use octokit to create a repo in the authed user if the organisation property is not set', async () => { mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({ data: { - clone_url: 'mockclone', + clone_url: 'https://github.com/backstage/backstage.git', }, } as OctokitResponse); mockGithubClient.users.getByUsername.mockResolvedValue({ @@ -116,7 +120,11 @@ describe('GitHub Publisher', () => { directory: '/tmp/test', }); - expect(result).toEqual({ remoteUrl: 'mockclone' }); + expect(result).toEqual({ + remoteUrl: 'https://github.com/backstage/backstage.git', + catalogInfoUrl: + 'https://github.com/backstage/backstage/blob/master/catalog-info.yaml', + }); expect( mockGithubClient.repos.createForAuthenticatedUser, ).toHaveBeenCalledWith({ @@ -126,7 +134,7 @@ describe('GitHub Publisher', () => { expect(mockGithubClient.repos.addCollaborator).not.toHaveBeenCalled(); expect(pushToRemoteUserPass).toHaveBeenCalledWith( '/tmp/test', - 'mockclone', + 'https://github.com/backstage/backstage.git', 'abc', 'x-oauth-basic', ); @@ -136,7 +144,7 @@ describe('GitHub Publisher', () => { it('should invite other user in the authed user', async () => { mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({ data: { - clone_url: 'mockclone', + clone_url: 'https://github.com/backstage/backstage.git', }, } as OctokitResponse); mockGithubClient.users.getByUsername.mockResolvedValue({ @@ -155,7 +163,11 @@ describe('GitHub Publisher', () => { directory: '/tmp/test', }); - expect(result).toEqual({ remoteUrl: 'mockclone' }); + expect(result).toEqual({ + remoteUrl: 'https://github.com/backstage/backstage.git', + catalogInfoUrl: + 'https://github.com/backstage/backstage/blob/master/catalog-info.yaml', + }); expect( mockGithubClient.repos.createForAuthenticatedUser, ).toHaveBeenCalledWith({ @@ -171,7 +183,7 @@ describe('GitHub Publisher', () => { }); expect(pushToRemoteUserPass).toHaveBeenCalledWith( '/tmp/test', - 'mockclone', + 'https://github.com/backstage/backstage.git', 'abc', 'x-oauth-basic', ); @@ -188,7 +200,7 @@ describe('GitHub Publisher', () => { it('creates a private repository in the organization with visibility set to internal', async () => { mockGithubClient.repos.createInOrg.mockResolvedValue({ data: { - clone_url: 'mockclone', + clone_url: 'https://github.com/backstage/backstage.git', }, } as OctokitResponse); mockGithubClient.users.getByUsername.mockResolvedValue({ @@ -206,7 +218,11 @@ describe('GitHub Publisher', () => { directory: '/tmp/test', }); - expect(result).toEqual({ remoteUrl: 'mockclone' }); + expect(result).toEqual({ + remoteUrl: 'https://github.com/backstage/backstage.git', + catalogInfoUrl: + 'https://github.com/backstage/backstage/blob/master/catalog-info.yaml', + }); expect(mockGithubClient.repos.createInOrg).toHaveBeenCalledWith({ org: 'blam', name: 'test', @@ -215,7 +231,7 @@ describe('GitHub Publisher', () => { }); expect(pushToRemoteUserPass).toHaveBeenCalledWith( '/tmp/test', - 'mockclone', + 'https://github.com/backstage/backstage.git', 'abc', 'x-oauth-basic', ); @@ -232,7 +248,7 @@ describe('GitHub Publisher', () => { it('creates a private repository', async () => { mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({ data: { - clone_url: 'mockclone', + clone_url: 'https://github.com/backstage/backstage.git', }, } as OctokitResponse); mockGithubClient.users.getByUsername.mockResolvedValue({ @@ -249,7 +265,11 @@ describe('GitHub Publisher', () => { directory: '/tmp/test', }); - expect(result).toEqual({ remoteUrl: 'mockclone' }); + expect(result).toEqual({ + remoteUrl: 'https://github.com/backstage/backstage.git', + catalogInfoUrl: + 'https://github.com/backstage/backstage/blob/master/catalog-info.yaml', + }); expect( mockGithubClient.repos.createForAuthenticatedUser, ).toHaveBeenCalledWith({ @@ -258,7 +278,7 @@ describe('GitHub Publisher', () => { }); expect(pushToRemoteUserPass).toHaveBeenCalledWith( '/tmp/test', - 'mockclone', + 'https://github.com/backstage/backstage.git', 'abc', 'x-oauth-basic', ); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts index 2e835ba3d4..64976b8e41 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts @@ -54,8 +54,12 @@ export class GithubPublisher implements PublisherBase { this.token, 'x-oauth-basic', ); + const catalogInfoUrl = remoteUrl.replace( + /\.git$/, + '/blob/master/catalog-info.yaml', + ); - return { remoteUrl }; + return { remoteUrl, catalogInfoUrl }; } private async createRemote( diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index fd25f2d845..ac3b1c80ec 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -156,11 +156,11 @@ export async function createRouter( handler: async (ctx: StageContext<{ resultDir: string }>) => { const publisher = publishers.get(ctx.entity); ctx.logger.info('Will now store the template'); - const { remoteUrl } = await publisher.publish({ + const result = await publisher.publish({ values: ctx.values, directory: ctx.resultDir, }); - return { remoteUrl }; + return result; }, }, ], From a13a090de99a08cb5e2f4a5f1473bcc3bb237f94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mattias=20Frinnstr=C3=B6m?= Date: Thu, 19 Nov 2020 13:52:21 +0100 Subject: [PATCH 4/5] Use the new catalogInfoUrl in TemplatePage --- .../src/components/TemplatePage/TemplatePage.tsx | 14 +++----------- plugins/scaffolder/src/types.ts | 1 + 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx index 4836cdab56..ffa2bfcec7 100644 --- a/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx +++ b/plugins/scaffolder/src/components/TemplatePage/TemplatePage.tsx @@ -106,18 +106,10 @@ export const TemplatePage = () => { ); const handleCreateComplete = async (job: Job) => { - const target = job.metadata.remoteUrl?.replace( - /\.git$/, - // TODO(Rugvip): This is not the location we want. As part of scaffolder v2 we - // want this to be more flexible, but before that we might want - // to update all templates to use catalog-info.yaml instead. - '/blob/master/component-info.yaml', - ); - - if (!target) { + if (!job.metadata.catalogInfoUrl) { errorApi.post( new Error( - `Failed to find component-info.yaml file in ${job.metadata.remoteUrl}.`, + `Failed to find catalog-info.yaml file in ${job.metadata.remoteUrl}.`, ), ); return; @@ -125,7 +117,7 @@ export const TemplatePage = () => { const { entities: [createdEntity], - } = await catalogApi.addLocation({ target }); + } = await catalogApi.addLocation({ target: job.metadata.catalogInfoUrl }); setEntity((createdEntity as any) as TemplateEntityV1alpha1); }; diff --git a/plugins/scaffolder/src/types.ts b/plugins/scaffolder/src/types.ts index 7106444904..e07581ad28 100644 --- a/plugins/scaffolder/src/types.ts +++ b/plugins/scaffolder/src/types.ts @@ -19,6 +19,7 @@ export type Job = { entity: any; values: any; remoteUrl?: string; + catalogInfoUrl?: string; }; status: 'PENDING' | 'STARTED' | 'COMPLETED' | 'FAILED'; stages: Stage[]; From ef2831dde3567a2c02f78e7bc8d31c62bc212ad5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mattias=20Frinnstr=C3=B6m?= Date: Thu, 19 Nov 2020 15:00:24 +0100 Subject: [PATCH 5/5] Add changeset --- .changeset/dull-pans-sip.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/dull-pans-sip.md diff --git a/.changeset/dull-pans-sip.md b/.changeset/dull-pans-sip.md new file mode 100644 index 0000000000..13616c1f8d --- /dev/null +++ b/.changeset/dull-pans-sip.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-scaffolder': patch +'@backstage/plugin-scaffolder-backend': patch +--- + +Move constructing the catalog-info.yaml URL for scaffolded components to the publishers