From 06eee145946607eb90c19e54b5ad31e938ec2542 Mon Sep 17 00:00:00 2001 From: Jente Sondervorst Date: Sat, 11 Jan 2025 00:10:43 +0100 Subject: [PATCH] Support empty repo and condition creation Fixes #28427 Signed-off-by: Jente Sondervorst --- .changeset/eleven-goats-yell.md | 5 + .../report.api.md | 3 +- .../src/actions/gitlab.examples.test.ts | 4 + .../src/actions/gitlab.examples.ts | 19 ++ .../src/actions/gitlab.test.ts | 89 ++++++ .../src/actions/gitlab.ts | 287 ++++++++++-------- 6 files changed, 287 insertions(+), 120 deletions(-) create mode 100644 .changeset/eleven-goats-yell.md diff --git a/.changeset/eleven-goats-yell.md b/.changeset/eleven-goats-yell.md new file mode 100644 index 0000000000..212b35497f --- /dev/null +++ b/.changeset/eleven-goats-yell.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend-module-gitlab': minor +--- + +Support empty repository creation in gitlab without workspace pushing and conditionally skip if the repository already exists. diff --git a/plugins/scaffolder-backend-module-gitlab/report.api.md b/plugins/scaffolder-backend-module-gitlab/report.api.md index 896c0a5899..22e5698836 100644 --- a/plugins/scaffolder-backend-module-gitlab/report.api.md +++ b/plugins/scaffolder-backend-module-gitlab/report.api.md @@ -132,7 +132,8 @@ export function createPublishGitlabAction(options: { repoUrl: string; defaultBranch?: string | undefined; repoVisibility?: 'internal' | 'private' | 'public' | undefined; - sourcePath?: string | undefined; + sourcePath?: string | boolean | undefined; + skipExisting?: boolean | undefined; token?: string | undefined; gitCommitMessage?: string | undefined; gitAuthorName?: string | undefined; diff --git a/plugins/scaffolder-backend-module-gitlab/src/actions/gitlab.examples.test.ts b/plugins/scaffolder-backend-module-gitlab/src/actions/gitlab.examples.test.ts index c7a7b01aad..d626c84c77 100644 --- a/plugins/scaffolder-backend-module-gitlab/src/actions/gitlab.examples.test.ts +++ b/plugins/scaffolder-backend-module-gitlab/src/actions/gitlab.examples.test.ts @@ -38,6 +38,9 @@ const mockGitlabClient = { Namespaces: { show: jest.fn(), }, + Groups: { + allProjects: jest.fn(), + }, Projects: { create: jest.fn(), }, @@ -88,6 +91,7 @@ describe('publish:gitlab', () => { it('should call initRepoAndPush with the correct values', async () => { mockGitlabClient.Users.showCurrentUser.mockResolvedValue({ id: 12345 }); mockGitlabClient.Namespaces.show.mockResolvedValue({ id: 1234 }); + mockGitlabClient.Groups.allProjects.mockResolvedValue([]); mockGitlabClient.Projects.create.mockResolvedValue({ http_url_to_repo: 'http://mockurl.git', }); diff --git a/plugins/scaffolder-backend-module-gitlab/src/actions/gitlab.examples.ts b/plugins/scaffolder-backend-module-gitlab/src/actions/gitlab.examples.ts index 3015850399..adcc3fdb0b 100644 --- a/plugins/scaffolder-backend-module-gitlab/src/actions/gitlab.examples.ts +++ b/plugins/scaffolder-backend-module-gitlab/src/actions/gitlab.examples.ts @@ -202,4 +202,23 @@ export const examples: TemplateExample[] = [ ], }), }, + { + description: + 'Initializes a GitLab repository with the default readme and no files from workspace only if this repository does not exist yet.', + example: yaml.stringify({ + steps: [ + { + id: 'publish', + action: 'publish:gitlab', + name: 'Publish to GitLab', + input: { + repoUrl: 'gitlab.com?repo=project_name&owner=group_name', + skipExisting: true, + initialize_with_readme: true, + sourcePath: false, + }, + }, + ], + }), + }, ]; diff --git a/plugins/scaffolder-backend-module-gitlab/src/actions/gitlab.test.ts b/plugins/scaffolder-backend-module-gitlab/src/actions/gitlab.test.ts index 6e131e91fc..b114788afa 100644 --- a/plugins/scaffolder-backend-module-gitlab/src/actions/gitlab.test.ts +++ b/plugins/scaffolder-backend-module-gitlab/src/actions/gitlab.test.ts @@ -36,6 +36,9 @@ const mockGitlabClient = { Namespaces: { show: jest.fn(), }, + Groups: { + allProjects: jest.fn(), + }, Projects: { create: jest.fn(), }, @@ -185,6 +188,7 @@ describe('publish:gitlab', () => { it('should work when there is a token provided through ctx.input', async () => { mockGitlabClient.Users.showCurrentUser.mockResolvedValue({ id: 12345 }); mockGitlabClient.Namespaces.show.mockResolvedValue({ id: 1234 }); + mockGitlabClient.Groups.allProjects.mockResolvedValue([]); mockGitlabClient.Projects.create.mockResolvedValue({ http_url_to_repo: 'http://mockurl.git', }); @@ -210,6 +214,7 @@ describe('publish:gitlab', () => { it('should call the correct Gitlab APIs when the owner is an organization', async () => { mockGitlabClient.Users.showCurrentUser.mockResolvedValue({ id: 12345 }); mockGitlabClient.Namespaces.show.mockResolvedValue({ id: 1234 }); + mockGitlabClient.Groups.allProjects.mockResolvedValue([]); mockGitlabClient.Projects.create.mockResolvedValue({ http_url_to_repo: 'http://mockurl.git', }); @@ -230,6 +235,7 @@ describe('publish:gitlab', () => { it('should call the correct Gitlab APIs when the owner is not an organization', async () => { mockGitlabClient.Namespaces.show.mockResolvedValue({ id: null }); mockGitlabClient.Users.showCurrentUser.mockResolvedValue({ id: 12345 }); + mockGitlabClient.Groups.allProjects.mockResolvedValue([]); mockGitlabClient.Projects.create.mockResolvedValue({ http_url_to_repo: 'http://mockurl.git', }); @@ -247,9 +253,60 @@ describe('publish:gitlab', () => { expect(mockGitlabClient.ProtectedBranches.protect).not.toHaveBeenCalled(); }); + it('should not call the creation Gitlab APIs when the repository already exists', async () => { + mockGitlabClient.Users.showCurrentUser.mockResolvedValue({ id: 12345 }); + mockGitlabClient.Namespaces.show.mockResolvedValue({ id: 1234 }); + mockGitlabClient.Groups.allProjects.mockResolvedValue([ + { + path: 'repo', + http_url_to_repo: 'http://mockurl.git', + }, + { + path: 'repo-name', + http_url_to_repo: 'http://mockurl.git', + }, + ]); + + await action.handler({ + ...mockContext, + input: { ...mockContext.input, skipExisting: true }, + }); + + expect(mockGitlabClient.Namespaces.show).toHaveBeenCalledWith('owner'); + expect(mockGitlabClient.Projects.create).not.toHaveBeenCalled(); + expect(mockGitlabClient.Branches.create).not.toHaveBeenCalled(); + expect(mockGitlabClient.ProtectedBranches.protect).not.toHaveBeenCalled(); + }); + + it('should call the creation Gitlab APIs when the repository does not yet exists', async () => { + mockGitlabClient.Users.showCurrentUser.mockResolvedValue({ id: 12345 }); + mockGitlabClient.Namespaces.show.mockResolvedValue({ id: 1234 }); + mockGitlabClient.Groups.allProjects.mockResolvedValue([ + { + path: 'repo-name', + }, + ]); + mockGitlabClient.Projects.create.mockResolvedValue({ + http_url_to_repo: 'http://mockurl.git', + }); + + await action.handler(mockContext); + + expect(mockGitlabClient.Namespaces.show).toHaveBeenCalledWith('owner'); + expect(mockGitlabClient.Projects.create).toHaveBeenCalledWith({ + namespaceId: 1234, + name: 'repo', + visibility: 'private', + ci_config_path: '.gitlab-ci.yml', + }); + expect(mockGitlabClient.Branches.create).not.toHaveBeenCalled(); + expect(mockGitlabClient.ProtectedBranches.protect).not.toHaveBeenCalled(); + }); + it('should call the correct Gitlab APIs when using project settings with override of visibility and topics', async () => { mockGitlabClient.Users.showCurrentUser.mockResolvedValue({ id: 12345 }); mockGitlabClient.Namespaces.show.mockResolvedValue({ id: 1234 }); + mockGitlabClient.Groups.allProjects.mockResolvedValue([]); mockGitlabClient.Projects.create.mockResolvedValue({ http_url_to_repo: 'http://mockurl.git', }); @@ -271,6 +328,7 @@ describe('publish:gitlab', () => { it('should call the correct Gitlab APIs for branches and protectd branches when branch settings provided', async () => { mockGitlabClient.Users.showCurrentUser.mockResolvedValue({ id: 12345 }); mockGitlabClient.Namespaces.show.mockResolvedValue({ id: 1234 }); + mockGitlabClient.Groups.allProjects.mockResolvedValue([]); mockGitlabClient.Projects.create.mockResolvedValue({ id: 123456, http_url_to_repo: 'http://mockurl.git', @@ -311,6 +369,7 @@ describe('publish:gitlab', () => { it('should call the correct Gitlab APIs for variables when their configuration is provided', async () => { mockGitlabClient.Users.showCurrentUser.mockResolvedValue({ id: 12345 }); mockGitlabClient.Namespaces.show.mockResolvedValue({ id: 1234 }); + mockGitlabClient.Groups.allProjects.mockResolvedValue([]); mockGitlabClient.Projects.create.mockResolvedValue({ id: 123456, http_url_to_repo: 'http://mockurl.git', @@ -343,6 +402,7 @@ describe('publish:gitlab', () => { it('should call initRepoAndPush with the correct values', async () => { mockGitlabClient.Users.showCurrentUser.mockResolvedValue({ id: 12345 }); mockGitlabClient.Namespaces.show.mockResolvedValue({ id: 1234 }); + mockGitlabClient.Groups.allProjects.mockResolvedValue([]); mockGitlabClient.Projects.create.mockResolvedValue({ http_url_to_repo: 'http://mockurl.git', }); @@ -360,9 +420,34 @@ describe('publish:gitlab', () => { }); }); + it('should not call initRepoAndPush when sourcePath is false', async () => { + mockGitlabClient.Users.showCurrentUser.mockResolvedValue({ id: 12345 }); + mockGitlabClient.Namespaces.show.mockResolvedValue({ id: 1234 }); + mockGitlabClient.Groups.allProjects.mockResolvedValue([]); + mockGitlabClient.Projects.create.mockResolvedValue({ + http_url_to_repo: 'http://mockurl.git', + }); + + await action.handler({ + ...mockContext, + input: { ...mockContext.input, sourcePath: false }, + }); + + expect(initRepoAndPush).not.toHaveBeenCalledWith({ + dir: mockContext.workspacePath, + defaultBranch: 'master', + remoteUrl: 'http://mockurl.git', + auth: { username: 'oauth2', password: 'tokenlols' }, + logger: mockContext.logger, + commitMessage: 'initial commit', + gitAuthorInfo: {}, + }); + }); + it('should call initRepoAndPush with the correct default branch', async () => { mockGitlabClient.Users.showCurrentUser.mockResolvedValue({ id: 12345 }); mockGitlabClient.Namespaces.show.mockResolvedValue({ id: 1234 }); + mockGitlabClient.Groups.allProjects.mockResolvedValue([]); mockGitlabClient.Projects.create.mockResolvedValue({ http_url_to_repo: 'http://mockurl.git', }); @@ -418,6 +503,7 @@ describe('publish:gitlab', () => { mockGitlabClient.Users.showCurrentUser.mockResolvedValue({ id: 12345 }); mockGitlabClient.Namespaces.show.mockResolvedValue({ id: 1234 }); + mockGitlabClient.Groups.allProjects.mockResolvedValue([]); mockGitlabClient.Projects.create.mockResolvedValue({ http_url_to_repo: 'http://mockurl.git', }); @@ -464,6 +550,7 @@ describe('publish:gitlab', () => { mockGitlabClient.Users.showCurrentUser.mockResolvedValue({ id: 12345 }); mockGitlabClient.Namespaces.show.mockResolvedValue({ id: 1234 }); + mockGitlabClient.Groups.allProjects.mockResolvedValue([]); mockGitlabClient.Projects.create.mockResolvedValue({ http_url_to_repo: 'http://mockurl.git', }); @@ -484,6 +571,7 @@ describe('publish:gitlab', () => { it('should call output with the remoteUrl and repoContentsUrl and projectId', async () => { mockGitlabClient.Users.showCurrentUser.mockResolvedValue({ id: 12345 }); mockGitlabClient.Namespaces.show.mockResolvedValue({ id: 1234 }); + mockGitlabClient.Groups.allProjects.mockResolvedValue([]); mockGitlabClient.Projects.create.mockResolvedValue({ http_url_to_repo: 'http://mockurl.git', id: 1234, @@ -524,6 +612,7 @@ describe('publish:gitlab', () => { mockGitlabClient.Namespaces.show.mockResolvedValue({ id: 1234 }); mockGitlabClient.Users.showCurrentUser.mockResolvedValue({ id: 12345 }); + mockGitlabClient.Groups.allProjects.mockResolvedValue([]); mockGitlabClient.Projects.create.mockResolvedValue({ id: 123456, http_url_to_repo: 'http://mockurl.git', diff --git a/plugins/scaffolder-backend-module-gitlab/src/actions/gitlab.ts b/plugins/scaffolder-backend-module-gitlab/src/actions/gitlab.ts index c682749f83..76c6faeda3 100644 --- a/plugins/scaffolder-backend-module-gitlab/src/actions/gitlab.ts +++ b/plugins/scaffolder-backend-module-gitlab/src/actions/gitlab.ts @@ -43,7 +43,8 @@ export function createPublishGitlabAction(options: { defaultBranch?: string; /** @deprecated in favour of settings.visibility field */ repoVisibility?: 'private' | 'internal' | 'public'; - sourcePath?: string; + sourcePath?: string | boolean; + skipExisting?: boolean; token?: string; gitCommitMessage?: string; gitAuthorName?: string; @@ -124,8 +125,14 @@ export function createPublishGitlabAction(options: { sourcePath: { title: 'Source Path', description: - 'Path within the workspace that will be used as the repository root. If omitted, the entire workspace will be published as the repository.', - type: 'string', + 'Path within the workspace that will be used as the repository root. If omitted or set to true, the entire workspace will be published as the repository. If set to false, the created repository will be empty.', + type: ['string', 'boolean'], + }, + skipExisting: { + title: 'Source Path', + description: + 'Do not publish the repository if it already exists. The default value is false.', + type: ['boolean'], }, token: { title: 'Authentication Token', @@ -321,6 +328,10 @@ export function createPublishGitlabAction(options: { title: 'The git commit hash of the initial commit', type: 'string', }, + created: { + title: 'Whether the repository was created or not', + type: 'boolean', + }, }, }, }, @@ -337,6 +348,7 @@ export function createPublishGitlabAction(options: { settings = {}, branches = [], projectVariables = [], + skipExisting = false, } = ctx.input; const { owner, repo, host } = parseRepoUrl(repoUrl, integrations); @@ -391,129 +403,166 @@ export function createPublishGitlabAction(options: { targetNamespaceId = userId; } - const { id: projectId, http_url_to_repo } = await client.Projects.create({ - namespaceId: targetNamespaceId, - name: repo, - visibility: repoVisibility, - ...(topics.length ? { topics } : {}), - ...(Object.keys(settings).length ? { ...settings } : {}), + const existingProjects = await client.Groups.allProjects(owner, { + search: repo, }); + const existingProject = existingProjects.find( + searchPathElem => searchPathElem.path === repo, + ); - // When setUserAsOwner is true the input token is expected to come from an unprivileged user GitLab - // OAuth flow. In this case GitLab works in a way that allows the unprivileged user to - // create the repository, but not to push the default protected branch (e.g. master). - // In order to set the user as owner of the newly created repository we need to check that the - // GitLab integration configuration for the matching host contains a token and use - // such token to bootstrap a new privileged client. - if (setUserAsOwner && integrationConfig.config.token) { - const adminClient = new Gitlab({ - host: integrationConfig.config.baseUrl, - token: integrationConfig.config.token, - }); - - await adminClient.ProjectMembers.add(projectId, userId, 50); - } - - const remoteUrl = (http_url_to_repo as string).replace(/\.git$/, ''); - const repoContentsUrl = `${remoteUrl}/-/blob/${defaultBranch}`; - - const gitAuthorInfo = { - name: gitAuthorName - ? gitAuthorName - : config.getOptionalString('scaffolder.defaultAuthor.name'), - email: gitAuthorEmail - ? gitAuthorEmail - : config.getOptionalString('scaffolder.defaultAuthor.email'), - }; - const commitResult = await initRepoAndPush({ - dir: getRepoSourceDirectory(ctx.workspacePath, ctx.input.sourcePath), - remoteUrl: http_url_to_repo as string, - defaultBranch, - auth: { - username: 'oauth2', - password: token, - }, - logger: ctx.logger, - commitMessage: gitCommitMessage - ? gitCommitMessage - : config.getOptionalString('scaffolder.defaultCommitMessage'), - gitAuthorInfo, - }); - - if (branches) { - for (const branch of branches) { - const { - name, - protect = false, - create = false, - ref = 'master', - } = branch; - - if (create) { - try { - await client.Branches.create(projectId, name, ref); - } catch (e) { - throw new InputError( - `Branch creation failed for ${name}. ${printGitlabError(e)}`, - ); - } - ctx.logger.info( - `Branch ${name} created for ${projectId} with ref ${ref}`, - ); - } - - if (protect) { - try { - await client.ProtectedBranches.protect(projectId, name); - } catch (e) { - throw new InputError( - `Branch protection failed for ${name}. ${printGitlabError(e)}`, - ); - } - ctx.logger.info(`Branch ${name} protected for ${projectId}`); - } - } - } - - if (projectVariables) { - for (const variable of projectVariables) { - const variableWithDefaults = Object.assign(variable, { - variable_type: (variable.variable_type ?? - 'env_var') as VariableType, - protected: variable.protected ?? false, - masked: variable.masked ?? false, - raw: variable.raw ?? false, - environment_scope: variable.environment_scope ?? '*', + if (!skipExisting || (skipExisting && !existingProject)) { + ctx.logger.info(`Creating repo ${repo} in namespace ${owner}.`); + const { id: projectId, http_url_to_repo } = + await client.Projects.create({ + namespaceId: targetNamespaceId, + name: repo, + visibility: repoVisibility, + ...(topics.length ? { topics } : {}), + ...(Object.keys(settings).length ? { ...settings } : {}), }); - try { - await client.ProjectVariables.create( - projectId, - variableWithDefaults.key, - variableWithDefaults.value, - { - variableType: variableWithDefaults.variable_type, - protected: variableWithDefaults.protected, - masked: variableWithDefaults.masked, - environmentScope: variableWithDefaults.environment_scope, - description: variableWithDefaults.description, - raw: variableWithDefaults.raw, - }, - ); - } catch (e) { - throw new InputError( - `Environment variable creation failed for ${ - variableWithDefaults.key - }. ${printGitlabError(e)}`, - ); + // When setUserAsOwner is true the input token is expected to come from an unprivileged user GitLab + // OAuth flow. In this case GitLab works in a way that allows the unprivileged user to + // create the repository, but not to push the default protected branch (e.g. master). + // In order to set the user as owner of the newly created repository we need to check that the + // GitLab integration configuration for the matching host contains a token and use + // such token to bootstrap a new privileged client. + if (setUserAsOwner && integrationConfig.config.token) { + const adminClient = new Gitlab({ + host: integrationConfig.config.baseUrl, + token: integrationConfig.config.token, + }); + + await adminClient.ProjectMembers.add(projectId, userId, 50); + } + + const remoteUrl = (http_url_to_repo as string).replace(/\.git$/, ''); + const repoContentsUrl = `${remoteUrl}/-/blob/${defaultBranch}`; + + const gitAuthorInfo = { + name: gitAuthorName + ? gitAuthorName + : config.getOptionalString('scaffolder.defaultAuthor.name'), + email: gitAuthorEmail + ? gitAuthorEmail + : config.getOptionalString('scaffolder.defaultAuthor.email'), + }; + const shouldSkipPublish = + typeof ctx.input.sourcePath === 'boolean' && !ctx.input.sourcePath; + if (!shouldSkipPublish) { + const commitResult = await initRepoAndPush({ + dir: + typeof ctx.input.sourcePath === 'boolean' + ? ctx.workspacePath + : getRepoSourceDirectory( + ctx.workspacePath, + ctx.input.sourcePath, + ), + remoteUrl: http_url_to_repo as string, + defaultBranch, + auth: { + username: 'oauth2', + password: token, + }, + logger: ctx.logger, + commitMessage: gitCommitMessage + ? gitCommitMessage + : config.getOptionalString('scaffolder.defaultCommitMessage'), + gitAuthorInfo, + }); + + if (branches) { + for (const branch of branches) { + const { + name, + protect = false, + create = false, + ref = 'master', + } = branch; + + if (create) { + try { + await client.Branches.create(projectId, name, ref); + } catch (e) { + throw new InputError( + `Branch creation failed for ${name}. ${printGitlabError( + e, + )}`, + ); + } + ctx.logger.info( + `Branch ${name} created for ${projectId} with ref ${ref}`, + ); + } + + if (protect) { + try { + await client.ProtectedBranches.protect(projectId, name); + } catch (e) { + throw new InputError( + `Branch protection failed for ${name}. ${printGitlabError( + e, + )}`, + ); + } + ctx.logger.info(`Branch ${name} protected for ${projectId}`); + } + } + } + ctx.output('commitHash', commitResult?.commitHash); + } + + if (projectVariables) { + for (const variable of projectVariables) { + const variableWithDefaults = Object.assign(variable, { + variable_type: (variable.variable_type ?? + 'env_var') as VariableType, + protected: variable.protected ?? false, + masked: variable.masked ?? false, + raw: variable.raw ?? false, + environment_scope: variable.environment_scope ?? '*', + }); + + try { + await client.ProjectVariables.create( + projectId, + variableWithDefaults.key, + variableWithDefaults.value, + { + variableType: variableWithDefaults.variable_type, + protected: variableWithDefaults.protected, + masked: variableWithDefaults.masked, + environmentScope: variableWithDefaults.environment_scope, + description: variableWithDefaults.description, + raw: variableWithDefaults.raw, + }, + ); + } catch (e) { + throw new InputError( + `Environment variable creation failed for ${ + variableWithDefaults.key + }. ${printGitlabError(e)}`, + ); + } } } + ctx.output('remoteUrl', remoteUrl); + ctx.output('repoContentsUrl', repoContentsUrl); + ctx.output('projectId', projectId); + ctx.output('created', true); + } else if (existingProject) { + ctx.logger.info(`Repo ${repo} already exists in namespace ${owner}.`); + const { + id: projectId, + http_url_to_repo, + default_branch, + } = existingProject; + const remoteUrl = (http_url_to_repo as string).replace(/\.git$/, ''); + ctx.output('remoteUrl', remoteUrl); + ctx.output('repoContentsUrl', `${remoteUrl}/-/blob/${default_branch}`); + ctx.output('projectId', projectId); + ctx.output('created', false); } - - ctx.output('commitHash', commitResult?.commitHash); - ctx.output('remoteUrl', remoteUrl); - ctx.output('repoContentsUrl', repoContentsUrl); - ctx.output('projectId', projectId); }, }); }