diff --git a/.changeset/proud-news-perform.md b/.changeset/proud-news-perform.md new file mode 100644 index 0000000000..9d5cbae8ab --- /dev/null +++ b/.changeset/proud-news-perform.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-cicd-statistics-module-gitlab': minor +--- + +Created a module to extract the CI/CD statistics from a Gitlab repository. +Read the `README.md` in the `cicd-statistics-module-gitlab` plugin folder on how to set it up. diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index 4070e5600a..d2257f1b6f 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -39,6 +39,7 @@ Changesets chanwit Chanwit ci +CI/CD classname cli cloudbuild diff --git a/plugins/cicd-statistics-module-gitlab/.eslintrc.js b/plugins/cicd-statistics-module-gitlab/.eslintrc.js new file mode 100644 index 0000000000..e2a53a6ad2 --- /dev/null +++ b/plugins/cicd-statistics-module-gitlab/.eslintrc.js @@ -0,0 +1 @@ +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/plugins/cicd-statistics-module-gitlab/README.md b/plugins/cicd-statistics-module-gitlab/README.md new file mode 100644 index 0000000000..0b11f6950a --- /dev/null +++ b/plugins/cicd-statistics-module-gitlab/README.md @@ -0,0 +1,38 @@ +# cicd-statistics-module-gitlab + +This is an extension module to the `cicd-statistics` plugin, providing a `CicdStatisticsApiGitlab` that you can use to extract the CI/CD statistics from your Gitlab repository. + +## Getting started + +1. Install the `cicd-statistics` and `cicd-statistics-module-gitlab` plugins in the `app` package. + +2. Configure your ApiFactory: + +```tsx +// packages/app/src/apis.ts +import { gitlabAuthApiRef } from '@backstage/core-plugin-api'; + +import { cicdStatisticsApiRef } from '@backstage/plugin-cicd-statistics'; +import { CicdStatisticsApiGitlab } from '@backstage plugin-cicd-statistics-module-gitlab'; + +export const apis: AnyApiFactory[] = [ + createApiFactory({ + api: cicdStatisticsApiRef, + deps: { gitlabAuthApi: gitlabAuthApiRef }, + factory({ gitlabAuthApi }) { + return new CicdStatisticsApiGitlab(gitlabAuthApi); + }, + }), +]; +``` + +3. Add the component to your EntityPage: + +```tsx +// packages/app/src/components/catalog/EntityPage.tsx +import { EntityCicdStatisticsContent } from '@backstage/plugin-cicd-statistics'; + + + +; +``` diff --git a/plugins/cicd-statistics-module-gitlab/api-report.md b/plugins/cicd-statistics-module-gitlab/api-report.md new file mode 100644 index 0000000000..5e07255f54 --- /dev/null +++ b/plugins/cicd-statistics-module-gitlab/api-report.md @@ -0,0 +1,32 @@ +## API Report File for "@backstage/plugin-cicd-statistics-module-gitlab" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { CicdConfiguration } from '@backstage/plugin-cicd-statistics'; +import { CicdState } from '@backstage/plugin-cicd-statistics'; +import { CicdStatisticsApi } from '@backstage/plugin-cicd-statistics'; +import { Entity } from '@backstage/catalog-model'; +import { FetchBuildsOptions } from '@backstage/plugin-cicd-statistics'; +import { Gitlab } from '@gitbeaker/browser'; +import { OAuthApi } from '@backstage/core-plugin-api'; + +// @public +export class CicdStatisticsApiGitlab implements CicdStatisticsApi { + constructor(gitLabAuthApi: OAuthApi); + // (undocumented) + createGitlabApi(entity: Entity, scopes: string[]): Promise; + // (undocumented) + fetchBuilds(options: FetchBuildsOptions): Promise; + // (undocumented) + getConfiguration(): Promise>; +} + +// @public +export type GitlabClient = { + api: InstanceType; + owner: string; +}; + +// (No @packageDocumentation comment for this package) +``` diff --git a/plugins/cicd-statistics-module-gitlab/package.json b/plugins/cicd-statistics-module-gitlab/package.json new file mode 100644 index 0000000000..b36bcf2476 --- /dev/null +++ b/plugins/cicd-statistics-module-gitlab/package.json @@ -0,0 +1,46 @@ +{ + "name": "@backstage/plugin-cicd-statistics-module-gitlab", + "description": "CI/CD Statistics plugin module; Gitlab CICD", + "version": "0.0.0", + "main": "src/index.ts", + "types": "src/index.ts", + "license": "Apache-2.0", + "private": false, + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "module": "dist/index.esm.js", + "types": "dist/index.d.ts" + }, + "backstage": { + "role": "frontend-plugin-module" + }, + "keywords": [ + "backstage", + "cicd statistics", + "gitlab" + ], + "scripts": { + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "clean": "backstage-cli package clean", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack" + }, + "dependencies": { + "@backstage/plugin-cicd-statistics": "^0.1.6-next.0", + "@gitbeaker/browser": "^35.6.0", + "@gitbeaker/core": "^35.6.0", + "luxon": "^2.0.2", + "p-limit": "^4.0.0", + "@backstage/core-plugin-api": "^1.0.0", + "@backstage/catalog-model": "^1.0.1-next.1" + }, + "devDependencies": { + "@backstage/cli": "^0.17.0-next.2" + }, + "files": [ + "dist" + ] +} diff --git a/plugins/cicd-statistics-module-gitlab/src/api/gitlab.ts b/plugins/cicd-statistics-module-gitlab/src/api/gitlab.ts new file mode 100644 index 0000000000..7439d1800c --- /dev/null +++ b/plugins/cicd-statistics-module-gitlab/src/api/gitlab.ts @@ -0,0 +1,159 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + CicdStatisticsApi, + CicdState, + CicdConfiguration, + Build, + FetchBuildsOptions, + Stage, +} from '@backstage/plugin-cicd-statistics'; +import { Gitlab } from '@gitbeaker/browser'; +import { OAuthApi } from '@backstage/core-plugin-api'; +import limiterFactory from 'p-limit'; +import { Entity, getEntitySourceLocation } from '@backstage/catalog-model'; +import { pipelinesToBuilds, jobsToStages } from './utils'; + +/** + * This type represents a initialized gitlab client with gitbeaker + * + * @public + */ +export type GitlabClient = { + /* the actual API of gitbeaker */ + api: InstanceType; + /* the owner the repository, retrieved from the entity source location */ + owner: string; +}; + +/** + * Extracts the CI/CD statistics from a Gitlab repository + * + * @public + */ +export class CicdStatisticsApiGitlab implements CicdStatisticsApi { + readonly #gitLabAuthApi: OAuthApi; + + constructor(gitLabAuthApi: OAuthApi) { + this.#gitLabAuthApi = gitLabAuthApi; + } + + public async createGitlabApi( + entity: Entity, + scopes: string[], + ): Promise { + const entityInfo = getEntitySourceLocation(entity); + const url = new URL(entityInfo.target); + const owner = url.pathname.split('/-/blob/')[0]; + const oauthToken = await this.#gitLabAuthApi.getAccessToken(scopes); + return { + api: new Gitlab({ + host: `https://${url.host}`, + oauthToken, + }), + owner: owner.substring(1), + }; + } + + private static async updateBuildWithStages( + gitbeaker: InstanceType, + owner: string, + build: Build, + ): Promise { + const jobs = await gitbeaker.Jobs.showPipelineJobs( + owner, + parseInt(build.id, 10), + ); + const stages = jobsToStages(jobs); + return stages; + } + + private static async getDurationOfBuild( + gitbeaker: InstanceType, + owner: string, + build: Build, + ): Promise { + const pipeline = await gitbeaker.Pipelines.show( + owner, + parseInt(build.id, 10), + ); + return parseInt(pipeline.duration as string, 10) * 1000; + } + + private static async getDefaultBranch( + gitbeaker: InstanceType, + owner: string, + ): Promise { + const branches = await gitbeaker.Branches.all(owner); + return branches.find(branch => branch.default)?.name; + } + + public async fetchBuilds(options: FetchBuildsOptions): Promise { + const { + entity, + updateProgress, + timeFrom, + timeTo, + filterStatus = ['all'], + filterType = 'all', + } = options; + const { api, owner } = await this.createGitlabApi(entity, ['read_api']); + updateProgress(0, 0, 0); + + const branch = + filterType === 'master' + ? await CicdStatisticsApiGitlab.getDefaultBranch(api, owner) + : undefined; + const pipelines = await api.Pipelines.all(owner, { + perPage: 25, + updated_after: timeFrom.toISOString(), + updated_before: timeTo.toISOString(), + ref: branch, + }); + + const limiter = limiterFactory(10); + const builds = pipelinesToBuilds(pipelines).map(async build => ({ + ...build, + duration: await limiter(() => + CicdStatisticsApiGitlab.getDurationOfBuild(api, owner, build), + ), + stages: await limiter(() => + CicdStatisticsApiGitlab.updateBuildWithStages(api, owner, build), + ), + })); + const promisedBuilds = (await Promise.all(builds)).filter(b => + filterStatus.includes(b.status), + ); + + return { builds: promisedBuilds }; + } + + public async getConfiguration(): Promise> { + return { + availableStatuses: [ + 'succeeded', + 'failed', + 'enqueued', + 'running', + 'aborted', + 'stalled', + 'expired', + 'unknown', + ] as const, + }; + } +} diff --git a/plugins/cicd-statistics-module-gitlab/src/api/index.ts b/plugins/cicd-statistics-module-gitlab/src/api/index.ts new file mode 100644 index 0000000000..21f5f622a6 --- /dev/null +++ b/plugins/cicd-statistics-module-gitlab/src/api/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export * from './gitlab'; diff --git a/plugins/cicd-statistics-module-gitlab/src/api/utils.test.ts b/plugins/cicd-statistics-module-gitlab/src/api/utils.test.ts new file mode 100644 index 0000000000..fda5e68540 --- /dev/null +++ b/plugins/cicd-statistics-module-gitlab/src/api/utils.test.ts @@ -0,0 +1,90 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { pipelinesToBuilds, jobsToStages } from './utils'; +import { Types } from '@gitbeaker/core'; + +const pipelineMock: Types.PipelineSchema[] = [ + { + id: 1000, + iid: 1, + project_id: 1, + sha: 'd40', + ref: 'main', + status: 'success', + source: 'schedule', + created_at: '2022-03-30T13:03:09.846Z', + updated_at: '2022-03-30T13:07:49.248Z', + web_url: 'https://gitlab.com/backstage/app/-/pipelines/1000', + user: { + name: 'user', + avatar_url: 'avatar_user', + }, + }, +]; + +// cast to unknown so we can omit a lot unused vars but also keep the type +const jobMock = [ + { + id: 6962883, + status: 'success', + stage: 'build', + name: 'docker', + ref: 'refs/merge-requests/209/train', + tag: false, + allow_failure: false, + created_at: new Date('2022-03-30T08:35:15.394Z'), + started_at: new Date('2022-03-30T08:35:16.532Z'), + finished_at: new Date('2022-03-30T08:35:37.731Z'), + duration: 21.199465, + queued_duration: 0.976313, + }, +] as unknown as Array; + +describe('util functionality', () => { + it('transforms the pipeline object to the build object', () => { + const builds = pipelinesToBuilds(pipelineMock); + expect(builds).toEqual([ + { + id: '1000', + status: 'succeeded', + branchType: 'master', + duration: 0, + requestedAt: new Date('2022-03-30T13:03:09.846Z'), + triggeredBy: 'internal', + stages: [], + }, + ]); + }); + + it('transforms the job object to the stage object', () => { + const stages = jobsToStages(jobMock); + expect(stages).toEqual([ + { + name: 'build', + status: 'succeeded', + duration: 21199.465, + stages: [ + { + name: 'docker', + status: 'succeeded', + duration: 21199.465, + }, + ], + }, + ]); + }); +}); diff --git a/plugins/cicd-statistics-module-gitlab/src/api/utils.ts b/plugins/cicd-statistics-module-gitlab/src/api/utils.ts new file mode 100644 index 0000000000..5f1d1cbeba --- /dev/null +++ b/plugins/cicd-statistics-module-gitlab/src/api/utils.ts @@ -0,0 +1,107 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + Build, + FilterStatusType, + TriggerReason, + Stage, +} from '@backstage/plugin-cicd-statistics'; +import { Types } from '@gitbeaker/core'; + +const statusMap: Record = { + manual: 'unknown', + created: 'enqueued', + waiting_for_resource: 'stalled', + preparing: 'unknown', + pending: 'scheduled', + running: 'running', + success: 'succeeded', + failed: 'failed', + canceled: 'aborted', + skipped: 'aborted', + scheduled: 'scheduled', +}; + +// all gitlab trigger reasons can be found here: https://docs.gitlab.com/ee/api/pipelines.html#list-project-pipelines +const triggerReasonMap: Record = { + push: 'scm', + trigger: 'manual', + merge_request_event: 'scm', + schedule: 'internal', +}; + +/** + * Takes the Pipeline object from Gitlab and transforms it to the Build object + * + * @param pipelines - Pipeline object that gets returned from Gitlab + * + * @public + */ +export function pipelinesToBuilds( + pipelines: Array, +): Build[] { + return pipelines.map(pipeline => { + return { + id: pipeline.id.toString(), + status: statusMap[pipeline.status], + branchType: 'master', + duration: 0, // will get filled in later in a seperate API call + requestedAt: new Date(pipeline.created_at), + triggeredBy: triggerReasonMap[pipeline.source as string] ?? 'other', + stages: [], + }; + }); +} + +/** + * Takes the Job object from Gitlab and transforms it to the Stage object + * + * @param jobs - Job object that gets returned from Gitlab + * + * @public + * + * @remarks + * + * The Gitlab API can only return the job (sub-stage) of a pipeline and not a whole stage + * The job does return from which stage it is + * So, for the stage name we use the parent stage name and in the sub-stages we add the current job + * In the end the cicd-statistics plugin will calculate the right durations for each stage + * + * Furthermore, we don't add the job to the sub-stage if it is has the same name as the parent stage + * We then assume that the stage has no sub-stages + */ +export function jobsToStages(jobs: Array): Stage[] { + return jobs.map(job => { + const status = statusMap[job.status] ? statusMap[job.status] : 'unknown'; + const duration = job.duration ? ((job.duration * 1000) as number) : 0; + return { + name: job.stage, + status, + duration, + stages: + job.name !== job.stage + ? [ + { + name: job.name, + status, + duration, + }, + ] + : [], + }; + }); +} diff --git a/plugins/cicd-statistics-module-gitlab/src/index.ts b/plugins/cicd-statistics-module-gitlab/src/index.ts new file mode 100644 index 0000000000..c8d53d5bd2 --- /dev/null +++ b/plugins/cicd-statistics-module-gitlab/src/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { CicdStatisticsApiGitlab } from './api'; +export type { GitlabClient } from './api'; diff --git a/plugins/cicd-statistics-module-gitlab/src/setupTests.ts b/plugins/cicd-statistics-module-gitlab/src/setupTests.ts new file mode 100644 index 0000000000..8b9b6bd586 --- /dev/null +++ b/plugins/cicd-statistics-module-gitlab/src/setupTests.ts @@ -0,0 +1,16 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export {}; diff --git a/yarn.lock b/yarn.lock index 85a59e20ce..dac475d2b1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2192,6 +2192,16 @@ resolved "https://registry.npmjs.org/@gar/promisify/-/promisify-1.1.2.tgz#30aa825f11d438671d585bd44e7fd564535fc210" integrity sha512-82cpyJyKRoQoRi+14ibCeGPu0CwypgtBAdBhq1WfvagpCZNKqwXbKwXllYSMG91DhmG4jt9gN8eP6lGOtozuaw== +"@gitbeaker/browser@^35.6.0": + version "35.6.0" + resolved "https://registry.npmjs.org/@gitbeaker/browser/-/browser-35.6.0.tgz#a639ae57eeb4d4829b73a634f757ef5841ad4116" + integrity sha512-cMkMG3r1HfVV7AUxVfp+PzDpISJYDQK5TnFfVT9vwVfW2uUDYALxMWYFwBLPkn3otqrOY1TkHXl/I9WgwsZMfg== + dependencies: + "@gitbeaker/core" "^35.6.0" + "@gitbeaker/requester-utils" "^35.6.0" + delay "^5.0.0" + ky "0.28.7" + "@gitbeaker/core@^35.5.0", "@gitbeaker/core@^35.6.0": version "35.6.0" resolved "https://registry.npmjs.org/@gitbeaker/core/-/core-35.6.0.tgz#c46e15986081ef1e647434c450964eed5fb0e390" @@ -16322,6 +16332,11 @@ kuler@^2.0.0: resolved "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz#e2c570a3800388fb44407e851531c1d670b061b3" integrity sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A== +ky@0.28.7: + version "0.28.7" + resolved "https://registry.npmjs.org/ky/-/ky-0.28.7.tgz#10c42be863fb96c1846d6e71e229263ffb72eb15" + integrity sha512-a23i6qSr/ep15vdtw/zyEQIDLoUaKDg9Jf04CYl/0ns/wXNYna26zJpI+MeIFaPeDvkrjLPrKtKOiiI3IE53RQ== + language-subtag-registry@~0.3.2: version "0.3.21" resolved "https://registry.npmjs.org/language-subtag-registry/-/language-subtag-registry-0.3.21.tgz#04ac218bea46f04cb039084602c6da9e788dd45a" @@ -19116,6 +19131,13 @@ p-limit@^2.0.0, p-limit@^2.2.0: dependencies: p-try "^2.0.0" +p-limit@^4.0.0: + version "4.0.0" + resolved "https://registry.npmjs.org/p-limit/-/p-limit-4.0.0.tgz#914af6544ed32bfa54670b061cafcbd04984b644" + integrity sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ== + dependencies: + yocto-queue "^1.0.0" + p-locate@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz#20a0103b222a70c8fd39cc2e580680f3dde5ec43" @@ -25801,6 +25823,11 @@ yocto-queue@^0.1.0: resolved "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b" integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== +yocto-queue@^1.0.0: + version "1.0.0" + resolved "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.0.0.tgz#7f816433fb2cbc511ec8bf7d263c3b58a1a3c251" + integrity sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g== + yup@^0.32.9: version "0.32.11" resolved "https://registry.npmjs.org/yup/-/yup-0.32.11.tgz#d67fb83eefa4698607982e63f7ca4c5ed3cf18c5"