paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 38 Packages Projects Releases Wiki Activity

Merge pull request #10140 from djamaile/feat/cicd-statistics-module-gitlab

Browse Source 
plugin: introduce gitlab ci/cd statistics module
This commit is contained in:
Fredrik Adelöw
2022-04-12 11:39:14 +02:00
committed by GitHub
parent 4e74532566 b4c5585234
commit 6de4927c75
13 changed files with 558 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/proud-news-perform.md
+6
View File
@@ -0,0 +1,6 @@
---
'@backstage/plugin-cicd-statistics-module-gitlab': minor
---
Created a module to extract the CI/CD statistics from a Gitlab repository.
Read the `README.md` in the `cicd-statistics-module-gitlab` plugin folder on how to set it up.
.github/styles/vocab.txt
+1
View File
@@ -39,6 +39,7 @@ Changesets
chanwit
Chanwit
ci
CI/CD
classname
cli
cloudbuild
plugins/cicd-statistics-module-gitlab/.eslintrc.js
+1
View File
@@ -0,0 +1 @@
module.exports = require('@backstage/cli/config/eslint-factory')(__dirname);
plugins/cicd-statistics-module-gitlab/README.md
+38
View File
@@ -0,0 +1,38 @@
# cicd-statistics-module-gitlab
This is an extension module to the `cicd-statistics` plugin, providing a `CicdStatisticsApiGitlab` that you can use to extract the CI/CD statistics from your Gitlab repository.
## Getting started
1. Install the `cicd-statistics` and `cicd-statistics-module-gitlab` plugins in the `app` package.
2. Configure your ApiFactory:
```tsx
// packages/app/src/apis.ts
import { gitlabAuthApiRef } from '@backstage/core-plugin-api';
import { cicdStatisticsApiRef } from '@backstage/plugin-cicd-statistics';
import { CicdStatisticsApiGitlab } from '@backstage plugin-cicd-statistics-module-gitlab';
export const apis: AnyApiFactory[] = [
createApiFactory({
api: cicdStatisticsApiRef,
deps: { gitlabAuthApi: gitlabAuthApiRef },
factory({ gitlabAuthApi }) {
return new CicdStatisticsApiGitlab(gitlabAuthApi);
},
}),
];
```
3. Add the component to your EntityPage:
```tsx
// packages/app/src/components/catalog/EntityPage.tsx
import { EntityCicdStatisticsContent } from '@backstage/plugin-cicd-statistics';
<EntityLayout.Route path="/ci-cd-statistics" title="CI/CD Statistics">
<EntityCicdStatisticsContent />
</EntityLayout.Route>;
```
plugins/cicd-statistics-module-gitlab/api-report.md
+32
View File
@@ -0,0 +1,32 @@
## API Report File for "@backstage/plugin-cicd-statistics-module-gitlab"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
```ts
import { CicdConfiguration } from '@backstage/plugin-cicd-statistics';
import { CicdState } from '@backstage/plugin-cicd-statistics';
import { CicdStatisticsApi } from '@backstage/plugin-cicd-statistics';
import { Entity } from '@backstage/catalog-model';
import { FetchBuildsOptions } from '@backstage/plugin-cicd-statistics';
import { Gitlab } from '@gitbeaker/browser';
import { OAuthApi } from '@backstage/core-plugin-api';
// @public
export class CicdStatisticsApiGitlab implements CicdStatisticsApi {
constructor(gitLabAuthApi: OAuthApi);
// (undocumented)
createGitlabApi(entity: Entity, scopes: string[]): Promise<GitlabClient>;
// (undocumented)
fetchBuilds(options: FetchBuildsOptions): Promise<CicdState>;
// (undocumented)
getConfiguration(): Promise<Partial<CicdConfiguration>>;
}
// @public
export type GitlabClient = {
api: InstanceType<typeof Gitlab>;
owner: string;
};
// (No @packageDocumentation comment for this package)
```
plugins/cicd-statistics-module-gitlab/package.json
+46
View File
@@ -0,0 +1,46 @@
{
"name": "@backstage/plugin-cicd-statistics-module-gitlab",
"description": "CI/CD Statistics plugin module; Gitlab CICD",
"version": "0.0.0",
"main": "src/index.ts",
"types": "src/index.ts",
"license": "Apache-2.0",
"private": false,
"publishConfig": {
"access": "public",
"main": "dist/index.cjs.js",
"module": "dist/index.esm.js",
"types": "dist/index.d.ts"
},
"backstage": {
"role": "frontend-plugin-module"
},
"keywords": [
"backstage",
"cicd statistics",
"gitlab"
],
"scripts": {
"build": "backstage-cli package build",
"lint": "backstage-cli package lint",
"test": "backstage-cli package test",
"clean": "backstage-cli package clean",
"prepack": "backstage-cli package prepack",
"postpack": "backstage-cli package postpack"
},
"dependencies": {
"@backstage/plugin-cicd-statistics": "^0.1.6-next.0",
"@gitbeaker/browser": "^35.6.0",
"@gitbeaker/core": "^35.6.0",
"luxon": "^2.0.2",
"p-limit": "^4.0.0",
"@backstage/core-plugin-api": "^1.0.0",
"@backstage/catalog-model": "^1.0.1-next.1"
},
"devDependencies": {
"@backstage/cli": "^0.17.0-next.2"
},
"files": [
"dist"
]
}
plugins/cicd-statistics-module-gitlab/src/api/gitlab.ts
+159
View File
@@ -0,0 +1,159 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
CicdStatisticsApi,
CicdState,
CicdConfiguration,
Build,
FetchBuildsOptions,
Stage,
} from '@backstage/plugin-cicd-statistics';
import { Gitlab } from '@gitbeaker/browser';
import { OAuthApi } from '@backstage/core-plugin-api';
import limiterFactory from 'p-limit';
import { Entity, getEntitySourceLocation } from '@backstage/catalog-model';
import { pipelinesToBuilds, jobsToStages } from './utils';
/**
* This type represents a initialized gitlab client with gitbeaker
*
* @public
*/
export type GitlabClient = {
/* the actual API of gitbeaker */
api: InstanceType<typeof Gitlab>;
/* the owner the repository, retrieved from the entity source location */
owner: string;
};
/**
* Extracts the CI/CD statistics from a Gitlab repository
*
* @public
*/
export class CicdStatisticsApiGitlab implements CicdStatisticsApi {
readonly #gitLabAuthApi: OAuthApi;
constructor(gitLabAuthApi: OAuthApi) {
this.#gitLabAuthApi = gitLabAuthApi;
}
public async createGitlabApi(
entity: Entity,
scopes: string[],
): Promise<GitlabClient> {
const entityInfo = getEntitySourceLocation(entity);
const url = new URL(entityInfo.target);
const owner = url.pathname.split('/-/blob/')[0];
const oauthToken = await this.#gitLabAuthApi.getAccessToken(scopes);
return {
api: new Gitlab({
host: `https://${url.host}`,
oauthToken,
}),
owner: owner.substring(1),
};
}
private static async updateBuildWithStages(
gitbeaker: InstanceType<typeof Gitlab>,
owner: string,
build: Build,
): Promise<Stage[]> {
const jobs = await gitbeaker.Jobs.showPipelineJobs(
owner,
parseInt(build.id, 10),
);
const stages = jobsToStages(jobs);
return stages;
}
private static async getDurationOfBuild(
gitbeaker: InstanceType<typeof Gitlab>,
owner: string,
build: Build,
): Promise<number> {
const pipeline = await gitbeaker.Pipelines.show(
owner,
parseInt(build.id, 10),
);
return parseInt(pipeline.duration as string, 10) * 1000;
}
private static async getDefaultBranch(
gitbeaker: InstanceType<typeof Gitlab>,
owner: string,
): Promise<string | undefined> {
const branches = await gitbeaker.Branches.all(owner);
return branches.find(branch => branch.default)?.name;
}
public async fetchBuilds(options: FetchBuildsOptions): Promise<CicdState> {
const {
entity,
updateProgress,
timeFrom,
timeTo,
filterStatus = ['all'],
filterType = 'all',
} = options;
const { api, owner } = await this.createGitlabApi(entity, ['read_api']);
updateProgress(0, 0, 0);
const branch =
filterType === 'master'
? await CicdStatisticsApiGitlab.getDefaultBranch(api, owner)
: undefined;
const pipelines = await api.Pipelines.all(owner, {
perPage: 25,
updated_after: timeFrom.toISOString(),
updated_before: timeTo.toISOString(),
ref: branch,
});
const limiter = limiterFactory(10);
const builds = pipelinesToBuilds(pipelines).map(async build => ({
...build,
duration: await limiter(() =>
CicdStatisticsApiGitlab.getDurationOfBuild(api, owner, build),
),
stages: await limiter(() =>
CicdStatisticsApiGitlab.updateBuildWithStages(api, owner, build),
),
}));
const promisedBuilds = (await Promise.all(builds)).filter(b =>
filterStatus.includes(b.status),
);
return { builds: promisedBuilds };
}
public async getConfiguration(): Promise<Partial<CicdConfiguration>> {
return {
availableStatuses: [
'succeeded',
'failed',
'enqueued',
'running',
'aborted',
'stalled',
'expired',
'unknown',
] as const,
};
}
}
plugins/cicd-statistics-module-gitlab/src/api/index.ts
+17
View File
@@ -0,0 +1,17 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export * from './gitlab';
plugins/cicd-statistics-module-gitlab/src/api/utils.test.ts
+90
View File
@@ -0,0 +1,90 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { pipelinesToBuilds, jobsToStages } from './utils';
import { Types } from '@gitbeaker/core';
const pipelineMock: Types.PipelineSchema[] = [
{
id: 1000,
iid: 1,
project_id: 1,
sha: 'd40',
ref: 'main',
status: 'success',
source: 'schedule',
created_at: '2022-03-30T13:03:09.846Z',
updated_at: '2022-03-30T13:07:49.248Z',
web_url: 'https://gitlab.com/backstage/app/-/pipelines/1000',
user: {
name: 'user',
avatar_url: 'avatar_user',
},
},
];
// cast to unknown so we can omit a lot unused vars but also keep the type
const jobMock = [
{
id: 6962883,
status: 'success',
stage: 'build',
name: 'docker',
ref: 'refs/merge-requests/209/train',
tag: false,
allow_failure: false,
created_at: new Date('2022-03-30T08:35:15.394Z'),
started_at: new Date('2022-03-30T08:35:16.532Z'),
finished_at: new Date('2022-03-30T08:35:37.731Z'),
duration: 21.199465,
queued_duration: 0.976313,
},
] as unknown as Array<Types.JobSchema>;
describe('util functionality', () => {
it('transforms the pipeline object to the build object', () => {
const builds = pipelinesToBuilds(pipelineMock);
expect(builds).toEqual([
{
id: '1000',
status: 'succeeded',
branchType: 'master',
duration: 0,
requestedAt: new Date('2022-03-30T13:03:09.846Z'),
triggeredBy: 'internal',
stages: [],
},
]);
});
it('transforms the job object to the stage object', () => {
const stages = jobsToStages(jobMock);
expect(stages).toEqual([
{
name: 'build',
status: 'succeeded',
duration: 21199.465,
stages: [
{
name: 'docker',
status: 'succeeded',
duration: 21199.465,
},
],
},
]);
});
});
plugins/cicd-statistics-module-gitlab/src/api/utils.ts
+107
View File
@@ -0,0 +1,107 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
Build,
FilterStatusType,
TriggerReason,
Stage,
} from '@backstage/plugin-cicd-statistics';
import { Types } from '@gitbeaker/core';
const statusMap: Record<string, FilterStatusType> = {
manual: 'unknown',
created: 'enqueued',
waiting_for_resource: 'stalled',
preparing: 'unknown',
pending: 'scheduled',
running: 'running',
success: 'succeeded',
failed: 'failed',
canceled: 'aborted',
skipped: 'aborted',
scheduled: 'scheduled',
};
// all gitlab trigger reasons can be found here: https://docs.gitlab.com/ee/api/pipelines.html#list-project-pipelines
const triggerReasonMap: Record<string, TriggerReason> = {
push: 'scm',
trigger: 'manual',
merge_request_event: 'scm',
schedule: 'internal',
};
/**
* Takes the Pipeline object from Gitlab and transforms it to the Build object
*
* @param pipelines - Pipeline object that gets returned from Gitlab
*
* @public
*/
export function pipelinesToBuilds(
pipelines: Array<Types.PipelineSchema>,
): Build[] {
return pipelines.map(pipeline => {
return {
id: pipeline.id.toString(),
status: statusMap[pipeline.status],
branchType: 'master',
duration: 0, // will get filled in later in a seperate API call
requestedAt: new Date(pipeline.created_at),
triggeredBy: triggerReasonMap[pipeline.source as string] ?? 'other',
stages: [],
};
});
}
/**
* Takes the Job object from Gitlab and transforms it to the Stage object
*
* @param jobs - Job object that gets returned from Gitlab
*
* @public
*
* @remarks
*
* The Gitlab API can only return the job (sub-stage) of a pipeline and not a whole stage
* The job does return from which stage it is
* So, for the stage name we use the parent stage name and in the sub-stages we add the current job
* In the end the cicd-statistics plugin will calculate the right durations for each stage
*
* Furthermore, we don't add the job to the sub-stage if it is has the same name as the parent stage
* We then assume that the stage has no sub-stages
*/
export function jobsToStages(jobs: Array<Types.JobSchema>): Stage[] {
return jobs.map(job => {
const status = statusMap[job.status] ? statusMap[job.status] : 'unknown';
const duration = job.duration ? ((job.duration * 1000) as number) : 0;
return {
name: job.stage,
status,
duration,
stages:
job.name !== job.stage
? [
{
name: job.name,
status,
duration,
},
]
: [],
};
});
}
plugins/cicd-statistics-module-gitlab/src/index.ts
+18
View File
@@ -0,0 +1,18 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { CicdStatisticsApiGitlab } from './api';
export type { GitlabClient } from './api';
plugins/cicd-statistics-module-gitlab/src/setupTests.ts
+16
View File
@@ -0,0 +1,16 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export {};
yarn.lock
+27
View File
@@ -2192,6 +2192,16 @@
resolved "https://registry.npmjs.org/@gar/promisify/-/promisify-1.1.2.tgz#30aa825f11d438671d585bd44e7fd564535fc210"
integrity sha512-82cpyJyKRoQoRi+14ibCeGPu0CwypgtBAdBhq1WfvagpCZNKqwXbKwXllYSMG91DhmG4jt9gN8eP6lGOtozuaw==
"@gitbeaker/browser@^35.6.0":
version "35.6.0"
resolved "https://registry.npmjs.org/@gitbeaker/browser/-/browser-35.6.0.tgz#a639ae57eeb4d4829b73a634f757ef5841ad4116"
integrity sha512-cMkMG3r1HfVV7AUxVfp+PzDpISJYDQK5TnFfVT9vwVfW2uUDYALxMWYFwBLPkn3otqrOY1TkHXl/I9WgwsZMfg==
dependencies:
"@gitbeaker/core" "^35.6.0"
"@gitbeaker/requester-utils" "^35.6.0"
delay "^5.0.0"
ky "0.28.7"
"@gitbeaker/core@^35.5.0", "@gitbeaker/core@^35.6.0":
version "35.6.0"
resolved "https://registry.npmjs.org/@gitbeaker/core/-/core-35.6.0.tgz#c46e15986081ef1e647434c450964eed5fb0e390"
@@ -16322,6 +16332,11 @@ kuler@^2.0.0:
resolved "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz#e2c570a3800388fb44407e851531c1d670b061b3"
integrity sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==
ky@0.28.7:
version "0.28.7"
resolved "https://registry.npmjs.org/ky/-/ky-0.28.7.tgz#10c42be863fb96c1846d6e71e229263ffb72eb15"
integrity sha512-a23i6qSr/ep15vdtw/zyEQIDLoUaKDg9Jf04CYl/0ns/wXNYna26zJpI+MeIFaPeDvkrjLPrKtKOiiI3IE53RQ==
language-subtag-registry@~0.3.2:
version "0.3.21"
resolved "https://registry.npmjs.org/language-subtag-registry/-/language-subtag-registry-0.3.21.tgz#04ac218bea46f04cb039084602c6da9e788dd45a"
@@ -19116,6 +19131,13 @@ p-limit@^2.0.0, p-limit@^2.2.0:
dependencies:
p-try "^2.0.0"
p-limit@^4.0.0:
version "4.0.0"
resolved "https://registry.npmjs.org/p-limit/-/p-limit-4.0.0.tgz#914af6544ed32bfa54670b061cafcbd04984b644"
integrity sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==
dependencies:
yocto-queue "^1.0.0"
p-locate@^2.0.0:
version "2.0.0"
resolved "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz#20a0103b222a70c8fd39cc2e580680f3dde5ec43"
@@ -25801,6 +25823,11 @@ yocto-queue@^0.1.0:
resolved "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b"
integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==
yocto-queue@^1.0.0:
version "1.0.0"
resolved "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.0.0.tgz#7f816433fb2cbc511ec8bf7d263c3b58a1a3c251"
integrity sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==
yup@^0.32.9:
version "0.32.11"
resolved "https://registry.npmjs.org/yup/-/yup-0.32.11.tgz#d67fb83eefa4698607982e63f7ca4c5ed3cf18c5"