feat(cicd-statistics): Added trigger reason

Some refactoring and tweaks. The trigger reason is an overlay to the build counts, and is a percentage of manually triggered builds vs all builds. When this is high, it indicates likelyhood of flaky tests.

Signed-off-by: Gustaf Räntilä <g.rantila@gmail.com>
This commit is contained in:
Gustaf Räntilä
2022-02-02 13:18:19 +01:00
committed by blam
parent ec1ddbd812
commit fdabd241d2
13 changed files with 436 additions and 79 deletions
+20
View File
@@ -53,6 +53,23 @@ export const statusTypes: Array<FilterStatusType> = [
*/
export type FilterBranchType = 'master' | 'branch';
export type TriggerReason =
/** Triggered by source code management, e.g. a Github hook */
| 'scm'
/** Triggered manually */
| 'manual'
/** Triggered internally (non-scm, or perhaps after being delayed/enqueued) */
| 'internal'
/** Triggered for some other reason */
| 'other';
export const triggerReasons: Array<TriggerReason> = [
'scm',
'manual',
'internal',
'other',
];
/**
* A Stage is a part of either a Build or a parent Stage.
*
@@ -84,6 +101,9 @@ export interface Build {
/** Build id */
id: string;
/** The reason this build was started */
triggeredBy?: TriggerReason;
/** The status of the build */
status: FilterStatusType;
+8 -1
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { FilterStatusType } from '../apis/types';
import { FilterStatusType, TriggerReason } from '../apis/types';
export const statusColorMap: Record<FilterStatusType, string> = {
unknown: '#3d01a4',
@@ -28,6 +28,13 @@ export const statusColorMap: Record<FilterStatusType, string> = {
expired: '#a7194b',
};
export const triggerColorMap: Record<TriggerReason, string> = {
scm: '#0391ce',
manual: '#a7194b',
internal: '#82ca9d',
other: '#f3f318',
};
export const fireColors: Array<[percent: string, color: string]> = [
['5%', '#e19678'],
['30%', '#dfe178'],
@@ -16,10 +16,11 @@
import { map } from 'already';
import { Build, Stage } from '../../apis/types';
import { Build, Stage, FilterStatusType } from '../../apis/types';
import { ChartableStage, ChartableStagesAnalysis } from '../types';
import { getOrSetStage, makeStage } from './utils';
import { getOrSetStage, makeStage, sortStatuses } from './utils';
import { finalizeStage } from './finalize-stage';
import { dailySummary } from './daily-summary';
export interface ChartableStagesOptions {
normalizeTimeRange: boolean;
@@ -86,5 +87,34 @@ export async function buildsToChartableStages(
);
finalizeStage(total, { allEpochs, averageWidth: 10 });
return { total, stages };
const daily = dailySummary(builds);
const statuses = findStatuses(total, [...stages.values()]);
return { daily, total, stages, statuses };
}
function findStatuses(
total: ChartableStage,
stages: Array<ChartableStage>,
): Array<string> {
const statuses = new Set<string>();
const addStatuses = (set: Set<FilterStatusType>) => {
set.forEach(status => {
statuses.add(status);
});
};
addStatuses(total.statusSet);
const recurse = (subStages: Array<ChartableStage>) => {
subStages.forEach(stage => {
addStatuses(stage.statusSet);
recurse([...stage.stages.values()]);
});
};
recurse(stages);
return sortStatuses([...statuses]);
}
@@ -14,20 +14,16 @@
* limitations under the License.
*/
import { DateTime } from 'luxon';
import { groupBy } from 'lodash';
import { FilterStatusType, statusTypes } from '../../apis/types';
import { Countify, ChartableStageDatapoints } from '../types';
function startOfDayEpoch(epoch: number) {
return DateTime.fromMillis(epoch).startOf('day').toMillis();
}
import { startOfDay } from './utils';
export function countBuildsPerDay(
values: ReadonlyArray<ChartableStageDatapoints>,
) {
const days = groupBy(values, value => startOfDayEpoch(value.__epoch));
const days = groupBy(values, value => startOfDay(value.__epoch));
Object.entries(days).forEach(([_startOfDay, valuesThisDay]) => {
const counts = Object.fromEntries(
statusTypes
@@ -35,7 +31,7 @@ export function countBuildsPerDay(
type =>
[
type,
valuesThisDay.map(value => value[type] !== undefined).length,
valuesThisDay.filter(value => value[type] !== undefined).length,
] as const,
)
.filter(([_type, count]) => count > 0)
@@ -0,0 +1,115 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { groupBy, countBy } from 'lodash';
import { Build } from '../../apis/types';
import {
Epoch,
TriggerReasonsDatapoint,
StatusesDatapoint,
ChartableDaily,
} from '../types';
import { sortStatuses, sortTriggerReasons, startOfDay } from './utils';
export function dailySummary(builds: ReadonlyArray<Build>): ChartableDaily {
const triggersDaily = countTriggersPerDay(builds);
const statusesDaily = countStatusesPerDay(builds);
const { triggerReasons } = triggersDaily;
const { statuses } = statusesDaily;
const reasonMap = new Map(
triggersDaily.values.map(value => [value.__epoch, value]),
);
const statusMap = new Map(
statusesDaily.values.map(value => [value.__epoch, value]),
);
const days = Object.keys(
groupBy(builds, value => startOfDay(value.requestedAt)),
)
.map(epoch => parseInt(epoch, 10))
.sort();
return {
values: days.map(epoch => ({
__epoch: epoch,
...reasonMap.get(epoch),
...statusMap.get(epoch),
})),
triggerReasons,
statuses,
};
}
function countTriggersPerDay(builds: ReadonlyArray<Build>) {
const days = groupBy(builds, value => startOfDay(value.requestedAt));
const triggerReasons = sortTriggerReasons([
...new Set(
builds
.map(({ triggeredBy }) => triggeredBy)
.filter((v): v is NonNullable<typeof v> => !!v),
),
]);
const values = Object.entries(days).map(([epoch, buildsThisDay]) => {
const datapoint = Object.fromEntries(
triggerReasons
.map(reason => [
reason,
buildsThisDay.filter(build => build.triggeredBy === reason).length,
])
.filter(([_type, count]) => count > 0),
) as Omit<TriggerReasonsDatapoint, '__epoch'>;
// Assign the count for this day to the first value this day
const value: Epoch & TriggerReasonsDatapoint = Object.assign(datapoint, {
__epoch: parseInt(epoch, 10),
});
return value;
});
return { triggerReasons, values };
}
function countStatusesPerDay(builds: ReadonlyArray<Build>) {
const days = groupBy(builds, value => startOfDay(value.requestedAt));
const foundStatuses = new Set<string>();
const values = Object.entries(days).map(([epoch, buildsThisDay]) => {
const byStatus = countBy(buildsThisDay, 'status');
const value: Epoch & StatusesDatapoint = {
__epoch: parseInt(epoch, 10),
...byStatus,
};
Object.keys(byStatus).forEach(status => {
foundStatuses.add(status);
});
return value;
});
return {
statuses: sortStatuses([...foundStatuses]),
values,
};
}
@@ -0,0 +1,27 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { sortTriggerReasons } from './daily-summary';
describe('daily-summary', () => {
it('sortTriggerReasons', () => {
const values = ['a', 'manual', 'b', 'other', 'c', 'scm', 'd'];
const expected = ['manual', 'scm', 'other', 'a', 'b', 'c', 'd'];
expect(sortTriggerReasons(values)).toStrictEqual(expected);
expect(sortTriggerReasons(values.reverse())).toStrictEqual(expected);
});
});
@@ -14,7 +14,9 @@
* limitations under the License.
*/
import { FilterStatusType } from '../../apis/types';
import { DateTime } from 'luxon';
import { FilterStatusType, statusTypes } from '../../apis/types';
import { ChartableStage } from '../types';
export function average(values: number[]): number {
@@ -55,3 +57,31 @@ export function makeStage(name: string): ChartableStage {
stages: new Map(),
};
}
export function startOfDay(date: number | Date) {
if (typeof date === 'number') {
return DateTime.fromMillis(date).startOf('day').toMillis();
}
return DateTime.fromJSDate(date).startOf('day').toMillis();
}
export function sortTriggerReasons(reasons: Array<string>): Array<string> {
return reasons.sort((a, b) => {
if (a === 'manual') return -1;
else if (b === 'manual') return 1;
else if (a === 'scm') return -1;
else if (b === 'scm') return 1;
else if (a === 'other') return -1;
else if (b === 'other') return 1;
return a.localeCompare(b);
});
}
export function sortStatuses(statuses: Array<string>): Array<string> {
return [
...statusTypes.filter(status => statuses.includes(status)),
...statuses
.filter(status => !(statusTypes as Array<string>).includes(status))
.sort((a, b) => a.localeCompare(b)),
];
}
@@ -38,6 +38,7 @@ import {
Typography,
} from '@material-ui/core';
import ExpandMoreIcon from '@material-ui/icons/ExpandMore';
import { capitalize } from 'lodash';
import { CicdDefaults, statusTypes } from '../apis/types';
import { ChartableStage } from './types';
@@ -95,7 +96,7 @@ export function StageChart(props: StageChartProps) {
const legendPayload = useMemo(
(): LegendProps['payload'] =>
statuses.map(status => ({
value: status,
value: capitalize(status),
type: 'line',
id: status,
color: statusColorMap[status],
@@ -172,7 +173,7 @@ export function StageChart(props: StageChartProps) {
labelFormatter={labelFormatter}
/>
{statuses.reverse().map(status => (
<>
<Fragment key={status}>
{!chartTypes[status].includes('duration') ? null : (
<Fragment key={status}>
<Area
@@ -220,7 +221,7 @@ export function StageChart(props: StageChartProps) {
fill={statusColorMap[status] ?? ''}
/>
)}
</>
</Fragment>
))}
</ComposedChart>
</ResponsiveContainer>
@@ -16,6 +16,7 @@
import React, { Fragment, useMemo } from 'react';
import {
Area,
Bar,
ComposedChart,
XAxis,
@@ -34,66 +35,101 @@ import {
Typography,
} from '@material-ui/core';
import ExpandMoreIcon from '@material-ui/icons/ExpandMore';
import { countBy } from 'lodash';
import { DateTime } from 'luxon';
import { capitalize } from 'lodash';
import { Build, FilterStatusType, statusTypes } from '../apis/types';
import { FilterStatusType, TriggerReason } from '../apis/types';
import { labelFormatterWithoutTime, tickFormatterX } from '../components/utils';
import { statusColorMap } from './colors';
import { statusColorMap, triggerColorMap } from './colors';
import { ChartableStagesAnalysis } from './types';
export interface StatusChartProps {
builds: ReadonlyArray<Build>;
analysis: ChartableStagesAnalysis;
}
export function StatusChart(props: StatusChartProps) {
const { builds } = props;
const { analysis } = props;
const { statuses, values } = useMemo(() => {
const buildsByDay = new Map<string, Array<Build>>();
const values = useMemo(() => {
return analysis.daily.values.map(value => {
const totTriggers = analysis.daily.triggerReasons.reduce(
(prev, cur) => prev + (value[cur as TriggerReason] ?? 0),
0,
);
const foundStatuses = new Set<string>();
builds.forEach(build => {
foundStatuses.add(build.status);
const dayString = DateTime.fromJSDate(build.requestedAt).toISODate();
const dayList = buildsByDay.get(dayString);
if (dayList) {
dayList.push(build);
} else {
buildsByDay.set(dayString, [build]);
if (!totTriggers) {
return value;
}
});
return {
statuses: [
...statusTypes.filter(status => foundStatuses.has(status)),
...[...foundStatuses].filter(
status => !(statusTypes as Array<string>).includes(status),
return {
...value,
...Object.fromEntries(
analysis.daily.triggerReasons.map(reason => [
reason,
(value[reason as TriggerReason] ?? 0) / totTriggers,
]),
),
],
values: [...buildsByDay.entries()].map(([dayString, buildThisDay]) => ({
__epoch: DateTime.fromISO(dayString).toMillis(),
...countBy(buildThisDay, 'status'),
})),
};
}, [builds]);
};
});
}, [analysis.daily]);
const legendPayload = useMemo(
(): LegendProps['payload'] =>
statuses.map(status => ({
value: status,
const triggerReasonLegendPayload = useMemo(
(): NonNullable<LegendProps['payload']> =>
analysis.daily.triggerReasons.map(reason => ({
value: humanTriggerReason(reason),
type: 'line',
id: reason,
color: triggerColorMap[reason as TriggerReason] ?? '',
})),
[analysis.daily.triggerReasons],
);
const statusesLegendPayload = useMemo(
(): NonNullable<LegendProps['payload']> =>
analysis.daily.statuses.map(status => ({
value: capitalize(status),
type: 'line',
id: status,
color: statusColorMap[status as FilterStatusType] ?? '',
})),
[statuses],
[analysis.daily.statuses],
);
const legendPayload = useMemo(
(): NonNullable<LegendProps['payload']> => [
...triggerReasonLegendPayload,
...statusesLegendPayload,
],
[statusesLegendPayload, triggerReasonLegendPayload],
);
const tooltipFormatter = useMemo(() => {
const reasonSet = new Set(analysis.daily.triggerReasons);
return (percentOrCount: number, name: string) => {
const label = reasonSet.has(name)
? humanTriggerReason(name)
: capitalize(name);
const valueText = reasonSet.has(name)
? `${(percentOrCount * 100).toFixed(0)}%`
: percentOrCount;
return [
<span>
{label}: {valueText}
</span>,
null,
];
};
}, [analysis.daily.triggerReasons]);
const barSize = getBarSize(analysis.daily.values.length);
return (
<Accordion defaultExpanded={statuses.length > 1}>
<Accordion defaultExpanded={analysis.daily.statuses.length > 1}>
<AccordionSummary expandIcon={<ExpandMoreIcon />}>
<Typography>Build count per status</Typography>
<Typography>
Build count per status over build trigger reason
</Typography>
</AccordionSummary>
<AccordionDetails>
{values.length === 0 ? (
@@ -108,14 +144,33 @@ export function StatusChart(props: StatusChartProps) {
type="category"
tickFormatter={tickFormatterX}
/>
<YAxis type="number" tickCount={5} name="Count" />
<Tooltip labelFormatter={labelFormatterWithoutTime} />
{statuses.map(status => (
<YAxis yAxisId={1} type="number" tickCount={5} name="Count" />
<YAxis yAxisId={2} type="number" name="Triggers" hide />
<Tooltip
labelFormatter={labelFormatterWithoutTime}
formatter={tooltipFormatter}
/>
{triggerReasonLegendPayload.map(reason => (
<Fragment key={reason.id}>
<Area
type="monotone"
dataKey={reason.id!}
stackId="triggers"
yAxisId={2}
stroke={triggerColorMap[reason.id as TriggerReason] ?? ''}
fillOpacity={0.5}
fill={triggerColorMap[reason.id as TriggerReason] ?? ''}
/>
</Fragment>
))}
{[...analysis.daily.statuses].reverse().map(status => (
<Fragment key={status}>
<Bar
type="monotone"
barSize={barSize}
dataKey={status}
stackId="1"
stackId="statuses"
yAxisId={1}
stroke={statusColorMap[status as FilterStatusType] ?? ''}
fillOpacity={0.8}
fill={statusColorMap[status as FilterStatusType] ?? ''}
@@ -129,3 +184,25 @@ export function StatusChart(props: StatusChartProps) {
</Accordion>
);
}
function humanTriggerReason(reason: string): string {
if ((reason as TriggerReason) === 'manual') {
return 'Triggered manually';
} else if ((reason as TriggerReason) === 'scm') {
return 'Triggered by SCM';
} else if ((reason as TriggerReason) === 'internal') {
return 'Triggered internally';
} else if ((reason as TriggerReason) === 'other') {
return 'Triggered by another reason';
}
return `Triggered by ${reason}`;
}
function getBarSize(count: number): number {
if (count < 20) {
return 10;
} else if (count < 40) {
return 8;
}
return 5;
}
+48 -7
View File
@@ -14,14 +14,14 @@
* limitations under the License.
*/
import { FilterStatusType } from '../apis/types';
import { FilterStatusType, TriggerReason } from '../apis/types';
export type Averagify<T extends string> = `${T} avg`;
export type Countify<T extends string> = `${T} count`;
export type ChartableStageDatapoints = {
__epoch: number;
} & {
export type Epoch = { __epoch: number };
export type ChartableStageDatapoints = Epoch & {
[status in FilterStatusType]?: number;
} & {
[status in Averagify<FilterStatusType>]?: number;
@@ -50,7 +50,48 @@ export interface ChartableStage {
stages: Map<string, ChartableStage>;
}
export interface ChartableStagesAnalysis {
total: ChartableStage;
stages: Map<string, ChartableStage>;
export type TriggerReasonsDatapoint = {
[K in TriggerReason]?: number;
};
export type StatusesDatapoint = { [status in FilterStatusType]?: number };
export type ChartableDailyDatapoint = Epoch &
TriggerReasonsDatapoint &
StatusesDatapoint;
export interface ChartableDaily {
values: Array<ChartableDailyDatapoint>;
/**
* The build trigger reasons
*/
triggerReasons: Array<string>;
/**
* The top-level (build) statuses
*/
statuses: Array<string>;
}
export interface ChartableStagesAnalysis {
/**
* Summary of statuses and trigger reasons per day
*/
daily: ChartableDaily;
/**
* Total aggregates of sub stages
*/
total: ChartableStage;
/**
* Top-level stages {name -> stage}
*/
stages: Map<string, ChartableStage>;
/**
* All statuses found deeper in the stage tree. A stage might have been
* _aborted_ although the build actually _failed_, e.g.
*/
statuses: Array<string>;
}
@@ -48,6 +48,7 @@ import {
FilterStatusType,
statusTypes,
} from '../apis/types';
import { ChartableStagesAnalysis } from '../charts/types';
import { ButtonSwitch, SwitchValue } from './button-switch';
import { Toggle } from './toggle';
import { DurationSlider } from './duration-slider';
@@ -93,8 +94,8 @@ export type StatusSelection = FilterStatusType;
export interface ChartFilter {
fromDate: Date;
toDate: Date;
branch: BranchSelection;
status: Array<StatusSelection>;
branch: string;
status: Array<string>;
}
export function getDefaultChartFilter(
@@ -175,6 +176,8 @@ const chartTypeValues: Array<SwitchValue<ChartType>> = [
];
export interface ChartFiltersProps {
analysis?: ChartableStagesAnalysis;
cicdConfiguration: CicdConfiguration;
initialFetchFilter: ChartFilter;
currentFetchFilter?: ChartFilter;
@@ -191,6 +194,7 @@ interface InternalRef {
export function ChartFilters(props: ChartFiltersProps) {
const {
analysis,
cicdConfiguration,
initialFetchFilter,
currentFetchFilter,
@@ -325,6 +329,8 @@ export function ChartFilters(props: ChartFiltersProps) {
});
}, [toDate, fromDate, branch, selectedStatus, updateFetchFilter]);
const inrefferedStatuses = analysis?.statuses ?? selectedStatus;
return (
<MuiPickersUtilsProvider utils={LuxonUtils}>
<Card className={classes.rootCard}>
@@ -395,7 +401,7 @@ export function ChartFilters(props: ChartFiltersProps) {
>
Branch
</Typography>
<ButtonSwitch<BranchSelection>
<ButtonSwitch<string>
values={branchValues}
selection={branch}
onChange={setBranch}
@@ -406,7 +412,7 @@ export function ChartFilters(props: ChartFiltersProps) {
>
Status
</Typography>
<ButtonSwitch<StatusSelection>
<ButtonSwitch<string>
values={statusValues}
multi
vertical
@@ -469,12 +475,12 @@ export function ChartFilters(props: ChartFiltersProps) {
>
Chart styles
</Typography>
{currentFetchFilter?.status.map(status => (
<Grid container spacing={0}>
{inrefferedStatuses.map(status => (
<Grid key={status} container spacing={0}>
<Grid item>
<ButtonSwitch<ChartType>
values={chartTypeValues}
selection={viewOptions.chartTypes[status]}
selection={viewOptions.chartTypes[status as FilterStatusType]}
onChange={setChartTypeSpecific[status]}
multi
/>
@@ -16,6 +16,7 @@
import React, { CSSProperties } from 'react';
import { DateTime, Duration } from 'luxon';
import { capitalize } from 'lodash';
const infoText: CSSProperties = { color: 'InfoText' };
@@ -79,7 +80,7 @@ export function tickFormatterY(duration: number) {
export function tooltipValueFormatter(durationOrCount: number, name: string) {
return [
<span style={infoText}>
{name}:{' '}
{capitalize(name)}:{' '}
{name.endsWith(' count')
? durationOrCount
: formatDuration(durationOrCount)}
+12 -6
View File
@@ -35,7 +35,11 @@ import {
getDefaultViewOptions,
ViewOptions,
} from './components/chart-filters';
import { CicdConfiguration } from './apis';
import {
CicdConfiguration,
FilterStatusType,
FilterBranchType,
} from './apis/types';
import { cleanupBuildTree } from './utils/stage-names';
import { renderFallbacks, useAsyncChain } from './components/progress';
import { sortFilterStatusType } from './utils/api';
@@ -68,7 +72,7 @@ function endOfDay(date: Date) {
function cleanChartFilter(filter: ChartFilter): ChartFilter {
return {
...filter,
status: sortFilterStatusType(filter.status),
status: sortFilterStatusType(filter.status as FilterStatusType[]),
};
}
@@ -104,8 +108,8 @@ function CicdCharts(props: CicdChartsProps) {
entity,
timeFrom: startOfDay(fetchedChartData.chartFilter.fromDate),
timeTo: endOfDay(fetchedChartData.chartFilter.toDate),
filterStatus: fetchedChartData.chartFilter.status,
filterType: fetchedChartData.chartFilter.branch,
filterStatus: fetchedChartData.chartFilter.status as FilterStatusType[],
filterType: fetchedChartData.chartFilter.branch as FilterBranchType,
};
}, [entity, fetchedChartData]);
@@ -156,6 +160,7 @@ function CicdCharts(props: CicdChartsProps) {
<Grid container>
<Grid item lg={2} className={classes.pane}>
<ChartFilters
analysis={chartableStagesState.value}
cicdConfiguration={cicdConfiguration}
initialFetchFilter={chartFilter}
currentFetchFilter={fetchedChartData.chartFilter}
@@ -168,8 +173,9 @@ function CicdCharts(props: CicdChartsProps) {
<Grid item xs={12} lg={10} className={classes.pane}>
{renderFallbacks(chartableStagesState, chartableStages => (
<>
{!statisticsState.value?.builds.length ? null : (
<StatusChart builds={statisticsState.value?.builds} />
{!statisticsState.value?.builds?.length ||
!chartableStagesState.value?.daily?.values?.length ? null : (
<StatusChart analysis={chartableStagesState.value} />
)}
<StageChart
stage={chartableStages.total}