Merge pull request #14506 from stichiboi/dags-run-3
Apache airflow add dag runs feature
This commit is contained in:
@@ -0,0 +1,8 @@
|
||||
---
|
||||
'@backstage/plugin-apache-airflow': patch
|
||||
---
|
||||
|
||||
1. Added a new column in the table to quickly view the latest DAG runs, plus a link to it if you want to have a deeper look.
|
||||
2. Table columns are togglable
|
||||
3. Set hidden columns
|
||||
4. Fixed bug with turning on/off the DAGs
|
||||
@@ -352,6 +352,7 @@ theia
|
||||
thumbsup
|
||||
todo
|
||||
todos
|
||||
togglable
|
||||
tolerations
|
||||
Tolerations
|
||||
toolchain
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
|
||||
import { createApiRef, DiscoveryApi } from '@backstage/core-plugin-api';
|
||||
import { Dag, InstanceStatus, InstanceVersion } from './types';
|
||||
import { DagRun } from './types/Dags';
|
||||
|
||||
export const apacheAirflowApiRef = createApiRef<ApacheAirflowApi>({
|
||||
id: 'plugin.apacheairflow.service',
|
||||
@@ -27,6 +28,10 @@ export type ApacheAirflowApi = {
|
||||
listDags(options?: { objectsPerRequest: number }): Promise<Dag[]>;
|
||||
getDags(dagIds: string[]): Promise<{ dags: Dag[]; dagsNotFound: string[] }>;
|
||||
updateDag(dagId: string, isPaused: boolean): Promise<any>;
|
||||
getDagRuns(
|
||||
dagId: string,
|
||||
options?: { objectsPerRequest?: number; limit?: number },
|
||||
): Promise<DagRun[]>;
|
||||
getInstanceStatus(): Promise<InstanceStatus>;
|
||||
getInstanceVersion(): Promise<InstanceVersion>;
|
||||
};
|
||||
|
||||
@@ -20,6 +20,7 @@ import { rest } from 'msw';
|
||||
import { setupServer } from 'msw/node';
|
||||
import { ApacheAirflowClient } from './index';
|
||||
import { Dag } from './types';
|
||||
import { DagRun } from './types/Dags';
|
||||
|
||||
const server = setupServer();
|
||||
|
||||
@@ -66,12 +67,45 @@ const dags: Dag[] = [
|
||||
},
|
||||
];
|
||||
|
||||
const dagRuns: DagRun[] = [
|
||||
{
|
||||
dag_run_id: 'mock dag run 1',
|
||||
dag_id: 'mock_dag_1',
|
||||
logical_date: '2022-05-27T11:25:23.251274+00:00',
|
||||
start_date: '2022-05-27T11:25:23.251274+00:00',
|
||||
end_date: '2022-05-27T11:25:23.251274+00:00',
|
||||
state: 'success',
|
||||
external_trigger: true,
|
||||
conf: {},
|
||||
},
|
||||
{
|
||||
dag_run_id: 'mock dag run 2',
|
||||
dag_id: 'mock_dag_2',
|
||||
logical_date: '2022-05-27T11:25:23.251274+00:00',
|
||||
start_date: '2022-05-27T11:25:23.251274+00:00',
|
||||
end_date: '2022-05-27T11:25:23.251274+00:00',
|
||||
state: 'running',
|
||||
external_trigger: true,
|
||||
conf: {},
|
||||
},
|
||||
{
|
||||
dag_run_id: 'mock dag run 3',
|
||||
dag_id: 'mock_dag_1',
|
||||
logical_date: '2022-05-27T11:25:23.251274+00:00',
|
||||
start_date: '2022-05-27T11:25:23.251274+00:00',
|
||||
end_date: '2022-05-27T11:25:23.251274+00:00',
|
||||
state: 'failed',
|
||||
external_trigger: true,
|
||||
conf: {},
|
||||
},
|
||||
];
|
||||
|
||||
describe('ApacheAirflowClient', () => {
|
||||
setupRequestMockHandlers(server);
|
||||
|
||||
const mockBaseUrl = 'http://backstage:9191/api/proxy';
|
||||
const discoveryApi = UrlPatternDiscovery.compile(mockBaseUrl);
|
||||
|
||||
let client: ApacheAirflowClient;
|
||||
const setupHandlers = () => {
|
||||
server.use(
|
||||
rest.get(`${mockBaseUrl}/airflow/dags`, (req, res, ctx) => {
|
||||
@@ -116,41 +150,61 @@ describe('ApacheAirflowClient', () => {
|
||||
return res(ctx.status(404));
|
||||
}),
|
||||
|
||||
rest.patch(`${mockBaseUrl}/airflow/dags/:dag_id`, (req, res, ctx) => {
|
||||
const { dag_id } = req.params;
|
||||
const body = JSON.parse(req.body as string);
|
||||
expect(body.is_paused).toBeDefined();
|
||||
return res(
|
||||
ctx.json({
|
||||
dag_id: dag_id,
|
||||
root_dag_id: 'string',
|
||||
is_paused: body.is_paused,
|
||||
is_active: true,
|
||||
is_subdag: true,
|
||||
fileloc: 'string',
|
||||
file_token: 'string',
|
||||
owners: ['string'],
|
||||
description: 'string',
|
||||
schedule_interval: {
|
||||
__type: 'string',
|
||||
days: 0,
|
||||
seconds: 0,
|
||||
microseconds: 0,
|
||||
},
|
||||
tags: [{}],
|
||||
}),
|
||||
);
|
||||
}),
|
||||
rest.get(
|
||||
`${mockBaseUrl}/airflow/dags/:dag_id/dagRuns`,
|
||||
(req, res, ctx) => {
|
||||
const { dag_id } = req.params;
|
||||
const runs = dagRuns.filter(run => run.dag_id === dag_id);
|
||||
// event if the dag_id is invalid, airflow returns a valid response (with 0 dag runs)
|
||||
return res(
|
||||
ctx.json({
|
||||
dag_runs: runs,
|
||||
total_entries: runs.length,
|
||||
}),
|
||||
);
|
||||
},
|
||||
),
|
||||
|
||||
rest.patch(
|
||||
`${mockBaseUrl}/airflow/dags/:dag_id`,
|
||||
async (req, res, ctx) => {
|
||||
const { dag_id } = req.params;
|
||||
const body = JSON.parse(await req.text());
|
||||
expect(body.is_paused).toBeDefined();
|
||||
return res(
|
||||
ctx.json({
|
||||
dag_id: dag_id,
|
||||
root_dag_id: 'string',
|
||||
is_paused: body.is_paused,
|
||||
is_active: true,
|
||||
is_subdag: true,
|
||||
fileloc: 'string',
|
||||
file_token: 'string',
|
||||
owners: ['string'],
|
||||
description: 'string',
|
||||
schedule_interval: {
|
||||
__type: 'string',
|
||||
days: 0,
|
||||
seconds: 0,
|
||||
microseconds: 0,
|
||||
},
|
||||
tags: [{}],
|
||||
}),
|
||||
);
|
||||
},
|
||||
),
|
||||
);
|
||||
};
|
||||
|
||||
it('list dags should return all dags with emulated pagination', async () => {
|
||||
beforeEach(() => {
|
||||
setupHandlers();
|
||||
const client = new ApacheAirflowClient({
|
||||
client = new ApacheAirflowClient({
|
||||
discoveryApi: discoveryApi,
|
||||
baseUrl: 'localhost:8080/',
|
||||
});
|
||||
});
|
||||
|
||||
it('list dags should return all dags with emulated pagination', async () => {
|
||||
// call with limit of 2, to force two paginations in requesting all dags
|
||||
// as our mocked response has 4 total entries
|
||||
const responseDags = await client.listDags({ objectsPerRequest: 2 });
|
||||
@@ -159,11 +213,6 @@ describe('ApacheAirflowClient', () => {
|
||||
});
|
||||
|
||||
it('update dag should return dag information with updated paused attribute', async () => {
|
||||
setupHandlers();
|
||||
const client = new ApacheAirflowClient({
|
||||
discoveryApi: discoveryApi,
|
||||
baseUrl: 'localhost:8080/',
|
||||
});
|
||||
const dagId = 'mock_dag_1';
|
||||
const response: Dag = await client.updateDag(dagId, true);
|
||||
expect(response.dag_id).toEqual(dagId);
|
||||
@@ -171,11 +220,6 @@ describe('ApacheAirflowClient', () => {
|
||||
});
|
||||
|
||||
it('get only some dags', async () => {
|
||||
setupHandlers();
|
||||
const client = new ApacheAirflowClient({
|
||||
discoveryApi: discoveryApi,
|
||||
baseUrl: 'localhost:8080/',
|
||||
});
|
||||
const dagIds = ['mock_dag_1', 'mock_dag_3'];
|
||||
const response = await client.getDags(dagIds);
|
||||
expect(response.dags.length).toEqual(dagIds.length);
|
||||
@@ -186,11 +230,6 @@ describe('ApacheAirflowClient', () => {
|
||||
});
|
||||
|
||||
it('get dags but ignore NOT FOUND errors', async () => {
|
||||
setupHandlers();
|
||||
const client = new ApacheAirflowClient({
|
||||
discoveryApi: discoveryApi,
|
||||
baseUrl: 'localhost:8080/',
|
||||
});
|
||||
const dagIds = ['mock_dag_1', 'a-random-DAG-id'];
|
||||
const response = await client.getDags(dagIds);
|
||||
expect(response.dags.length).toEqual(1);
|
||||
@@ -198,4 +237,11 @@ describe('ApacheAirflowClient', () => {
|
||||
expect(response.dagsNotFound.length).toEqual(1);
|
||||
expect(response.dagsNotFound[0]).toEqual('a-random-DAG-id');
|
||||
});
|
||||
|
||||
it('should get dag runs', async () => {
|
||||
const dagId = 'mock_dag_1';
|
||||
const response = await client.getDagRuns(dagId);
|
||||
expect(response.length).toEqual(2);
|
||||
response.forEach(run => expect(run.dag_id).toEqual(dagId));
|
||||
});
|
||||
});
|
||||
|
||||
@@ -25,6 +25,7 @@ import {
|
||||
InstanceVersion,
|
||||
ListDagsParams,
|
||||
} from './types';
|
||||
import { DagRun } from './types/Dags';
|
||||
|
||||
export class ApacheAirflowClient implements ApacheAirflowApi {
|
||||
discoveryApi: DiscoveryApi;
|
||||
@@ -97,11 +98,56 @@ export class ApacheAirflowClient implements ApacheAirflowApi {
|
||||
async updateDag(dagId: string, isPaused: boolean): Promise<Dag> {
|
||||
const init = {
|
||||
method: 'PATCH',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({ is_paused: isPaused }),
|
||||
};
|
||||
return await this.fetch<Dag>(`/dags/${dagId}`, init);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the latest DAG Runs of a specific DAG.
|
||||
* The DAG runs are ordered by start date
|
||||
*
|
||||
* @remarks
|
||||
*
|
||||
* The "limit" option means the maximum number of DAG runs to return
|
||||
* "objectsPerRequest" means the maximum number of DAG runs to be taken per fetch request.
|
||||
* This is just to make sure the response payloads are not too big
|
||||
*/
|
||||
async getDagRuns(
|
||||
dagId: string,
|
||||
options = { objectsPerRequest: 100, limit: 5 },
|
||||
): Promise<DagRun[]> {
|
||||
const dagRuns: DagRun[] = [];
|
||||
const searchParams: ListDagsParams = {
|
||||
limit: Math.min(options.objectsPerRequest || 100, options.limit || 5),
|
||||
offset: 0,
|
||||
order_by: '-start_date',
|
||||
};
|
||||
|
||||
for (;;) {
|
||||
const response = await this.fetch<{
|
||||
dag_runs: DagRun[];
|
||||
total_entries: number;
|
||||
}>(`/dags/${dagId}/dagRuns?${qs.stringify(searchParams)}`);
|
||||
dagRuns.push(...response.dag_runs);
|
||||
|
||||
if (response.dag_runs.length < searchParams.limit!) {
|
||||
break;
|
||||
}
|
||||
if (
|
||||
dagRuns.length >= response.total_entries ||
|
||||
dagRuns.length >= options.limit
|
||||
) {
|
||||
break;
|
||||
}
|
||||
searchParams.offset! += response.dag_runs.length;
|
||||
}
|
||||
return dagRuns;
|
||||
}
|
||||
|
||||
async getInstanceStatus(): Promise<InstanceStatus> {
|
||||
return await this.fetch<InstanceStatus>('/health');
|
||||
}
|
||||
|
||||
@@ -44,6 +44,17 @@ export interface Dag {
|
||||
tags: Tag[];
|
||||
}
|
||||
|
||||
export interface DagRun {
|
||||
dag_run_id: string;
|
||||
dag_id: string;
|
||||
logical_date: string;
|
||||
start_date: string;
|
||||
end_date: string;
|
||||
state: 'queued' | 'running' | 'success' | 'failed';
|
||||
external_trigger: boolean;
|
||||
conf: {};
|
||||
}
|
||||
|
||||
export interface TimeDelta {
|
||||
__type: 'TimeDelta';
|
||||
days: number;
|
||||
|
||||
@@ -22,7 +22,7 @@ import {
|
||||
TableColumn,
|
||||
WarningPanel,
|
||||
} from '@backstage/core-components';
|
||||
import { useApi } from '@backstage/core-plugin-api';
|
||||
import { storageApiRef, useApi } from '@backstage/core-plugin-api';
|
||||
import Box from '@material-ui/core/Box';
|
||||
import Chip from '@material-ui/core/Chip';
|
||||
import IconButton from '@material-ui/core/IconButton';
|
||||
@@ -31,101 +31,151 @@ import Tooltip from '@material-ui/core/Tooltip';
|
||||
import Typography from '@material-ui/core/Typography';
|
||||
import OpenInBrowserIcon from '@material-ui/icons/OpenInBrowser';
|
||||
import Alert from '@material-ui/lab/Alert';
|
||||
import React from 'react';
|
||||
import React, { useEffect, useState } from 'react';
|
||||
import useAsync from 'react-use/lib/useAsync';
|
||||
import { apacheAirflowApiRef } from '../../api';
|
||||
import { Dag } from '../../api/types';
|
||||
import { ScheduleIntervalLabel } from '../ScheduleIntervalLabel';
|
||||
import { LatestDagRunsStatus } from '../LatestDagRunsStatus';
|
||||
|
||||
type DagTableRow = Dag & {
|
||||
id: string;
|
||||
dagUrl: string;
|
||||
};
|
||||
|
||||
const columns: TableColumn[] = [
|
||||
{
|
||||
title: 'Paused',
|
||||
field: 'is_paused',
|
||||
render: (row: Partial<DagTableRow>) => (
|
||||
<Tooltip title="Pause/Unpause DAG">
|
||||
<Switch checked={!row.is_paused} disabled />
|
||||
</Tooltip>
|
||||
),
|
||||
width: '5%',
|
||||
},
|
||||
{
|
||||
title: 'DAG',
|
||||
field: 'id',
|
||||
render: (row: Partial<DagTableRow>) => (
|
||||
<div>
|
||||
<Typography variant="subtitle2" gutterBottom noWrap>
|
||||
{row.id}
|
||||
</Typography>
|
||||
<Box display="flex" alignItems="center">
|
||||
{row.tags?.map((tag, ix) => (
|
||||
<Chip label={tag.name} key={ix} size="small" />
|
||||
))}
|
||||
</Box>
|
||||
</div>
|
||||
),
|
||||
width: '60%',
|
||||
},
|
||||
{
|
||||
title: 'Owner',
|
||||
field: 'owners',
|
||||
render: (row: Partial<DagTableRow>) => (
|
||||
<Box display="flex" alignItems="center">
|
||||
{row.owners?.map((owner, ix) => (
|
||||
<Chip label={owner} key={ix} size="small" />
|
||||
))}
|
||||
</Box>
|
||||
),
|
||||
width: '10%',
|
||||
},
|
||||
{
|
||||
title: 'Active',
|
||||
render: (row: Partial<DagTableRow>) => (
|
||||
<Box display="flex" alignItems="center">
|
||||
{row.is_active ? <StatusOK /> : <StatusError />}
|
||||
<Typography variant="body2">
|
||||
{row.is_active ? 'Active' : 'Inactive'}
|
||||
</Typography>
|
||||
</Box>
|
||||
),
|
||||
width: '10%',
|
||||
},
|
||||
{
|
||||
title: 'Schedule',
|
||||
render: (row: Partial<DagTableRow>) => (
|
||||
<ScheduleIntervalLabel interval={row.schedule_interval} />
|
||||
),
|
||||
width: '10%',
|
||||
},
|
||||
{
|
||||
title: 'Link',
|
||||
field: 'dagUrl',
|
||||
render: (row: Partial<DagTableRow>) => (
|
||||
<a href={row.dagUrl}>
|
||||
<IconButton aria-label="details">
|
||||
<OpenInBrowserIcon />
|
||||
</IconButton>
|
||||
</a>
|
||||
),
|
||||
width: '5%',
|
||||
},
|
||||
];
|
||||
|
||||
type DenseTableProps = {
|
||||
dags: Dag[];
|
||||
rowClick: Function;
|
||||
};
|
||||
|
||||
export const DenseTable = ({ dags }: DenseTableProps) => {
|
||||
export const DenseTable = ({ dags, rowClick }: DenseTableProps) => {
|
||||
const storage = useApi(storageApiRef);
|
||||
const hiddenColumnsKey = 'dag-table-hidden-columns';
|
||||
const [hiddenColumns, setHiddenColumns] = useState<string[]>([]);
|
||||
|
||||
useEffect(() => {
|
||||
const hiddenState = storage.snapshot(hiddenColumnsKey);
|
||||
if (hiddenState.presence === 'present') {
|
||||
setHiddenColumns(hiddenState.value as string[]);
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, []);
|
||||
|
||||
const columns: TableColumn[] = [
|
||||
{
|
||||
title: 'Paused',
|
||||
field: 'is_paused',
|
||||
render: (row: Partial<DagTableRow>) => (
|
||||
<Tooltip title="Pause/Unpause DAG">
|
||||
<Switch checked={!row.is_paused} />
|
||||
</Tooltip>
|
||||
),
|
||||
width: '5%',
|
||||
hidden: hiddenColumns.some(field => field === 'is_paused'),
|
||||
},
|
||||
{
|
||||
title: 'DAG',
|
||||
field: 'id',
|
||||
render: (row: Partial<DagTableRow>) => (
|
||||
<div>
|
||||
<Typography variant="subtitle2" gutterBottom noWrap>
|
||||
{row.id}
|
||||
</Typography>
|
||||
<Box display="flex" alignItems="center">
|
||||
{row.tags?.map((tag, ix) => (
|
||||
<Chip label={tag.name} key={ix} size="small" />
|
||||
))}
|
||||
</Box>
|
||||
</div>
|
||||
),
|
||||
width: '50%',
|
||||
disableClick: true,
|
||||
hidden: hiddenColumns.some(field => field === 'id'),
|
||||
},
|
||||
{
|
||||
title: 'Runs',
|
||||
field: 'runs',
|
||||
render: (row: Partial<DagTableRow>) => (
|
||||
<LatestDagRunsStatus dagId={row.dag_id || ''} />
|
||||
),
|
||||
width: '10%',
|
||||
disableClick: true,
|
||||
hidden: hiddenColumns.some(field => field === 'runs'),
|
||||
},
|
||||
{
|
||||
title: 'Owner',
|
||||
field: 'owners',
|
||||
render: (row: Partial<DagTableRow>) => (
|
||||
<Box display="flex" alignItems="center">
|
||||
{row.owners?.map((owner, ix) => (
|
||||
<Chip label={owner} key={ix} size="small" />
|
||||
))}
|
||||
</Box>
|
||||
),
|
||||
width: '10%',
|
||||
disableClick: true,
|
||||
hidden: hiddenColumns.some(field => field === 'owners'),
|
||||
},
|
||||
{
|
||||
title: 'Active',
|
||||
field: 'active',
|
||||
render: (row: Partial<DagTableRow>) => (
|
||||
<Box display="flex" alignItems="center">
|
||||
{row.is_active ? <StatusOK /> : <StatusError />}
|
||||
<Typography variant="body2">
|
||||
{row.is_active ? 'Active' : 'Inactive'}
|
||||
</Typography>
|
||||
</Box>
|
||||
),
|
||||
width: '10%',
|
||||
disableClick: true,
|
||||
hidden: hiddenColumns.some(field => field === 'active'),
|
||||
},
|
||||
{
|
||||
title: 'Schedule',
|
||||
field: 'schedule',
|
||||
render: (row: Partial<DagTableRow>) => (
|
||||
<ScheduleIntervalLabel interval={row.schedule_interval} />
|
||||
),
|
||||
width: '10%',
|
||||
disableClick: true,
|
||||
hidden: hiddenColumns.some(field => field === 'schedule'),
|
||||
},
|
||||
{
|
||||
title: 'Link',
|
||||
field: 'dagUrl',
|
||||
render: (row: Partial<DagTableRow>) => (
|
||||
<a href={row.dagUrl}>
|
||||
<IconButton aria-label="details">
|
||||
<OpenInBrowserIcon />
|
||||
</IconButton>
|
||||
</a>
|
||||
),
|
||||
width: '5%',
|
||||
disableClick: true,
|
||||
hidden: hiddenColumns.some(field => field === 'dagUrl'),
|
||||
},
|
||||
];
|
||||
|
||||
return (
|
||||
<Table
|
||||
title="DAGs"
|
||||
options={{ pageSize: 5 }}
|
||||
options={{ pageSize: 5, columnsButton: true }}
|
||||
columns={columns}
|
||||
data={dags}
|
||||
onRowClick={(_event, rowData) => rowClick(rowData)}
|
||||
onChangeColumnHidden={(column, hidden) => {
|
||||
if (column.field) {
|
||||
let newHiddenColumns: string[];
|
||||
if (hidden) {
|
||||
newHiddenColumns = hiddenColumns.concat(column.field);
|
||||
} else {
|
||||
newHiddenColumns = hiddenColumns.filter(v => v !== column.field);
|
||||
}
|
||||
setHiddenColumns(newHiddenColumns);
|
||||
storage.set(hiddenColumnsKey, newHiddenColumns);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
);
|
||||
};
|
||||
@@ -136,27 +186,54 @@ type DagTableComponentProps = {
|
||||
|
||||
export const DagTableComponent = (props: DagTableComponentProps) => {
|
||||
const { dagIds } = props;
|
||||
const [dagsData, setDagsData] = useState<DagTableRow[]>([]);
|
||||
const apiClient = useApi(apacheAirflowApiRef);
|
||||
|
||||
const { value, loading, error } = useAsync(async (): Promise<Dag[]> => {
|
||||
const updatePaused = async (rowData: Dag): Promise<Dag> => {
|
||||
const newDag = await apiClient.updateDag(
|
||||
rowData.dag_id,
|
||||
!rowData.is_paused,
|
||||
);
|
||||
|
||||
const newDags = dagsData.map(el => {
|
||||
if (el.dag_id === newDag.dag_id) {
|
||||
return { ...el, is_paused: newDag.is_paused };
|
||||
}
|
||||
return el;
|
||||
});
|
||||
|
||||
setDagsData(newDags);
|
||||
return newDag;
|
||||
};
|
||||
|
||||
const { value, loading, error } = useAsync(async (): Promise<
|
||||
DagTableRow[]
|
||||
> => {
|
||||
let dags: Dag[] = [];
|
||||
if (dagIds) {
|
||||
const { dags } = await apiClient.getDags(dagIds);
|
||||
return dags;
|
||||
dags = (await apiClient.getDags(dagIds)).dags;
|
||||
} else {
|
||||
dags = await apiClient.listDags();
|
||||
}
|
||||
return await apiClient.listDags();
|
||||
return dags.map(el => ({
|
||||
...el,
|
||||
id: el.dag_id, // table records require `id` attribute
|
||||
dagUrl: `${apiClient.baseUrl}dag_details?dag_id=${el.dag_id}`, // construct path to DAG using `baseUrl`
|
||||
}));
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (value) {
|
||||
setDagsData(value);
|
||||
}
|
||||
}, [value]);
|
||||
|
||||
if (loading) {
|
||||
return <Progress />;
|
||||
} else if (error) {
|
||||
return <Alert severity="error">{error.message}</Alert>;
|
||||
}
|
||||
|
||||
const data = value?.map(el => ({
|
||||
...el,
|
||||
id: el.dag_id, // table records require `id` attribute
|
||||
dagUrl: `${apiClient.baseUrl}dag_details?dag_id=${el.dag_id}`, // construct path to DAG using `baseUrl`
|
||||
}));
|
||||
const dagsNotFound =
|
||||
dagIds && value
|
||||
? dagIds.filter(id => !value.find(d => d.dag_id === id))
|
||||
@@ -172,7 +249,7 @@ export const DagTableComponent = (props: DagTableComponentProps) => {
|
||||
) : (
|
||||
''
|
||||
)}
|
||||
<DenseTable dags={data || []} />
|
||||
<DenseTable dags={dagsData} rowClick={updatePaused} />
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import { ApacheAirflowApi, apacheAirflowApiRef } from '../../api';
|
||||
import { DagRun } from '../../api/types/Dags';
|
||||
import { renderInTestApp, TestApiProvider } from '@backstage/test-utils';
|
||||
import React from 'react';
|
||||
import { LatestDagRunsStatus } from './LatestDagRunsStatus';
|
||||
|
||||
describe('LatestDagRunsStatus', () => {
|
||||
const baseDagRun: Partial<DagRun> = {
|
||||
dag_run_id: 'mock dag run 1',
|
||||
dag_id: 'mock_dag_1',
|
||||
logical_date: '2022-05-27T11:25:23.251274+00:00',
|
||||
start_date: '2022-05-27T11:25:23.251274+00:00',
|
||||
end_date: '2022-05-27T11:25:23.251274+00:00',
|
||||
state: 'success',
|
||||
external_trigger: true,
|
||||
conf: {},
|
||||
};
|
||||
const mockApi: jest.Mocked<ApacheAirflowApi> = {
|
||||
getDagRuns: jest.fn().mockResolvedValue([
|
||||
baseDagRun,
|
||||
{
|
||||
...baseDagRun,
|
||||
dag_run_id: 'mock dag run 2',
|
||||
state: 'running',
|
||||
},
|
||||
{
|
||||
...baseDagRun,
|
||||
dag_run_id: 'mock dag run 3',
|
||||
state: 'failed',
|
||||
},
|
||||
{
|
||||
...baseDagRun,
|
||||
dag_run_id: 'mock dag run 3',
|
||||
state: 'queued',
|
||||
},
|
||||
] as DagRun[]),
|
||||
} as any;
|
||||
|
||||
it('should render the status of mock dag 1', async () => {
|
||||
const dagId = 'mock_dag_1';
|
||||
|
||||
const { getByLabelText } = await renderInTestApp(
|
||||
<TestApiProvider apis={[[apacheAirflowApiRef, mockApi]]}>
|
||||
<LatestDagRunsStatus dagId={dagId} />
|
||||
</TestApiProvider>,
|
||||
);
|
||||
expect(getByLabelText('Status ok')).toBeInTheDocument();
|
||||
expect(getByLabelText('Status running')).toBeInTheDocument();
|
||||
expect(getByLabelText('Status error')).toBeInTheDocument();
|
||||
expect(getByLabelText('Status pending')).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,161 @@
|
||||
/*
|
||||
* Copyright 2022 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import React from 'react';
|
||||
import { apacheAirflowApiRef } from '../../api';
|
||||
import useAsync from 'react-use/lib/useAsync';
|
||||
import { DagRun } from '../../api/types/Dags';
|
||||
import { useApi } from '@backstage/core-plugin-api';
|
||||
import {
|
||||
Box,
|
||||
Button,
|
||||
CircularProgress,
|
||||
List,
|
||||
ListItem,
|
||||
ListItemIcon,
|
||||
makeStyles,
|
||||
Tooltip,
|
||||
Typography,
|
||||
} from '@material-ui/core';
|
||||
import {
|
||||
Link,
|
||||
StatusError,
|
||||
StatusOK,
|
||||
StatusPending,
|
||||
StatusRunning,
|
||||
} from '@backstage/core-components';
|
||||
import DirectionsRun from '@material-ui/icons/DirectionsRun';
|
||||
import Check from '@material-ui/icons/Check';
|
||||
import CalendarToday from '@material-ui/icons/CalendarToday';
|
||||
import qs from 'qs';
|
||||
import AccountTree from '@material-ui/icons/AccountTree';
|
||||
|
||||
interface LatestDagRunsStatusProps {
|
||||
dagId: string;
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
const useStyles = makeStyles(() => ({
|
||||
noMaxWidth: {
|
||||
maxWidth: 'none',
|
||||
},
|
||||
}));
|
||||
|
||||
const DagRunTooltip = ({
|
||||
dagRun,
|
||||
graphUrl,
|
||||
}: {
|
||||
dagRun: DagRun;
|
||||
graphUrl: string;
|
||||
}) => {
|
||||
return (
|
||||
<List>
|
||||
<ListItem>
|
||||
<ListItemIcon aria-label="DAG Run ID">
|
||||
<DirectionsRun />
|
||||
</ListItemIcon>
|
||||
<Typography>{dagRun.dag_run_id}</Typography>
|
||||
</ListItem>
|
||||
<ListItem>
|
||||
<ListItemIcon aria-label="DAG Start Date">
|
||||
<CalendarToday />
|
||||
</ListItemIcon>
|
||||
<Typography>{new Date(dagRun.start_date).toLocaleString()}</Typography>
|
||||
</ListItem>
|
||||
<ListItem>
|
||||
<ListItemIcon aria-label="DAG End Date">
|
||||
<Check />
|
||||
</ListItemIcon>
|
||||
<Typography>
|
||||
{dagRun.end_date ? new Date(dagRun.end_date).toLocaleString() : '-'}
|
||||
</Typography>
|
||||
</ListItem>
|
||||
<ListItem>
|
||||
<Button
|
||||
startIcon={<AccountTree />}
|
||||
aria-label="Link To Detail"
|
||||
color="primary"
|
||||
variant="outlined"
|
||||
>
|
||||
<Link to={graphUrl} color="inherit">
|
||||
Graph
|
||||
</Link>
|
||||
</Button>
|
||||
</ListItem>
|
||||
</List>
|
||||
);
|
||||
};
|
||||
|
||||
export const LatestDagRunsStatus = ({
|
||||
dagId,
|
||||
limit = 5,
|
||||
}: LatestDagRunsStatusProps) => {
|
||||
const classes = useStyles();
|
||||
const apiClient = useApi(apacheAirflowApiRef);
|
||||
const { value, loading, error } = useAsync(
|
||||
async (): Promise<DagRun[]> => await apiClient.getDagRuns(dagId, { limit }),
|
||||
[dagId, limit],
|
||||
);
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
<Box>
|
||||
<CircularProgress />
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
if (error) {
|
||||
return <Typography>Can't get dag runs</Typography>;
|
||||
}
|
||||
|
||||
const statusDots: JSX.Element[] | undefined = value?.map(dagRun => {
|
||||
function status() {
|
||||
switch (dagRun.state) {
|
||||
case 'success':
|
||||
return <StatusOK />;
|
||||
case 'failed':
|
||||
return <StatusError />;
|
||||
case 'running':
|
||||
return <StatusRunning />;
|
||||
case 'queued':
|
||||
return <StatusPending />;
|
||||
default:
|
||||
return <Typography>Unrecognized state</Typography>;
|
||||
}
|
||||
}
|
||||
|
||||
const key = dagRun.dag_id + dagRun.dag_run_id;
|
||||
const dagRunParams = {
|
||||
dag_id: dagRun.dag_id,
|
||||
execution_date: dagRun.logical_date,
|
||||
};
|
||||
const graphUrl = `${apiClient.baseUrl}graph?${qs.stringify(dagRunParams)}`;
|
||||
return (
|
||||
<Tooltip
|
||||
title={<DagRunTooltip dagRun={dagRun} graphUrl={graphUrl} />}
|
||||
key={key}
|
||||
classes={{ tooltip: classes.noMaxWidth }}
|
||||
interactive
|
||||
>
|
||||
<Box width="fit-content" component="span">
|
||||
{status()}
|
||||
</Box>
|
||||
</Tooltip>
|
||||
);
|
||||
});
|
||||
|
||||
return <Box>{statusDots}</Box>;
|
||||
};
|
||||
@@ -0,0 +1,17 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
export { LatestDagRunsStatus } from './LatestDagRunsStatus';
|
||||
Reference in New Issue
Block a user