Merge pull request #8348 from cmpadden/plugin-apache-airflow

Apache Airflow Plugin
This commit is contained in:
Ben Lambert
2021-12-09 17:17:40 +01:00
committed by GitHub
36 changed files with 1502 additions and 0 deletions
+14
View File
@@ -0,0 +1,14 @@
---
'@backstage/plugin-apache-airflow': minor
---
Introduces a new plugin for the Apache Airflow workflow management platform.
This implementation has been tested with the Apache Airflow v2 API,
authenticating with basic authentication through the Backstage proxy plugin.
Supported functionality includes:
- Information card of version information of the Airflow instance
- Information card of instance health for the meta-database and scheduler
- Table of DAGs with meta information and status, along with a link to view
details in the Airflow UI
+8
View File
@@ -97,6 +97,11 @@ proxy:
headers:
Authorization: ${ILERT_AUTH_HEADER}
'/airflow':
target: https://your.airflow.instance.com/api/v1
headers:
Authorization: ${AIRFLOW_BASIC_AUTH_HEADER}
organization:
name: My Company
@@ -437,3 +442,6 @@ azureDevOps:
host: dev.azure.com
token: my-token
organization: my-company
apacheAirflow:
baseUrl: https://your.airflow.instance.com
+1
View File
@@ -13,6 +13,7 @@
"@backstage/integration-react": "^0.1.15",
"@backstage/plugin-api-docs": "^0.6.18",
"@backstage/plugin-azure-devops": "^0.1.6",
"@backstage/plugin-apache-airflow": "^0.0.0",
"@backstage/plugin-badges": "^0.2.16",
"@backstage/plugin-catalog": "^0.7.4",
"@backstage/plugin-catalog-graph": "^0.2.3",
+2
View File
@@ -87,6 +87,7 @@ import { providers } from './identityProviders';
import * as plugins from './plugins';
import { techDocsPage } from './components/techdocs/TechDocsPage';
import { ApacheAirflowPage } from '@backstage/plugin-apache-airflow';
const app = createApp({
apis,
@@ -218,6 +219,7 @@ const routes = (
/>
<Route path="/settings" element={<UserSettingsPage />} />
<Route path="/azure-pull-requests" element={<AzurePullRequestsPage />} />
<Route path="/apache-airflow" element={<ApacheAirflowPage />} />
</FlatRoutes>
);
+3
View File
@@ -0,0 +1,3 @@
module.exports = {
extends: [require.resolve('@backstage/cli/config/eslint')],
};
+2
View File
@@ -0,0 +1,2 @@
airflow/
node_modules/
+124
View File
@@ -0,0 +1,124 @@
# Apache Airflow Plugin
Welcome to the apache-airflow plugin!
## Feature Requests & Ideas
- [ ] Add support for running multiple instances of Airflow for monitoring
various deployment stages or business domains. ([Suggested by @JGoldman110](https://github.com/backstage/backstage/issues/735#issuecomment-985063468))
- [ ] Make owner chips in the DAG table clickable, resolving to a user or group
in the entity catalog. ([Suggested by @julioz](https://github.com/backstage/backstage/pull/8348#discussion_r764766295))
## Installation
1. Install the plugin with `yarn` in the root of your Backstage directory
```sh
yarn --cwd packages/app add @backstage/plugin-apache-airflow
```
2. Import and use the plugin extension in `spp/src/App.tsx`
```diff
--- a/packages/app/src/App.tsx
+++ b/packages/app/src/App.tsx
@@ -86,6 +86,7 @@ import { providers } from './identityProviders';
import * as plugins from './plugins';
import { techDocsPage } from './components/techdocs/TechDocsPage';
+import { ApacheAirflowPage } from '@backstage/plugin-apache-airflow';
const app = createApp({
apis,
@@ -203,6 +204,7 @@ const routes = (
element={<CostInsightsLabelDataflowInstructionsPage />}
/>
<Route path="/settings" element={<UserSettingsPage />} />
+ <Route path="/apache-airflow" element={<ApacheAirflowPage />} />
</FlatRoutes>
);
```
## Configuration
For links to the Airflow instance, the `baseUrl` must be defined in
`app-config.yaml`.
```yaml
apacheAirflow:
baseUrl: https://your.airflow.instance.com
```
This plugin uses the Backstage proxy to securely communicate with the Apache
Airflow API. Add the following to your `app-config.yaml` to enable this
configuration:
```yaml
proxy:
'/airflow':
target: https://your.airflow.instance.com/api/v1
headers:
Authorization: ${AIRFLOW_BASIC_AUTH_HEADER}
```
In your production deployment of Backstage, you would also need to ensure that
you've set the `AIRFLOW_BASIC_AUTH_HEADER` environment variable before starting
the backend.
While working locally, you may wish to hard-code your API key in your
`app-config.local.yaml` like this:
```yaml
# app-config.local.yaml
proxy:
'/airflow':
target: http://localhost:8080/api/v1
headers:
Authorization: Basic YWlyZmxvdzphaXJmbG93
```
Where the basic authorization token is the base64 encoding of the username and
password of your instance.
```sh
echo -n "airflow:airflow" | base64 -w0
```
## Development
For local development, you can setup a local Airflow instance for development
purposes by [running Airflow with Docker Compose][2].
To verify that Airflow is running, and the API is functioning as expected, you
can run the following `curl` command:
```sh
curl -X GET \
--user "airflow:airflow" \
localhost:8080/api/v1/dags
```
To run the Backstage proxy, you will have to run start the `example-backend`
plugin.
```sh
yarn workspace example-backend start
```
To verify that the proxy is configured correctly, you can curl the Backstage
proxy endpoint. If using basic authentication, you will have to base64 encode
the username and password:
```sh
curl http://localhost:7007/api/proxy/airflow/dags
```
And finally, to run an instance of this plugin, you can run:
```sh
yarn start
```
[1]: https://airflow.apache.org/docs/apache-airflow/stable/security/api.html
[2]: https://airflow.apache.org/docs/apache-airflow/stable/start/docker.html
[3]: https://airflow.apache.org/docs/apache-airflow/stable/stable-rest-api-ref.html
+25
View File
@@ -0,0 +1,25 @@
## API Report File for "@backstage/plugin-apache-airflow"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
```ts
/// <reference types="react" />
import { BackstagePlugin } from '@backstage/core-plugin-api';
import { RouteRef } from '@backstage/core-plugin-api';
// Warning: (ae-missing-release-tag) "ApacheAirflowPage" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
export const ApacheAirflowPage: () => JSX.Element;
// Warning: (ae-missing-release-tag) "apacheAirflowPlugin" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
export const apacheAirflowPlugin: BackstagePlugin<
{
root: RouteRef<undefined>;
},
{}
>;
```
+26
View File
@@ -0,0 +1,26 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export interface Config {
/** Configurations for the Apache Airflow plugin */
apacheAirflow: {
/**
* The base url of the Apache Airflow installation.
* @visibility frontend
*/
baseUrl: string;
};
}
+28
View File
@@ -0,0 +1,28 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import React from 'react';
import { createDevApp } from '@backstage/dev-utils';
import { apacheAirflowPlugin, ApacheAirflowPage } from '../src/plugin';
createDevApp()
.registerPlugin(apacheAirflowPlugin)
.addPage({
element: <ApacheAirflowPage />,
title: 'Root Page',
path: '/apache-airflow',
})
.render();
+52
View File
@@ -0,0 +1,52 @@
{
"name": "@backstage/plugin-apache-airflow",
"version": "0.0.0",
"main": "src/index.ts",
"types": "src/index.ts",
"license": "Apache-2.0",
"publishConfig": {
"access": "public",
"main": "dist/index.esm.js",
"types": "dist/index.d.ts"
},
"scripts": {
"build": "backstage-cli plugin:build",
"start": "backstage-cli plugin:serve",
"lint": "backstage-cli lint",
"test": "backstage-cli test",
"diff": "backstage-cli plugin:diff",
"prepack": "backstage-cli prepack",
"postpack": "backstage-cli postpack",
"clean": "backstage-cli clean"
},
"dependencies": {
"@backstage/core-components": "^0.8.0",
"@backstage/core-plugin-api": "^0.3.0",
"@material-ui/core": "^4.12.2",
"@material-ui/icons": "^4.9.1",
"@material-ui/lab": "4.0.0-alpha.57",
"cross-fetch": "^3.0.6",
"qs": "^6.10.1",
"react-use": "^17.2.4"
},
"peerDependencies": {
"react": "^16.13.1 || ^17.0.0"
},
"devDependencies": {
"@backstage/cli": "^0.10.1",
"@backstage/core-app-api": "^0.2.0",
"@backstage/dev-utils": "^0.2.14",
"@backstage/test-utils": "^0.1.24",
"@testing-library/jest-dom": "^5.10.1",
"@testing-library/react": "^11.2.5",
"@testing-library/user-event": "^13.1.8",
"@types/jest": "^26.0.7",
"@types/node": "^14.14.32",
"msw": "^0.35.0"
},
"files": [
"dist",
"config.d.ts"
],
"configSchema": "config.d.ts"
}
@@ -0,0 +1,32 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { createApiRef, DiscoveryApi } from '@backstage/core-plugin-api';
import { Dag, InstanceStatus, InstanceVersion } from './types';
export const apacheAirflowApiRef = createApiRef<ApacheAirflowApi>({
id: 'plugin.apacheairflow.service',
description: 'Used by the Apache Airflow plugin to make requests',
});
export type ApacheAirflowApi = {
discoveryApi: DiscoveryApi;
baseUrl: string;
listDags(options?: { objectsPerRequest: number }): Promise<Dag[]>;
updateDag(dagId: string, isPaused: boolean): Promise<any>;
getInstanceStatus(): Promise<InstanceStatus>;
getInstanceVersion(): Promise<InstanceVersion>;
};
@@ -0,0 +1,163 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { UrlPatternDiscovery } from '@backstage/core-app-api';
import { setupRequestMockHandlers } from '@backstage/test-utils';
import { rest } from 'msw';
import { setupServer } from 'msw/node';
import { ApacheAirflowClient } from './index';
import { Dag } from './types';
const server = setupServer();
const dags: Dag[] = [
{
dag_id: 'mock_dag_1',
fileloc: '',
file_token: '',
owners: ['admin'],
schedule_interval: { __type: 'CronExpression', value: '* * 0 0 0' },
tags: [{ name: 'exmaple' }],
},
{
dag_id: 'mock_dag_2',
fileloc: '',
file_token: '',
owners: ['admin'],
schedule_interval: { __type: 'CronExpression', value: '* * 0 0 0' },
tags: [{ name: 'exmaple' }],
},
{
dag_id: 'mock_dag_3',
fileloc: '',
file_token: '',
owners: ['admin'],
schedule_interval: { __type: 'CronExpression', value: '* * 0 0 0' },
tags: [{ name: 'exmaple' }],
},
{
dag_id: 'mock_dag_4',
fileloc: '',
file_token: '',
owners: ['admin'],
schedule_interval: { __type: 'CronExpression', value: '* * 0 0 0' },
tags: [{ name: 'exmaple' }],
},
{
dag_id: 'mock_dag_5',
fileloc: '',
file_token: '',
owners: ['admin'],
schedule_interval: { __type: 'CronExpression', value: '* * 0 0 0' },
tags: [{ name: 'exmaple' }],
},
];
describe('ApacheAirflowClient', () => {
setupRequestMockHandlers(server);
const mockBaseUrl = 'http://backstage:9191/api/proxy';
const discoveryApi = UrlPatternDiscovery.compile(mockBaseUrl);
const setupHandlers = () => {
server.use(
rest.get(`${mockBaseUrl}/airflow/dags`, (req, res, ctx) => {
expect(req.url.searchParams.get('limit')).toBe('2');
// emulate paging to check if everything is requested
if (req.url.searchParams.get('offset') === '0') {
return res(
ctx.json({
dags: dags.slice(0, 2),
total_entries: dags.length,
}),
);
}
// page offset 2
if (req.url.searchParams.get('offset') === '2') {
return res(
ctx.json({
dags: dags.slice(2, 4),
total_entries: dags.length,
}),
);
}
// page offset 4
expect(req.url.searchParams.get('offset')).toBe('4');
return res(
ctx.json({
dags: dags.slice(4),
total_entries: dags.length,
}),
);
}),
rest.patch(`${mockBaseUrl}/airflow/dags/:dag_id`, (req, res, ctx) => {
const { dag_id } = req.params;
const body = JSON.parse(req.body as string);
expect(body.is_paused).toBeDefined();
return res(
ctx.json({
dag_id: dag_id,
root_dag_id: 'string',
is_paused: body.is_paused,
is_active: true,
is_subdag: true,
fileloc: 'string',
file_token: 'string',
owners: ['string'],
description: 'string',
schedule_interval: {
__type: 'string',
days: 0,
seconds: 0,
microseconds: 0,
},
tags: [{}],
}),
);
}),
);
};
it('list dags should return all dags with emulated pagination', async () => {
setupHandlers();
const client = new ApacheAirflowClient({
discoveryApi: discoveryApi,
baseUrl: 'localhost:8080/',
});
// call with limit of 2, to force two paginations in requesting all dags
// as our mocked response has 4 total entries
const responseDags = await client.listDags({ objectsPerRequest: 2 });
expect(responseDags.length).toEqual(5);
expect(responseDags).toEqual(dags);
});
it('update dag should return dag information with updated paused attribute', async () => {
setupHandlers();
const client = new ApacheAirflowClient({
discoveryApi: discoveryApi,
baseUrl: 'localhost:8080/',
});
const dagId = 'mock_dag_1';
const response: Dag = await client.updateDag(dagId, true);
expect(response.dag_id).toEqual(dagId);
expect(response.is_paused).toEqual(true);
});
});
@@ -0,0 +1,100 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { DiscoveryApi } from '@backstage/core-plugin-api';
import fetch from 'cross-fetch';
import qs from 'qs';
import { ApacheAirflowApi } from './ApacheAirflowApi';
import {
Dag,
Dags,
InstanceStatus,
InstanceVersion,
ListDagsParams,
} from './types';
export class ApacheAirflowClient implements ApacheAirflowApi {
discoveryApi: DiscoveryApi;
baseUrl: string;
constructor({
discoveryApi,
baseUrl = 'http://localhost:8080',
}: {
discoveryApi: DiscoveryApi;
baseUrl: string;
}) {
this.discoveryApi = discoveryApi;
this.baseUrl = baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`;
}
/**
* List all DAGs in the Airflow instance
*
* @remarks
* All DAGs with a limit of 100 results per request are returned; this may be
* bogged-down for instances with many DAGs, in which case table pagination
* should be implemented
*
* @param {number} objectsPerRequest records returned per request in pagination
* @returns {Promise<Dag[]>}
*/
async listDags(options = { objectsPerRequest: 100 }): Promise<Dag[]> {
const dags: Dag[] = [];
const searchParams: ListDagsParams = {
limit: options.objectsPerRequest,
offset: 0,
};
for (;;) {
const response = await this.fetch<Dags>(
`/dags?${qs.stringify(searchParams)}`,
);
dags.push(...response.dags);
if (dags.length >= response.total_entries) {
break;
}
if (typeof searchParams.offset !== 'undefined') {
searchParams.offset += options.objectsPerRequest;
}
}
return dags;
}
async updateDag(dagId: string, isPaused: boolean): Promise<Dag> {
const init = {
method: 'PATCH',
body: JSON.stringify({ is_paused: isPaused }),
};
return await this.fetch<Dag>(`/dags/${dagId}`, init);
}
async getInstanceStatus(): Promise<InstanceStatus> {
return await this.fetch<InstanceStatus>('/health');
}
async getInstanceVersion(): Promise<InstanceVersion> {
return await this.fetch<InstanceVersion>('/version');
}
private async fetch<T = any>(input: string, init?: RequestInit): Promise<T> {
const proxyUri = `${await this.discoveryApi.getBaseUrl('proxy')}/airflow`;
const response = await fetch(`${proxyUri}${input}`, init);
if (!response.ok) throw new Error(response.statusText);
return await response.json();
}
}
+25
View File
@@ -0,0 +1,25 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* An API definition and client for integrating with the Apache Airflow v2+
* API
*
* @packageDocumentation
*/
export * from './ApacheAirflowApi';
export * from './ApacheAirflowClient';
@@ -0,0 +1,83 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Query parameters for listing DAGs
*/
export interface ListDagsParams {
limit?: number;
offset?: number;
order_by?: string;
tags?: Tag[];
only_active?: boolean;
}
export interface Dags {
dags: Dag[];
total_entries: number;
}
export interface Dag {
dag_id: string;
root_dag_id?: string;
is_paused?: boolean;
is_active?: boolean;
is_subdag?: boolean;
fileloc: string;
file_token: string;
owners: string[];
description?: string;
schedule_interval: ScheduleInterval;
tags: Tag[];
}
export interface TimeDelta {
__type: 'TimeDelta';
days: number;
seconds: number;
microseconds: number;
}
export interface RelativeDelta {
__type: 'RelativeDelta';
years: number;
months: number;
days: number;
leapdays: number;
hours: number;
minutes: number;
seconds: number;
microseconds: number;
year: number;
month: number;
day: number;
hour: number;
minute: number;
second: number;
microsecond: number;
}
export interface CronExpression {
__type: 'CronExpression';
value: string;
}
// discrimant union of possible schedule interval types
export type ScheduleInterval = TimeDelta | RelativeDelta | CronExpression;
export interface Tag {
name: string;
}
@@ -0,0 +1,29 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export interface InstanceStatus {
metadatabase: MetadatabaseStatus;
scheduler: SchedulerStatus;
}
export interface MetadatabaseStatus {
status: 'healthy' | 'unhealthy';
}
export interface SchedulerStatus {
status: 'healthy' | 'unhealthy';
latest_scheduler_heartbeat?: string;
}
@@ -0,0 +1,20 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export interface InstanceVersion {
version: string;
git_version?: string;
}
@@ -0,0 +1,28 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export type {
ListDagsParams,
Dag,
Dags,
Tag,
TimeDelta,
RelativeDelta,
CronExpression,
ScheduleInterval,
} from './Dags';
export type { InstanceStatus } from './InstanceStatus';
export type { InstanceVersion } from './InstanceVersion';
@@ -0,0 +1,151 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
Progress,
StatusError,
StatusOK,
Table,
TableColumn,
} from '@backstage/core-components';
import { useApi } from '@backstage/core-plugin-api';
import Box from '@material-ui/core/Box';
import Chip from '@material-ui/core/Chip';
import IconButton from '@material-ui/core/IconButton';
import Switch from '@material-ui/core/Switch';
import Tooltip from '@material-ui/core/Tooltip';
import Typography from '@material-ui/core/Typography';
import OpenInBrowserIcon from '@material-ui/icons/OpenInBrowser';
import Alert from '@material-ui/lab/Alert';
import React from 'react';
import { useAsync } from 'react-use';
import { apacheAirflowApiRef } from '../../api';
import { Dag } from '../../api/types';
import { ScheduleIntervalLabel } from '../ScheduleIntervalLabel';
type DagTableRow = Dag & {
id: string;
dagUrl: string;
};
const columns: TableColumn[] = [
{
title: 'Paused',
field: 'is_paused',
render: (row: Partial<DagTableRow>) => (
<Tooltip title="Pause/Unpause DAG">
<Switch checked={!row.is_paused} disabled />
</Tooltip>
),
width: '5%',
},
{
title: 'DAG',
field: 'id',
render: (row: Partial<DagTableRow>) => (
<div>
<Typography variant="subtitle2" gutterBottom noWrap>
{row.id}
</Typography>
<Box display="flex" alignItems="center">
{row.tags?.map((tag, ix) => (
<Chip label={tag.name} key={ix} size="small" />
))}
</Box>
</div>
),
width: '60%',
},
{
title: 'Owner',
field: 'owners',
render: (row: Partial<DagTableRow>) => (
<Box display="flex" alignItems="center">
{row.owners?.map((owner, ix) => (
<Chip label={owner} key={ix} size="small" />
))}
</Box>
),
width: '10%',
},
{
title: 'Active',
render: (row: Partial<DagTableRow>) => (
<Box display="flex" alignItems="center">
{row.is_active ? <StatusOK /> : <StatusError />}
<Typography variant="body2">
{row.is_active ? 'Active' : 'Inactive'}
</Typography>
</Box>
),
width: '10%',
},
{
title: 'Schedule',
render: (row: Partial<DagTableRow>) => (
<ScheduleIntervalLabel interval={row.schedule_interval} />
),
width: '10%',
},
{
title: 'Link',
field: 'dagUrl',
render: (row: Partial<DagTableRow>) => (
<a href={row.dagUrl}>
<IconButton aria-label="details">
<OpenInBrowserIcon />
</IconButton>
</a>
),
width: '5%',
},
];
type DenseTableProps = {
dags: Dag[];
};
export const DenseTable = ({ dags }: DenseTableProps) => {
return (
<Table
title="DAGs"
options={{ pageSize: 5 }}
columns={columns}
data={dags}
/>
);
};
export const DagTableComponent = () => {
const apiClient = useApi(apacheAirflowApiRef);
const { value, loading, error } = useAsync(async (): Promise<Dag[]> => {
return await apiClient.listDags();
}, []);
if (loading) {
return <Progress />;
} else if (error) {
return <Alert severity="error">{error.message}</Alert>;
}
const data = value?.map(el => ({
...el,
id: el.dag_id, // table records require `id` attribute
dagUrl: `${apiClient.baseUrl}dag_details?dag_id=${el.dag_id}`, // construct path to DAG using `baseUrl`
}));
return <DenseTable dags={data || []} />;
};
@@ -0,0 +1,17 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { DagTableComponent } from './DagTableComponent';
@@ -0,0 +1,46 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { renderInTestApp, TestApiProvider } from '@backstage/test-utils';
import React from 'react';
import { ApacheAirflowApi, apacheAirflowApiRef } from '../../api';
import { HomePage } from './HomePage';
describe('<HomePage />', () => {
const mockApi: jest.Mocked<ApacheAirflowApi> = {
getInstanceStatus: jest.fn().mockResolvedValue({
metadatabase: { status: 'healthy' },
scheduler: { status: 'healthy' },
}),
getInstanceVersion: jest.fn().mockResolvedValue({
version: 'v2.0.0',
}),
listDags: jest.fn().mockResolvedValue([
{
dag_id: 'mock_dag_1',
},
]),
} as any;
it('homepage should render', async () => {
const { getByText } = await renderInTestApp(
<TestApiProvider apis={[[apacheAirflowApiRef, mockApi]]}>
<HomePage />
</TestApiProvider>,
);
expect(getByText('Apache Airflow')).toBeInTheDocument();
});
});
@@ -0,0 +1,55 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
Content,
ContentHeader,
Header,
HeaderLabel,
Page,
SupportButton,
} from '@backstage/core-components';
import { Grid } from '@material-ui/core';
import React from 'react';
import { DagTableComponent } from '../DagTableComponent';
import { StatusComponent } from '../StatusComponent';
import { VersionComponent } from '../VersionComponent';
export const HomePage = () => (
<Page themeId="tool">
<Header title="Apache Airflow" subtitle="Workflow management platform">
<HeaderLabel label="Lifecycle" value="Alpha" />
</Header>
<Content>
<ContentHeader title="Overview">
<SupportButton>
See an overview of your Apache Airflow instance, and manage workflows
</SupportButton>
</ContentHeader>
<Grid container spacing={3} direction="row">
<Grid item sm={12} lg={6}>
<VersionComponent />
</Grid>
<Grid item sm={12} lg={6}>
<StatusComponent />
</Grid>
<Grid item sm={12}>
<DagTableComponent />
</Grid>
</Grid>
</Content>
</Page>
);
@@ -0,0 +1,17 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { HomePage } from './HomePage';
@@ -0,0 +1,86 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { render } from '@testing-library/react';
import React from 'react';
import { CronExpression, RelativeDelta, TimeDelta } from '../../api/types';
import { ScheduleIntervalLabel } from './ScheduleIntervalLabel';
describe('ScheduleIntervalLabel', () => {
it('should render with cronexpression interval', async () => {
const interval = {
__type: 'CronExpression',
value: '0 0 * * *',
} as CronExpression;
const { findByText } = render(
<ScheduleIntervalLabel interval={interval} />,
);
expect(await findByText('0 0 * * *')).toBeInTheDocument();
});
it('should render with time delta interval', async () => {
const interval = {
__type: 'TimeDelta',
days: 5,
seconds: 750,
microseconds: 600,
} as TimeDelta;
const { findByText } = render(
<ScheduleIntervalLabel interval={interval} />,
);
expect(await findByText('5 days 00:12:30')).toBeInTheDocument();
});
it('should not render days with time delta interval with 0 days', async () => {
const interval = {
__type: 'TimeDelta',
days: 0,
seconds: 750,
microseconds: 600,
} as TimeDelta;
const { findByText } = render(
<ScheduleIntervalLabel interval={interval} />,
);
expect(await findByText('00:12:30')).toBeInTheDocument();
});
it('should render singular day with time delta interval and 1 day', async () => {
const interval = {
__type: 'TimeDelta',
days: 1,
seconds: 750,
microseconds: 600,
} as TimeDelta;
const { findByText } = render(
<ScheduleIntervalLabel interval={interval} />,
);
expect(await findByText('1 day 00:12:30')).toBeInTheDocument();
});
it('should render with relative delta interval', async () => {
const interval = {
__type: 'RelativeDelta',
days: 15,
months: 6,
} as RelativeDelta;
const { findByText } = render(
<ScheduleIntervalLabel interval={interval} />,
);
expect(
await findByText('relativedelta(days=+15,months=+6)'),
).toBeInTheDocument();
});
});
@@ -0,0 +1,63 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import Chip from '@material-ui/core/Chip';
import React from 'react';
import { ScheduleInterval, TimeDelta, RelativeDelta } from '../../api/types';
interface Props {
interval: ScheduleInterval | undefined;
}
const timeDeltaToLabel = (delta: TimeDelta): String => {
let label = '';
const date = new Date(0);
date.setSeconds(delta.seconds);
const time = date.toISOString().substr(11, 8);
if (delta.days === 0) {
label = `${time}`;
} else if (delta.days === 1) {
label = `1 day ${time}`;
} else {
label = `${delta.days} days ${time}`;
}
return label;
};
const relativeDeltaToLabel = (delta: RelativeDelta) => {
const params = Object.entries(delta)
.filter(o => o[0] !== '__type' && o[1] !== null && o[1] !== 0)
.map(o => (o[1] > 0 ? `${o[0]}=+${o[1]}` : `${o[0]}=-${o[1]}`));
return `relativedelta(${params})`;
};
export const ScheduleIntervalLabel = ({ interval }: Props) => {
let label: String = '';
switch (interval?.__type) {
case 'TimeDelta':
label = timeDeltaToLabel(interval);
break;
case 'RelativeDelta':
label = relativeDeltaToLabel(interval);
break;
case 'CronExpression':
label = interval.value;
break;
default:
label = 'None';
}
return <Chip label={label} size="small" />;
};
@@ -0,0 +1,17 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { ScheduleIntervalLabel } from './ScheduleIntervalLabel';
@@ -0,0 +1,60 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
InfoCard,
Progress,
StructuredMetadataTable,
} from '@backstage/core-components';
import { useApi } from '@backstage/core-plugin-api';
import Alert from '@material-ui/lab/Alert';
import React from 'react';
import { useAsync } from 'react-use';
import { apacheAirflowApiRef } from '../../api';
import { InstanceStatus } from '../../api/types';
export const StatusComponent = () => {
const apiClient = useApi(apacheAirflowApiRef);
const { value, loading, error } =
useAsync(async (): Promise<InstanceStatus> => {
return await apiClient.getInstanceStatus();
}, []);
if (loading) {
return <Progress />;
} else if (error) {
return <Alert severity="error">{error.message}</Alert>;
}
if (value) {
const metadata = {
'Metadatabase Status': value.metadatabase.status,
'Scheduler Status': value.scheduler.status,
'Latest Scheduler Heartbeat': value.scheduler.latest_scheduler_heartbeat,
};
return (
<InfoCard
title="Instance Status"
subheader="Scheduler and Metadatabase Status"
variant="fullHeight"
>
<StructuredMetadataTable metadata={metadata} />
</InfoCard>
);
}
return <Alert severity="warning">No status information found...</Alert>;
};
@@ -0,0 +1,17 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { StatusComponent } from './StatusComponent';
@@ -0,0 +1,55 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
InfoCard,
Progress,
StructuredMetadataTable,
} from '@backstage/core-components';
import { useApi } from '@backstage/core-plugin-api';
import Alert from '@material-ui/lab/Alert';
import React from 'react';
import { useAsync } from 'react-use';
import { apacheAirflowApiRef } from '../../api';
import { InstanceVersion } from '../../api/types';
export const VersionComponent = () => {
const apiClient = useApi(apacheAirflowApiRef);
const { value, loading, error } =
useAsync(async (): Promise<InstanceVersion> => {
return await apiClient.getInstanceVersion();
}, []);
if (loading) {
return <Progress />;
} else if (error) {
return <Alert severity="error">{error.message}</Alert>;
}
if (value) {
const metadata = {
Version: value.version,
'Git Version': value.git_version,
};
return (
<InfoCard title="Instance Version" variant="fullHeight">
<StructuredMetadataTable metadata={metadata} />
</InfoCard>
);
}
return <Alert severity="warning">No status information found...</Alert>;
};
@@ -0,0 +1,17 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { VersionComponent } from './VersionComponent';
+23
View File
@@ -0,0 +1,23 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* A Backstage plugin that integrates towards Apache Airflow
*
* @packageDocumentation
*/
export { apacheAirflowPlugin, ApacheAirflowPage } from './plugin';
+23
View File
@@ -0,0 +1,23 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { apacheAirflowPlugin } from './plugin';
describe('apache-airflow', () => {
it('should export plugin', () => {
expect(apacheAirflowPlugin).toBeDefined();
});
});
+51
View File
@@ -0,0 +1,51 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { rootRouteRef } from './routes';
import { apacheAirflowApiRef, ApacheAirflowClient } from './api';
import {
createApiFactory,
createPlugin,
createRoutableExtension,
discoveryApiRef,
configApiRef,
} from '@backstage/core-plugin-api';
export const apacheAirflowPlugin = createPlugin({
id: 'apache-airflow',
routes: {
root: rootRouteRef,
},
apis: [
createApiFactory({
api: apacheAirflowApiRef,
deps: { configApi: configApiRef, discoveryApi: discoveryApiRef },
factory: ({ configApi, discoveryApi }) =>
new ApacheAirflowClient({
discoveryApi,
baseUrl: configApi.getString('apacheAirflow.baseUrl'),
}),
}),
],
});
export const ApacheAirflowPage = apacheAirflowPlugin.provide(
createRoutableExtension({
name: 'ApacheAirflowPage',
component: () => import('./components/HomePage').then(m => m.HomePage),
mountPoint: rootRouteRef,
}),
);
+21
View File
@@ -0,0 +1,21 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { createRouteRef } from '@backstage/core-plugin-api';
export const rootRouteRef = createRouteRef({
id: 'apache-airflow',
});
+18
View File
@@ -0,0 +1,18 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import '@testing-library/jest-dom';
import 'cross-fetch/polyfill';