From 389f6602bcb4e988366e105725f5d616bbdb2a69 Mon Sep 17 00:00:00 2001 From: "Daniele.Mazzotta" Date: Fri, 10 Jun 2022 17:38:19 +0400 Subject: [PATCH 01/16] added new getDags method to fetch specific dagIds Signed-off-by: Daniele.Mazzotta --- .../src/api/ApacheAirflowApi.ts | 1 + .../src/api/ApacheAirflowClient.test.ts | 27 +++++++++++++++++++ .../src/api/ApacheAirflowClient.ts | 19 +++++++++++++ .../DagTableComponent/DagTableComponent.tsx | 17 +++++++++++- 4 files changed, 63 insertions(+), 1 deletion(-) diff --git a/plugins/apache-airflow/src/api/ApacheAirflowApi.ts b/plugins/apache-airflow/src/api/ApacheAirflowApi.ts index dde8c57802..838292524a 100644 --- a/plugins/apache-airflow/src/api/ApacheAirflowApi.ts +++ b/plugins/apache-airflow/src/api/ApacheAirflowApi.ts @@ -25,6 +25,7 @@ export type ApacheAirflowApi = { discoveryApi: DiscoveryApi; baseUrl: string; listDags(options?: { objectsPerRequest: number }): Promise; + getDags(dagIds: string[]): Promise<{ dags: Dag[]; dagsNotFound: string[] }>; updateDag(dagId: string, isPaused: boolean): Promise; getInstanceStatus(): Promise; getInstanceVersion(): Promise; diff --git a/plugins/apache-airflow/src/api/ApacheAirflowClient.test.ts b/plugins/apache-airflow/src/api/ApacheAirflowClient.test.ts index 704b17eb34..e56c622ff2 100644 --- a/plugins/apache-airflow/src/api/ApacheAirflowClient.test.ts +++ b/plugins/apache-airflow/src/api/ApacheAirflowClient.test.ts @@ -160,4 +160,31 @@ describe('ApacheAirflowClient', () => { expect(response.dag_id).toEqual(dagId); expect(response.is_paused).toEqual(true); }); + + it('get only some dags', async () => { + setupHandlers(); + const client = new ApacheAirflowClient({ + discoveryApi: discoveryApi, + baseUrl: 'localhost:8080/', + }); + const dagIds = ['mock_dag_1', 'mock_dag_3']; + const response = await client.getDags(dagIds); + expect(response.dags.length).toEqual(dagIds.length); + response.dags.forEach((dag, index) => + expect(dag.dag_id).toEqual(dagIds[index]), + ); + expect(response.dagsNotFound.length).toEqual(0); + }); + + it('get dags but ignore NOT FOUND errors', async () => { + setupHandlers(); + const client = new ApacheAirflowClient({ + discoveryApi: discoveryApi, + baseUrl: 'localhost:8080/', + }); + const dagIds = ['mock_dag_1', 'a-random-DAG-id']; + const response = await client.getDags(dagIds); + expect(response.dags[0].dag_id).toEqual('mock_dag_1'); + expect(response.dagsNotFound[0]).toEqual('a-random-DAG-id'); + }); }); diff --git a/plugins/apache-airflow/src/api/ApacheAirflowClient.ts b/plugins/apache-airflow/src/api/ApacheAirflowClient.ts index 847a1b1ca6..bf095a96ba 100644 --- a/plugins/apache-airflow/src/api/ApacheAirflowClient.ts +++ b/plugins/apache-airflow/src/api/ApacheAirflowClient.ts @@ -75,6 +75,25 @@ export class ApacheAirflowClient implements ApacheAirflowApi { return dags; } + async getDags( + dagIds: string[], + ): Promise<{ dags: Dag[]; dagsNotFound: string[] }> { + const dagsNotFound: string[] = []; + const response = await Promise.all( + dagIds.map(id => { + return this.fetch(`/dags/${id}`).catch(e => { + if (e.message === 'NOT FOUND') { + dagsNotFound.push(id); + } else { + throw e; + } + }); + }), + ); + const dags = response.filter(Boolean) as Dag[]; + return { dags, dagsNotFound }; + } + async updateDag(dagId: string, isPaused: boolean): Promise { const init = { method: 'PATCH', diff --git a/plugins/apache-airflow/src/components/DagTableComponent/DagTableComponent.tsx b/plugins/apache-airflow/src/components/DagTableComponent/DagTableComponent.tsx index cf343a596e..18439e1902 100644 --- a/plugins/apache-airflow/src/components/DagTableComponent/DagTableComponent.tsx +++ b/plugins/apache-airflow/src/components/DagTableComponent/DagTableComponent.tsx @@ -129,9 +129,24 @@ export const DenseTable = ({ dags }: DenseTableProps) => { ); }; -export const DagTableComponent = () => { +type DagTableComponentProps = { + dagIds?: string[]; +}; + +export const DagTableComponent = ({ dagIds }: DagTableComponentProps) => { const apiClient = useApi(apacheAirflowApiRef); const { value, loading, error } = useAsync(async (): Promise => { + if (dagIds) { + const { dags, dagsNotFound } = await apiClient.getDags(dagIds); + if (dagsNotFound.length) { + throw new Error( + `${dagsNotFound.length} DAGs were not found:\n${dagsNotFound.join( + ';\n', + )}`, + ); + } + return dags; + } return await apiClient.listDags(); }, []); From c05f081e61b5f02b77fc0e98838d96c2ddc51f7e Mon Sep 17 00:00:00 2001 From: "Daniele.Mazzotta" Date: Fri, 17 Jun 2022 12:11:47 +0400 Subject: [PATCH 02/16] test for dags found and not found length Signed-off-by: Daniele.Mazzotta --- plugins/apache-airflow/src/api/ApacheAirflowClient.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plugins/apache-airflow/src/api/ApacheAirflowClient.test.ts b/plugins/apache-airflow/src/api/ApacheAirflowClient.test.ts index e56c622ff2..329f89ee52 100644 --- a/plugins/apache-airflow/src/api/ApacheAirflowClient.test.ts +++ b/plugins/apache-airflow/src/api/ApacheAirflowClient.test.ts @@ -184,7 +184,9 @@ describe('ApacheAirflowClient', () => { }); const dagIds = ['mock_dag_1', 'a-random-DAG-id']; const response = await client.getDags(dagIds); + expect(response.dags.length).toEqual(1); expect(response.dags[0].dag_id).toEqual('mock_dag_1'); + expect(response.dagsNotFound.length).toEqual(1); expect(response.dagsNotFound[0]).toEqual('a-random-DAG-id'); }); }); From 95d2f8831376c6e8318e0e5a870622112a820583 Mon Sep 17 00:00:00 2001 From: "Daniele.Mazzotta" Date: Fri, 17 Jun 2022 12:42:30 +0400 Subject: [PATCH 03/16] add warning for missing dag ids Signed-off-by: Daniele.Mazzotta --- .../DagTableComponent/DagTableComponent.tsx | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/plugins/apache-airflow/src/components/DagTableComponent/DagTableComponent.tsx b/plugins/apache-airflow/src/components/DagTableComponent/DagTableComponent.tsx index 18439e1902..7619b06497 100644 --- a/plugins/apache-airflow/src/components/DagTableComponent/DagTableComponent.tsx +++ b/plugins/apache-airflow/src/components/DagTableComponent/DagTableComponent.tsx @@ -20,6 +20,7 @@ import { StatusOK, Table, TableColumn, + WarningPanel, } from '@backstage/core-components'; import { useApi } from '@backstage/core-plugin-api'; import Box from '@material-ui/core/Box'; @@ -30,7 +31,7 @@ import Tooltip from '@material-ui/core/Tooltip'; import Typography from '@material-ui/core/Typography'; import OpenInBrowserIcon from '@material-ui/icons/OpenInBrowser'; import Alert from '@material-ui/lab/Alert'; -import React from 'react'; +import React, { useState } from 'react'; import useAsync from 'react-use/lib/useAsync'; import { apacheAirflowApiRef } from '../../api'; import { Dag } from '../../api/types'; @@ -135,15 +136,14 @@ type DagTableComponentProps = { export const DagTableComponent = ({ dagIds }: DagTableComponentProps) => { const apiClient = useApi(apacheAirflowApiRef); + const [dagsNotFound, setDagsNotFound] = useState(); + const { value, loading, error } = useAsync(async (): Promise => { if (dagIds) { + // eslint-disable-next-line @typescript-eslint/no-shadow const { dags, dagsNotFound } = await apiClient.getDags(dagIds); if (dagsNotFound.length) { - throw new Error( - `${dagsNotFound.length} DAGs were not found:\n${dagsNotFound.join( - ';\n', - )}`, - ); + setDagsNotFound(dagsNotFound); } return dags; } @@ -162,5 +162,18 @@ export const DagTableComponent = ({ dagIds }: DagTableComponentProps) => { dagUrl: `${apiClient.baseUrl}dag_details?dag_id=${el.dag_id}`, // construct path to DAG using `baseUrl` })); - return ; + return ( + <> + {dagsNotFound ? ( + + {dagsNotFound.map(dagId => ( + {dagId} + ))} + + ) : ( + '' + )} + + + ); }; From 6b8093497f3a2e9a677f1f476813ca730fa7993a Mon Sep 17 00:00:00 2001 From: "Daniele.Mazzotta" Date: Fri, 17 Jun 2022 12:43:28 +0400 Subject: [PATCH 04/16] written tests for all dags + selected dags Signed-off-by: Daniele.Mazzotta --- .../DagTableComponent.test.tsx | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 plugins/apache-airflow/src/components/DagTableComponent/DagTableComponent.test.tsx diff --git a/plugins/apache-airflow/src/components/DagTableComponent/DagTableComponent.test.tsx b/plugins/apache-airflow/src/components/DagTableComponent/DagTableComponent.test.tsx new file mode 100644 index 0000000000..16ff3120be --- /dev/null +++ b/plugins/apache-airflow/src/components/DagTableComponent/DagTableComponent.test.tsx @@ -0,0 +1,62 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { ApacheAirflowApi, apacheAirflowApiRef } from '../../api'; +import { renderInTestApp, TestApiProvider } from '@backstage/test-utils'; +import React from 'react'; +import { DagTableComponent } from './DagTableComponent'; + +describe('DagTableComponent', () => { + const mockApi: jest.Mocked = { + listDags: jest.fn().mockResolvedValue([ + { + dag_id: 'mock_dag_1', + }, + { + dag_id: 'mock_dag_2', + }, + { + dag_id: 'mock_dag_3', + }, + ]), + getDags: jest.fn().mockResolvedValue({ + dags: [{ dag_id: 'mock_dag_1' }], + dagsNotFound: ['a-random-id'], + }), + } as any; + + it('should render all DAGs', async () => { + const { getByText } = await renderInTestApp( + + + , + ); + + ['mock_dag_1', 'mock_dag_2', 'mock_dag_3'].forEach(dagId => { + expect(getByText(dagId)).toBeInTheDocument(); + }); + }); + + it('should render only selected DAGs', async () => { + const { getByText } = await renderInTestApp( + + + , + ); + expect(getByText('mock_dag_1')).toBeInTheDocument(); + expect(getByText('Warning: 1 DAGs were not found')).toBeInTheDocument(); + expect(getByText('a-random-id')).toBeInTheDocument(); + }); +}); From fdffb40c70b3cce481e16be2abc66fd064269b87 Mon Sep 17 00:00:00 2001 From: "Daniele.Mazzotta" Date: Fri, 17 Jun 2022 12:47:33 +0400 Subject: [PATCH 05/16] export DagTableComponent Signed-off-by: Daniele.Mazzotta --- plugins/apache-airflow/src/index.ts | 6 +++++- plugins/apache-airflow/src/plugin.ts | 13 ++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/plugins/apache-airflow/src/index.ts b/plugins/apache-airflow/src/index.ts index 7186839c0a..e8353c7760 100644 --- a/plugins/apache-airflow/src/index.ts +++ b/plugins/apache-airflow/src/index.ts @@ -20,4 +20,8 @@ * @packageDocumentation */ -export { apacheAirflowPlugin, ApacheAirflowPage } from './plugin'; +export { + apacheAirflowPlugin, + ApacheAirflowPage, + ApacheAirflowDagTable, +} from './plugin'; diff --git a/plugins/apache-airflow/src/plugin.ts b/plugins/apache-airflow/src/plugin.ts index 04ef1974d4..f7e25c7003 100644 --- a/plugins/apache-airflow/src/plugin.ts +++ b/plugins/apache-airflow/src/plugin.ts @@ -17,11 +17,12 @@ import { rootRouteRef } from './routes'; import { apacheAirflowApiRef, ApacheAirflowClient } from './api'; import { + configApiRef, createApiFactory, + createComponentExtension, createPlugin, createRoutableExtension, discoveryApiRef, - configApiRef, } from '@backstage/core-plugin-api'; export const apacheAirflowPlugin = createPlugin({ @@ -49,3 +50,13 @@ export const ApacheAirflowPage = apacheAirflowPlugin.provide( mountPoint: rootRouteRef, }), ); + +export const ApacheAirflowDagTable = apacheAirflowPlugin.provide( + createComponentExtension({ + name: 'ApacheAirflowDagTable', + component: { + lazy: () => + import('./components/DagTableComponent').then(m => m.DagTableComponent), + }, + }), +); From 01f976ea728a4fe40f9d5c7a387506e09120e32b Mon Sep 17 00:00:00 2001 From: "Daniele.Mazzotta" Date: Fri, 17 Jun 2022 17:06:56 +0400 Subject: [PATCH 06/16] added changeset Signed-off-by: Daniele.Mazzotta --- .changeset/lemon-goats-obey.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/lemon-goats-obey.md diff --git a/.changeset/lemon-goats-obey.md b/.changeset/lemon-goats-obey.md new file mode 100644 index 0000000000..2c45c67282 --- /dev/null +++ b/.changeset/lemon-goats-obey.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-apache-airflow': patch +--- + +Exposed DagTableComponent as standalone component + added a prop to get only select DAGs instead of the full list From 221fbffe4464bb272b925a175538362beebd5a57 Mon Sep 17 00:00:00 2001 From: "Daniele.Mazzotta" Date: Fri, 17 Jun 2022 17:16:28 +0400 Subject: [PATCH 07/16] updated README Signed-off-by: Daniele.Mazzotta --- plugins/apache-airflow/README.md | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/plugins/apache-airflow/README.md b/plugins/apache-airflow/README.md index bdbb1b981f..11fce89e4c 100644 --- a/plugins/apache-airflow/README.md +++ b/plugins/apache-airflow/README.md @@ -3,14 +3,17 @@ Welcome to the apache-airflow plugin! This plugin serves as frontend to the REST API exposed by Apache Airflow. -Note only [Airflow v2 (and later)](https://airflow.apache.org/docs/apache-airflow/stable/deprecated-rest-api-ref.html) integrate with the plugin. +Note only [Airflow v2 (and later)](https://airflow.apache.org/docs/apache-airflow/stable/deprecated-rest-api-ref.html) +integrate with the plugin. ## Feature Requests & Ideas - [ ] Add support for running multiple instances of Airflow for monitoring - various deployment stages or business domains. ([Suggested by @JGoldman110](https://github.com/backstage/backstage/issues/735#issuecomment-985063468)) + various deployment stages or business + domains. ([Suggested by @JGoldman110](https://github.com/backstage/backstage/issues/735#issuecomment-985063468)) - [ ] Make owner chips in the DAG table clickable, resolving to a user or group - in the entity catalog. ([Suggested by @julioz](https://github.com/backstage/backstage/pull/8348#discussion_r764766295)) + in the entity + catalog. ([Suggested by @julioz](https://github.com/backstage/backstage/pull/8348#discussion_r764766295)) ## Installation @@ -42,6 +45,27 @@ yarn --cwd packages/app add @backstage/plugin-apache-airflow ); ``` +If you just want to embed the DAGs into an existing page, you can use the `ApacheAirflowDagTable` + +```typescript +import {ApacheAirflowDagTable} from '@backstage/plugin-apache-airflow'; + +export function SomeEntityPage(): JSX.Element { + return ( + + + < /Grid> +) + ; +} +``` + ## Configuration For links to the Airflow instance, the `baseUrl` must be defined in From d9ebda237e8020fe641361d635c348f8d43a4262 Mon Sep 17 00:00:00 2001 From: "Daniele.Mazzotta" Date: Fri, 17 Jun 2022 17:50:27 +0400 Subject: [PATCH 08/16] build api-reports Signed-off-by: Daniele.Mazzotta --- plugins/apache-airflow/api-report.md | 11 +++++++++-- plugins/apache-airflow/src/plugin.ts | 7 +++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/plugins/apache-airflow/api-report.md b/plugins/apache-airflow/api-report.md index ca151f0734..09123de613 100644 --- a/plugins/apache-airflow/api-report.md +++ b/plugins/apache-airflow/api-report.md @@ -5,8 +5,15 @@ ```ts /// -import { BackstagePlugin } from '@backstage/core-plugin-api'; -import { RouteRef } from '@backstage/core-plugin-api'; +import {BackstagePlugin} from '@backstage/core-plugin-api'; +import {RouteRef} from '@backstage/core-plugin-api'; + +// @public +export const ApacheAirflowDagTable: ({ + dagIds, + }: { + dagIds?: string[] | undefined; +}) => JSX.Element; // Warning: (ae-missing-release-tag) "ApacheAirflowPage" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // diff --git a/plugins/apache-airflow/src/plugin.ts b/plugins/apache-airflow/src/plugin.ts index f7e25c7003..adf5879196 100644 --- a/plugins/apache-airflow/src/plugin.ts +++ b/plugins/apache-airflow/src/plugin.ts @@ -51,6 +51,13 @@ export const ApacheAirflowPage = apacheAirflowPlugin.provide( }), ); +/** + * Render the DAGs in a table + * If the dagIds is specified, only those DAGs are loaded. + * Otherwise, it's going to list all the DAGs + * @public + * @param dagIds - string[] + */ export const ApacheAirflowDagTable = apacheAirflowPlugin.provide( createComponentExtension({ name: 'ApacheAirflowDagTable', From 8b8c692be318d4b7468ca8f2b2867a28a15ade5c Mon Sep 17 00:00:00 2001 From: Daniele Mazzotta Date: Mon, 20 Jun 2022 12:58:38 +0400 Subject: [PATCH 09/16] updated api-reports Signed-off-by: Daniele Mazzotta --- plugins/apache-airflow/src/plugin.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/apache-airflow/src/plugin.ts b/plugins/apache-airflow/src/plugin.ts index adf5879196..664cde36cc 100644 --- a/plugins/apache-airflow/src/plugin.ts +++ b/plugins/apache-airflow/src/plugin.ts @@ -56,7 +56,7 @@ export const ApacheAirflowPage = apacheAirflowPlugin.provide( * If the dagIds is specified, only those DAGs are loaded. * Otherwise, it's going to list all the DAGs * @public - * @param dagIds - string[] + * @param dagIds - optional string[] of the DAGs to show in the table. If undefined, it will list all DAGs */ export const ApacheAirflowDagTable = apacheAirflowPlugin.provide( createComponentExtension({ From c8a502ce3e0354d87f6b5cb42acde1d744cd66b7 Mon Sep 17 00:00:00 2001 From: Daniele Mazzotta Date: Tue, 21 Jun 2022 17:36:35 +0400 Subject: [PATCH 10/16] fixed formatting Signed-off-by: Daniele Mazzotta --- plugins/apache-airflow/api-report.md | 4 ++-- .../src/components/DagTableComponent/DagTableComponent.tsx | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/plugins/apache-airflow/api-report.md b/plugins/apache-airflow/api-report.md index 09123de613..333e4543cf 100644 --- a/plugins/apache-airflow/api-report.md +++ b/plugins/apache-airflow/api-report.md @@ -5,8 +5,8 @@ ```ts /// -import {BackstagePlugin} from '@backstage/core-plugin-api'; -import {RouteRef} from '@backstage/core-plugin-api'; +import { BackstagePlugin } from '@backstage/core-plugin-api'; +import { RouteRef } from '@backstage/core-plugin-api'; // @public export const ApacheAirflowDagTable: ({ diff --git a/plugins/apache-airflow/src/components/DagTableComponent/DagTableComponent.tsx b/plugins/apache-airflow/src/components/DagTableComponent/DagTableComponent.tsx index 7619b06497..ff7b961c48 100644 --- a/plugins/apache-airflow/src/components/DagTableComponent/DagTableComponent.tsx +++ b/plugins/apache-airflow/src/components/DagTableComponent/DagTableComponent.tsx @@ -164,14 +164,12 @@ export const DagTableComponent = ({ dagIds }: DagTableComponentProps) => { return ( <> - {dagsNotFound ? ( + {dagsNotFound && ( {dagsNotFound.map(dagId => ( {dagId} ))} - ) : ( - '' )} From f74361a5850c3d20923553221172ad947dc68352 Mon Sep 17 00:00:00 2001 From: "Daniele.Mazzotta" Date: Wed, 22 Jun 2022 17:29:58 +0400 Subject: [PATCH 11/16] removed dagsNotFound state Signed-off-by: Daniele.Mazzotta --- .../DagTableComponent/DagTableComponent.tsx | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/plugins/apache-airflow/src/components/DagTableComponent/DagTableComponent.tsx b/plugins/apache-airflow/src/components/DagTableComponent/DagTableComponent.tsx index ff7b961c48..46127e0b5e 100644 --- a/plugins/apache-airflow/src/components/DagTableComponent/DagTableComponent.tsx +++ b/plugins/apache-airflow/src/components/DagTableComponent/DagTableComponent.tsx @@ -31,7 +31,7 @@ import Tooltip from '@material-ui/core/Tooltip'; import Typography from '@material-ui/core/Typography'; import OpenInBrowserIcon from '@material-ui/icons/OpenInBrowser'; import Alert from '@material-ui/lab/Alert'; -import React, { useState } from 'react'; +import React from 'react'; import useAsync from 'react-use/lib/useAsync'; import { apacheAirflowApiRef } from '../../api'; import { Dag } from '../../api/types'; @@ -136,15 +136,10 @@ type DagTableComponentProps = { export const DagTableComponent = ({ dagIds }: DagTableComponentProps) => { const apiClient = useApi(apacheAirflowApiRef); - const [dagsNotFound, setDagsNotFound] = useState(); const { value, loading, error } = useAsync(async (): Promise => { if (dagIds) { - // eslint-disable-next-line @typescript-eslint/no-shadow - const { dags, dagsNotFound } = await apiClient.getDags(dagIds); - if (dagsNotFound.length) { - setDagsNotFound(dagsNotFound); - } + const { dags } = await apiClient.getDags(dagIds); return dags; } return await apiClient.listDags(); @@ -161,13 +156,16 @@ export const DagTableComponent = ({ dagIds }: DagTableComponentProps) => { id: el.dag_id, // table records require `id` attribute dagUrl: `${apiClient.baseUrl}dag_details?dag_id=${el.dag_id}`, // construct path to DAG using `baseUrl` })); - + const dagsNotFound = + dagIds && value + ? dagIds.filter(id => !value.find(d => d.dag_id === id)) + : []; return ( <> - {dagsNotFound && ( + {dagsNotFound.length && ( {dagsNotFound.map(dagId => ( - {dagId} + {dagId} ))} )} From 007cba9eb12304057f3663ec48de6754c7ab7414 Mon Sep 17 00:00:00 2001 From: "Daniele.Mazzotta" Date: Thu, 23 Jun 2022 10:07:08 +0400 Subject: [PATCH 12/16] fixed issue where error message would be 0 instead of empty element Signed-off-by: Daniele.Mazzotta --- .../src/components/DagTableComponent/DagTableComponent.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/apache-airflow/src/components/DagTableComponent/DagTableComponent.tsx b/plugins/apache-airflow/src/components/DagTableComponent/DagTableComponent.tsx index 46127e0b5e..2514d70d45 100644 --- a/plugins/apache-airflow/src/components/DagTableComponent/DagTableComponent.tsx +++ b/plugins/apache-airflow/src/components/DagTableComponent/DagTableComponent.tsx @@ -162,12 +162,14 @@ export const DagTableComponent = ({ dagIds }: DagTableComponentProps) => { : []; return ( <> - {dagsNotFound.length && ( + {dagsNotFound.length ? ( {dagsNotFound.map(dagId => ( {dagId} ))} + ) : ( + '' )} From aed6b691cb8c8f2b85b3edf999f6b57b39e66b58 Mon Sep 17 00:00:00 2001 From: "Daniele.Mazzotta" Date: Thu, 23 Jun 2022 10:29:10 +0400 Subject: [PATCH 13/16] fixed failing test while fetching getDags Signed-off-by: Daniele.Mazzotta --- .../apache-airflow/src/api/ApacheAirflowClient.test.ts | 9 +++++++++ plugins/apache-airflow/src/api/ApacheAirflowClient.ts | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/plugins/apache-airflow/src/api/ApacheAirflowClient.test.ts b/plugins/apache-airflow/src/api/ApacheAirflowClient.test.ts index 329f89ee52..b9bdb02283 100644 --- a/plugins/apache-airflow/src/api/ApacheAirflowClient.test.ts +++ b/plugins/apache-airflow/src/api/ApacheAirflowClient.test.ts @@ -107,6 +107,15 @@ describe('ApacheAirflowClient', () => { ); }), + rest.get(`${mockBaseUrl}/airflow/dags/:dag_id`, (req, res, ctx) => { + const { dag_id } = req.params; + const dag = dags.find(d => d.dag_id === dag_id); + if (dag) { + return res(ctx.json(dag)); + } + return res(ctx.status(404)); + }), + rest.patch(`${mockBaseUrl}/airflow/dags/:dag_id`, (req, res, ctx) => { const { dag_id } = req.params; const body = JSON.parse(req.body as string); diff --git a/plugins/apache-airflow/src/api/ApacheAirflowClient.ts b/plugins/apache-airflow/src/api/ApacheAirflowClient.ts index bf095a96ba..b6c0a09c3a 100644 --- a/plugins/apache-airflow/src/api/ApacheAirflowClient.ts +++ b/plugins/apache-airflow/src/api/ApacheAirflowClient.ts @@ -82,7 +82,7 @@ export class ApacheAirflowClient implements ApacheAirflowApi { const response = await Promise.all( dagIds.map(id => { return this.fetch(`/dags/${id}`).catch(e => { - if (e.message === 'NOT FOUND') { + if (e.message === 'Not Found') { dagsNotFound.push(id); } else { throw e; From 7bc7b1d37617b5de113547979c224f27a10f3ad8 Mon Sep 17 00:00:00 2001 From: "Daniele.Mazzotta" Date: Thu, 23 Jun 2022 10:35:50 +0400 Subject: [PATCH 14/16] case-insensitive NOT FOUND error catching in getDags Signed-off-by: Daniele.Mazzotta --- plugins/apache-airflow/src/api/ApacheAirflowClient.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/apache-airflow/src/api/ApacheAirflowClient.ts b/plugins/apache-airflow/src/api/ApacheAirflowClient.ts index b6c0a09c3a..c722f5977d 100644 --- a/plugins/apache-airflow/src/api/ApacheAirflowClient.ts +++ b/plugins/apache-airflow/src/api/ApacheAirflowClient.ts @@ -82,7 +82,7 @@ export class ApacheAirflowClient implements ApacheAirflowApi { const response = await Promise.all( dagIds.map(id => { return this.fetch(`/dags/${id}`).catch(e => { - if (e.message === 'Not Found') { + if (e.message.toUpperCase('en-US') === 'NOT FOUND') { dagsNotFound.push(id); } else { throw e; From b9822464b37c266ba2785000ef1e8b45cfb6e6d2 Mon Sep 17 00:00:00 2001 From: "Daniele.Mazzotta" Date: Thu, 23 Jun 2022 14:56:37 +0400 Subject: [PATCH 15/16] fixed line wraps Signed-off-by: Daniele.Mazzotta --- plugins/apache-airflow/README.md | 34 ++++++++++++++------------------ 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/plugins/apache-airflow/README.md b/plugins/apache-airflow/README.md index 11fce89e4c..305a94f081 100644 --- a/plugins/apache-airflow/README.md +++ b/plugins/apache-airflow/README.md @@ -3,17 +3,14 @@ Welcome to the apache-airflow plugin! This plugin serves as frontend to the REST API exposed by Apache Airflow. -Note only [Airflow v2 (and later)](https://airflow.apache.org/docs/apache-airflow/stable/deprecated-rest-api-ref.html) -integrate with the plugin. +Note only [Airflow v2 (and later)](https://airflow.apache.org/docs/apache-airflow/stable/deprecated-rest-api-ref.html) integrate with the plugin. ## Feature Requests & Ideas - [ ] Add support for running multiple instances of Airflow for monitoring - various deployment stages or business - domains. ([Suggested by @JGoldman110](https://github.com/backstage/backstage/issues/735#issuecomment-985063468)) + various deployment stages or business domains. ([Suggested by @JGoldman110](https://github.com/backstage/backstage/issues/735#issuecomment-985063468)) - [ ] Make owner chips in the DAG table clickable, resolving to a user or group - in the entity - catalog. ([Suggested by @julioz](https://github.com/backstage/backstage/pull/8348#discussion_r764766295)) + in the entity catalog. ([Suggested by @julioz](https://github.com/backstage/backstage/pull/8348#discussion_r764766295)) ## Installation @@ -47,22 +44,21 @@ yarn --cwd packages/app add @backstage/plugin-apache-airflow If you just want to embed the DAGs into an existing page, you can use the `ApacheAirflowDagTable` -```typescript -import {ApacheAirflowDagTable} from '@backstage/plugin-apache-airflow'; +```tsx +import { ApacheAirflowDagTable } from '@backstage/plugin-apache-airflow'; export function SomeEntityPage(): JSX.Element { return ( - - - < /Grid> -) - ; + + + + ); } ``` From 707bb20343306ad7c046d0916408ab9df149f51b Mon Sep 17 00:00:00 2001 From: "Daniele.Mazzotta" Date: Thu, 23 Jun 2022 14:57:21 +0400 Subject: [PATCH 16/16] set changeset to minor instead of patch Signed-off-by: Daniele.Mazzotta --- .changeset/lemon-goats-obey.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/lemon-goats-obey.md b/.changeset/lemon-goats-obey.md index 2c45c67282..7bdb59bd61 100644 --- a/.changeset/lemon-goats-obey.md +++ b/.changeset/lemon-goats-obey.md @@ -1,5 +1,5 @@ --- -'@backstage/plugin-apache-airflow': patch +'@backstage/plugin-apache-airflow': minor --- Exposed DagTableComponent as standalone component + added a prop to get only select DAGs instead of the full list