From c75d80fdf0ae02a06a38d0a87b77954ebe5c7027 Mon Sep 17 00:00:00 2001 From: Justin De Burgo Date: Fri, 11 Nov 2022 11:02:33 -0600 Subject: [PATCH 01/32] adding filters to catalog import Signed-off-by: Lucas De Souza --- .../src/components/ImportStepper/ImportStepper.tsx | 5 ++++- .../src/components/ImportStepper/defaults.tsx | 4 +++- .../StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx | 11 +++++++---- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx b/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx index b2638e582e..7724a3c7d1 100644 --- a/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx +++ b/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx @@ -46,6 +46,7 @@ export interface ImportStepperProps { defaults: StepperProvider, ) => StepperProvider; variant?: InfoCardVariants; + filters?: Array; } /** @@ -58,6 +59,7 @@ export const ImportStepper = (props: ImportStepperProps) => { initialUrl, generateStepper = defaultGenerateStepper, variant, + filters = [], } = props; const catalogImportApi = useApi(catalogImportApiRef); @@ -88,7 +90,8 @@ export const ImportStepper = (props: ImportStepperProps) => { {render( states.analyze( state as Extract, - { apis: { catalogImportApi } }, + { apis: { catalogImportApi } }, + filters, ), )} {render( diff --git a/plugins/catalog-import/src/components/ImportStepper/defaults.tsx b/plugins/catalog-import/src/components/ImportStepper/defaults.tsx index 94dff39d0f..6d39a38e4e 100644 --- a/plugins/catalog-import/src/components/ImportStepper/defaults.tsx +++ b/plugins/catalog-import/src/components/ImportStepper/defaults.tsx @@ -51,6 +51,7 @@ export interface StepperProvider { analyze: ( s: Extract, opts: { apis: StepperApis }, + filters?: String[], ) => StepConfiguration; prepare: ( s: Extract, @@ -263,7 +264,7 @@ export function defaultGenerateStepper( } export const defaultStepper: StepperProvider = { - analyze: (state, { apis }) => ({ + analyze: (state, { apis }, filters=[]) => ({ stepLabel: Select URL, content: ( ), }), diff --git a/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx b/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx index 3c655ec11b..51c5a4d3fd 100644 --- a/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx +++ b/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx @@ -42,6 +42,7 @@ export interface StepInitAnalyzeUrlProps { disablePullRequest?: boolean; analysisUrl?: string; exampleLocationUrl?: string; + filters?: Array, } /** @@ -58,6 +59,7 @@ export const StepInitAnalyzeUrl = (props: StepInitAnalyzeUrlProps) => { analysisUrl = '', disablePullRequest = false, exampleLocationUrl = 'https://github.com/backstage/backstage/blob/master/catalog-info.yaml', + filters = [], } = props; const errorApi = useApi(errorApiRef); @@ -78,6 +80,8 @@ export const StepInitAnalyzeUrl = (props: StepInitAnalyzeUrlProps) => { const [submitted, setSubmitted] = useState(false); const [error, setError] = useState(undefined); + const filteredRegex = new RegExp(`^http[s]?://${filters ? `[${filters.map((filter, i) => i === 0 ? filter : `|${filter}`)}]` : ''}`) + const handleResult = useCallback( async ({ url }: FormData) => { setSubmitted(true); @@ -113,9 +117,8 @@ export const StepInitAnalyzeUrl = (props: StepInitAnalyzeUrlProps) => { } default: { - const err = `Received unknown analysis result of type ${ - (analysisResult as any).type - }. Please contact the support team.`; + const err = `Received unknown analysis result of type ${(analysisResult as any).type + }. Please contact the support team.`; setError(err); setSubmitted(false); @@ -140,7 +143,7 @@ export const StepInitAnalyzeUrl = (props: StepInitAnalyzeUrlProps) => { validate: { httpsValidator: (value: any) => (typeof value === 'string' && - value.match(/^http[s]?:\/\//) !== null) || + value.match(filteredRegex) !== null) || 'Must start with http:// or https://.', }, }), From 723dc1bb514c192b6557b2e4d78db59160f7857d Mon Sep 17 00:00:00 2001 From: Justin De Burgo Date: Fri, 11 Nov 2022 11:13:41 -0600 Subject: [PATCH 02/32] adding filter to default import page Signed-off-by: Lucas De Souza --- .../components/DefaultImportPage/DefaultImportPage.tsx | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-import/src/components/DefaultImportPage/DefaultImportPage.tsx b/plugins/catalog-import/src/components/DefaultImportPage/DefaultImportPage.tsx index 30cce9603a..0236e7c4d0 100644 --- a/plugins/catalog-import/src/components/DefaultImportPage/DefaultImportPage.tsx +++ b/plugins/catalog-import/src/components/DefaultImportPage/DefaultImportPage.tsx @@ -27,12 +27,17 @@ import React from 'react'; import { ImportInfoCard } from '../ImportInfoCard'; import { ImportStepper } from '../ImportStepper'; +interface DefaultImportPageProps { + filters?: Array; +} + /** * The default catalog import page. * * @public */ -export const DefaultImportPage = () => { +export const DefaultImportPage = (props: DefaultImportPageProps = { filters: [] }) => { + const { filters } = props; const configApi = useApi(configApiRef); const appTitle = configApi.getOptional('app.title') || 'Backstage'; @@ -53,7 +58,7 @@ export const DefaultImportPage = () => { - + From b4c84ca06bb2f97f6d364b4d70bff002d5a53442 Mon Sep 17 00:00:00 2001 From: Justin De Burgo Date: Fri, 11 Nov 2022 11:19:54 -0600 Subject: [PATCH 03/32] adding filter to import page Signed-off-by: Lucas De Souza --- .../catalog-import/src/components/ImportPage/ImportPage.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-import/src/components/ImportPage/ImportPage.tsx b/plugins/catalog-import/src/components/ImportPage/ImportPage.tsx index 129e714952..beae0e5b3a 100644 --- a/plugins/catalog-import/src/components/ImportPage/ImportPage.tsx +++ b/plugins/catalog-import/src/components/ImportPage/ImportPage.tsx @@ -23,8 +23,8 @@ import { DefaultImportPage } from '../DefaultImportPage'; * * @public */ -export const ImportPage = () => { +export const ImportPage = ({ filters=[] }) => { const outlet = useOutlet(); - return outlet || ; + return outlet || ; }; From 8985c47ed2278cb0e253d158fc22d6142d8188c3 Mon Sep 17 00:00:00 2001 From: Justin De Burgo Date: Tue, 15 Nov 2022 12:16:58 -0600 Subject: [PATCH 04/32] removing filter from component and add to rules Co-authored-by: Zeky Abubaker Signed-off-by: Lucas De Souza --- app-config.yaml | 3 +++ .../src/ingestion/CatalogRules.ts | 27 ++++++++++++++++--- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index dd72051189..3eb8e7e01c 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -218,6 +218,9 @@ catalog: - System - Domain - Location + - owners: + - Spotify + - Backstage processors: ldapOrg: diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index 548c71d833..d25e936d1a 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -32,6 +32,9 @@ export type CatalogRule = { target?: string; type: string; }>; + owners?: Array<{ + owner: string + }>; }; /** @@ -94,6 +97,7 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { if (config.has('catalog.rules')) { const globalRules = config.getConfigArray('catalog.rules').map(sub => ({ allow: sub.getStringArray('allow').map(kind => ({ kind })), + owners: sub.getStringArray('owners').map(kind => ({ kind })), })); rules.push(...globalRules); } else { @@ -122,7 +126,7 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { return new DefaultCatalogRulesEnforcer(rules); } - constructor(private readonly rules: CatalogRule[]) {} + constructor(private readonly rules: CatalogRule[]) { } /** * Checks whether a specific entity/location combination is allowed @@ -134,9 +138,13 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { continue; } - if (this.matchEntity(entity, rule.allow)) { - return true; + if (!this.matchOwners(entity, rule.owners)) { + return false; } + + if (this.matchEntity(entity, rule.allow)) { + return true; + } } return false; @@ -178,6 +186,19 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { return false; } + + private matchOwners(entity: Entity, matchers?: { owner: string }[]): boolean { + if (!matchers) { + return true; + } + + const filteredRegex = new RegExp(`^http[s]?://${`(?:${matchers.map((filter, i) => i === 0 ? filter.owner : `|${filter.owner}`)})`}`); + + if ( entity?.metadata.links && entity?.metadata?.links?.length > 0) { + return filteredRegex.test(entity?.metadata?.links[0].url); + } + return false; + } } function resolveTarget(type: string, target: string): string { From ea96a0206429e6be499b750eba2e1704c0f5001e Mon Sep 17 00:00:00 2001 From: Justin De Burgo Date: Tue, 15 Nov 2022 16:32:05 -0600 Subject: [PATCH 05/32] using sources instead of owners Signed-off-by: Lucas De Souza --- app-config.yaml | 3 --- .../src/ingestion/CatalogRules.ts | 19 ++++++++++--------- .../ImportStepper/ImportStepper.tsx | 2 -- .../src/components/ImportStepper/defaults.tsx | 3 +-- .../StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx | 5 +---- 5 files changed, 12 insertions(+), 20 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index 3eb8e7e01c..dd72051189 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -218,9 +218,6 @@ catalog: - System - Domain - Location - - owners: - - Spotify - - Backstage processors: ldapOrg: diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index d25e936d1a..a48d29add6 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -32,8 +32,8 @@ export type CatalogRule = { target?: string; type: string; }>; - owners?: Array<{ - owner: string + sources?: Array<{ + source: string }>; }; @@ -58,6 +58,7 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { static readonly defaultRules: CatalogRule[] = [ { allow: ['Component', 'API', 'Location'].map(kind => ({ kind })), + sources: [], }, ]; @@ -97,7 +98,7 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { if (config.has('catalog.rules')) { const globalRules = config.getConfigArray('catalog.rules').map(sub => ({ allow: sub.getStringArray('allow').map(kind => ({ kind })), - owners: sub.getStringArray('owners').map(kind => ({ kind })), + sources: sub.getStringArray('sources').map(source => ({ source })), })); rules.push(...globalRules); } else { @@ -138,7 +139,7 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { continue; } - if (!this.matchOwners(entity, rule.owners)) { + if (!this.matchSources(location, rule.sources)) { return false; } @@ -187,15 +188,15 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { return false; } - private matchOwners(entity: Entity, matchers?: { owner: string }[]): boolean { - if (!matchers) { + private matchSources(location: LocationSpec, matchers?: { source: string }[]): boolean { + if (!matchers || matchers.length === 0) { return true; } - const filteredRegex = new RegExp(`^http[s]?://${`(?:${matchers.map((filter, i) => i === 0 ? filter.owner : `|${filter.owner}`)})`}`); + const filteredRegex = new RegExp(`^http[s]?://${`(?:${matchers.map((filter, i) => i === 0 ? filter.source : `|${filter.source}`)})`}`); - if ( entity?.metadata.links && entity?.metadata?.links?.length > 0) { - return filteredRegex.test(entity?.metadata?.links[0].url); + if ( location.target && location.target.length > 0) { + return filteredRegex.test(location.target); } return false; } diff --git a/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx b/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx index 7724a3c7d1..7352741326 100644 --- a/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx +++ b/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx @@ -59,7 +59,6 @@ export const ImportStepper = (props: ImportStepperProps) => { initialUrl, generateStepper = defaultGenerateStepper, variant, - filters = [], } = props; const catalogImportApi = useApi(catalogImportApiRef); @@ -91,7 +90,6 @@ export const ImportStepper = (props: ImportStepperProps) => { states.analyze( state as Extract, { apis: { catalogImportApi } }, - filters, ), )} {render( diff --git a/plugins/catalog-import/src/components/ImportStepper/defaults.tsx b/plugins/catalog-import/src/components/ImportStepper/defaults.tsx index 6d39a38e4e..d44a71dbdb 100644 --- a/plugins/catalog-import/src/components/ImportStepper/defaults.tsx +++ b/plugins/catalog-import/src/components/ImportStepper/defaults.tsx @@ -264,7 +264,7 @@ export function defaultGenerateStepper( } export const defaultStepper: StepperProvider = { - analyze: (state, { apis }, filters=[]) => ({ + analyze: (state, { apis }) => ({ stepLabel: Select URL, content: ( ), }), diff --git a/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx b/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx index 51c5a4d3fd..49e4a4fa4a 100644 --- a/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx +++ b/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx @@ -59,7 +59,6 @@ export const StepInitAnalyzeUrl = (props: StepInitAnalyzeUrlProps) => { analysisUrl = '', disablePullRequest = false, exampleLocationUrl = 'https://github.com/backstage/backstage/blob/master/catalog-info.yaml', - filters = [], } = props; const errorApi = useApi(errorApiRef); @@ -80,8 +79,6 @@ export const StepInitAnalyzeUrl = (props: StepInitAnalyzeUrlProps) => { const [submitted, setSubmitted] = useState(false); const [error, setError] = useState(undefined); - const filteredRegex = new RegExp(`^http[s]?://${filters ? `[${filters.map((filter, i) => i === 0 ? filter : `|${filter}`)}]` : ''}`) - const handleResult = useCallback( async ({ url }: FormData) => { setSubmitted(true); @@ -143,7 +140,7 @@ export const StepInitAnalyzeUrl = (props: StepInitAnalyzeUrlProps) => { validate: { httpsValidator: (value: any) => (typeof value === 'string' && - value.match(filteredRegex) !== null) || + value.match(/^http[s]?:\/\//) !== null) || 'Must start with http:// or https://.', }, }), From 1b6bc7ca4c95fee525734899c8a89450b3805b31 Mon Sep 17 00:00:00 2001 From: Justin De Burgo <57914589+jpdeburgo@users.noreply.github.com> Date: Tue, 15 Nov 2022 16:40:20 -0600 Subject: [PATCH 06/32] Update plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx Co-authored-by: Lucas Desouza Signed-off-by: Justin De Burgo <57914589+jpdeburgo@users.noreply.github.com> Signed-off-by: Lucas De Souza --- .../src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx b/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx index 49e4a4fa4a..d936eaeba0 100644 --- a/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx +++ b/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx @@ -114,8 +114,9 @@ export const StepInitAnalyzeUrl = (props: StepInitAnalyzeUrlProps) => { } default: { - const err = `Received unknown analysis result of type ${(analysisResult as any).type - }. Please contact the support team.`; + const err = `Received unknown analysis result of type ${ + (analysisResult as any).type + }. Please contact the support team.` setError(err); setSubmitted(false); From 3a5d705fd70834077022614666581653d42186e0 Mon Sep 17 00:00:00 2001 From: Justin De Burgo <57914589+jpdeburgo@users.noreply.github.com> Date: Tue, 15 Nov 2022 16:40:31 -0600 Subject: [PATCH 07/32] Update plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx Co-authored-by: Lucas Desouza Signed-off-by: Justin De Burgo <57914589+jpdeburgo@users.noreply.github.com> Signed-off-by: Lucas De Souza --- .../src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx b/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx index d936eaeba0..aa9daffee1 100644 --- a/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx +++ b/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx @@ -42,7 +42,6 @@ export interface StepInitAnalyzeUrlProps { disablePullRequest?: boolean; analysisUrl?: string; exampleLocationUrl?: string; - filters?: Array, } /** From 430db47ad297ee16ad41d4997a4910eb80a73290 Mon Sep 17 00:00:00 2001 From: Justin De Burgo <57914589+jpdeburgo@users.noreply.github.com> Date: Tue, 15 Nov 2022 16:40:36 -0600 Subject: [PATCH 08/32] Update plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx Co-authored-by: Lucas Desouza Signed-off-by: Justin De Burgo <57914589+jpdeburgo@users.noreply.github.com> Signed-off-by: Lucas De Souza --- .../src/components/ImportStepper/ImportStepper.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx b/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx index 7352741326..ad1ad98465 100644 --- a/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx +++ b/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx @@ -89,7 +89,7 @@ export const ImportStepper = (props: ImportStepperProps) => { {render( states.analyze( state as Extract, - { apis: { catalogImportApi } }, + { apis: { catalogImportApi } }, ), )} {render( From cdbd1c000a3e2ce6a3ebd49d5121142964ed85d2 Mon Sep 17 00:00:00 2001 From: Justin De Burgo <57914589+jpdeburgo@users.noreply.github.com> Date: Tue, 15 Nov 2022 16:40:47 -0600 Subject: [PATCH 09/32] Update plugins/catalog-backend/src/ingestion/CatalogRules.ts Co-authored-by: Lucas Desouza Signed-off-by: Justin De Burgo <57914589+jpdeburgo@users.noreply.github.com> Signed-off-by: Lucas De Souza --- plugins/catalog-backend/src/ingestion/CatalogRules.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index a48d29add6..2c19773e41 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -33,7 +33,7 @@ export type CatalogRule = { type: string; }>; sources?: Array<{ - source: string + source: string; }>; }; From bec0c068c977f183005488c0181de9abb51180e2 Mon Sep 17 00:00:00 2001 From: Justin De Burgo <57914589+jpdeburgo@users.noreply.github.com> Date: Tue, 15 Nov 2022 16:40:54 -0600 Subject: [PATCH 10/32] Update plugins/catalog-backend/src/ingestion/CatalogRules.ts Co-authored-by: Lucas Desouza Signed-off-by: Justin De Burgo <57914589+jpdeburgo@users.noreply.github.com> Signed-off-by: Lucas De Souza --- plugins/catalog-backend/src/ingestion/CatalogRules.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index 2c19773e41..903774b2f9 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -127,7 +127,7 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { return new DefaultCatalogRulesEnforcer(rules); } - constructor(private readonly rules: CatalogRule[]) { } + constructor(private readonly rules: CatalogRule[]) {} /** * Checks whether a specific entity/location combination is allowed From 6b325f6b61eb66299974f7519bd1fc90326333a5 Mon Sep 17 00:00:00 2001 From: Justin De Burgo Date: Tue, 15 Nov 2022 16:43:17 -0600 Subject: [PATCH 11/32] using optional string array Signed-off-by: Lucas De Souza --- app-config.yaml | 2 ++ plugins/catalog-backend/src/ingestion/CatalogRules.ts | 8 ++++---- .../components/DefaultImportPage/DefaultImportPage.tsx | 8 ++------ .../src/components/ImportStepper/defaults.tsx | 1 - 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index dd72051189..fd9263a5e8 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -218,6 +218,8 @@ catalog: - System - Domain - Location + sources: + - github.com/backstage processors: ldapOrg: diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index 903774b2f9..398133036a 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -98,7 +98,7 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { if (config.has('catalog.rules')) { const globalRules = config.getConfigArray('catalog.rules').map(sub => ({ allow: sub.getStringArray('allow').map(kind => ({ kind })), - sources: sub.getStringArray('sources').map(source => ({ source })), + sources: (sub.getOptionalStringArray('sources') || []).map(source => ({ source })), })); rules.push(...globalRules); } else { @@ -143,9 +143,9 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { return false; } - if (this.matchEntity(entity, rule.allow)) { - return true; - } + if (this.matchEntity(entity, rule.allow)) { + return true; + } } return false; diff --git a/plugins/catalog-import/src/components/DefaultImportPage/DefaultImportPage.tsx b/plugins/catalog-import/src/components/DefaultImportPage/DefaultImportPage.tsx index 0236e7c4d0..cec35574fc 100644 --- a/plugins/catalog-import/src/components/DefaultImportPage/DefaultImportPage.tsx +++ b/plugins/catalog-import/src/components/DefaultImportPage/DefaultImportPage.tsx @@ -27,17 +27,13 @@ import React from 'react'; import { ImportInfoCard } from '../ImportInfoCard'; import { ImportStepper } from '../ImportStepper'; -interface DefaultImportPageProps { - filters?: Array; -} /** * The default catalog import page. * * @public */ -export const DefaultImportPage = (props: DefaultImportPageProps = { filters: [] }) => { - const { filters } = props; +export const DefaultImportPage = () => { const configApi = useApi(configApiRef); const appTitle = configApi.getOptional('app.title') || 'Backstage'; @@ -58,7 +54,7 @@ export const DefaultImportPage = (props: DefaultImportPageProps = { filters: [] - + diff --git a/plugins/catalog-import/src/components/ImportStepper/defaults.tsx b/plugins/catalog-import/src/components/ImportStepper/defaults.tsx index d44a71dbdb..94dff39d0f 100644 --- a/plugins/catalog-import/src/components/ImportStepper/defaults.tsx +++ b/plugins/catalog-import/src/components/ImportStepper/defaults.tsx @@ -51,7 +51,6 @@ export interface StepperProvider { analyze: ( s: Extract, opts: { apis: StepperApis }, - filters?: String[], ) => StepConfiguration; prepare: ( s: Extract, From 08c58453d513329146559c46168ddb67866070f0 Mon Sep 17 00:00:00 2001 From: Justin De Burgo Date: Tue, 15 Nov 2022 16:46:15 -0600 Subject: [PATCH 12/32] removing old filter Co-authored-by: Zeky Abubaker Signed-off-by: Lucas De Souza --- .../src/components/DefaultImportPage/DefaultImportPage.tsx | 1 - .../catalog-import/src/components/ImportPage/ImportPage.tsx | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/plugins/catalog-import/src/components/DefaultImportPage/DefaultImportPage.tsx b/plugins/catalog-import/src/components/DefaultImportPage/DefaultImportPage.tsx index cec35574fc..30cce9603a 100644 --- a/plugins/catalog-import/src/components/DefaultImportPage/DefaultImportPage.tsx +++ b/plugins/catalog-import/src/components/DefaultImportPage/DefaultImportPage.tsx @@ -27,7 +27,6 @@ import React from 'react'; import { ImportInfoCard } from '../ImportInfoCard'; import { ImportStepper } from '../ImportStepper'; - /** * The default catalog import page. * diff --git a/plugins/catalog-import/src/components/ImportPage/ImportPage.tsx b/plugins/catalog-import/src/components/ImportPage/ImportPage.tsx index beae0e5b3a..129e714952 100644 --- a/plugins/catalog-import/src/components/ImportPage/ImportPage.tsx +++ b/plugins/catalog-import/src/components/ImportPage/ImportPage.tsx @@ -23,8 +23,8 @@ import { DefaultImportPage } from '../DefaultImportPage'; * * @public */ -export const ImportPage = ({ filters=[] }) => { +export const ImportPage = () => { const outlet = useOutlet(); - return outlet || ; + return outlet || ; }; From bc405c9e37d5ab68434f20b1b499d7c1b5e59766 Mon Sep 17 00:00:00 2001 From: Justin De Burgo Date: Tue, 15 Nov 2022 16:50:13 -0600 Subject: [PATCH 13/32] removing space Co-authored-by: Zeky Abubaker Signed-off-by: Lucas De Souza --- .../src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx b/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx index aa9daffee1..3c655ec11b 100644 --- a/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx +++ b/plugins/catalog-import/src/components/StepInitAnalyzeUrl/StepInitAnalyzeUrl.tsx @@ -114,8 +114,8 @@ export const StepInitAnalyzeUrl = (props: StepInitAnalyzeUrlProps) => { default: { const err = `Received unknown analysis result of type ${ - (analysisResult as any).type - }. Please contact the support team.` + (analysisResult as any).type + }. Please contact the support team.`; setError(err); setSubmitted(false); From b711d136df6b8e13e087b5618bf789cc616b40fd Mon Sep 17 00:00:00 2001 From: Justin De Burgo Date: Tue, 15 Nov 2022 16:51:29 -0600 Subject: [PATCH 14/32] remove filter in import stepper Co-authored-by: Zeky Abubaker Signed-off-by: Lucas De Souza --- .../src/components/ImportStepper/ImportStepper.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx b/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx index ad1ad98465..b2638e582e 100644 --- a/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx +++ b/plugins/catalog-import/src/components/ImportStepper/ImportStepper.tsx @@ -46,7 +46,6 @@ export interface ImportStepperProps { defaults: StepperProvider, ) => StepperProvider; variant?: InfoCardVariants; - filters?: Array; } /** From f75ec75d3e38790f6957ac27fc4975181b628480 Mon Sep 17 00:00:00 2001 From: Justin De Burgo Date: Thu, 17 Nov 2022 20:37:11 -0600 Subject: [PATCH 15/32] adding tests Signed-off-by: Lucas De Souza --- .../src/ingestion/CatalogRules.test.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts index 2592b8b94b..43c3cd8f2f 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts @@ -35,6 +35,10 @@ const entity = { }; const location: Record = { + w: { + type: 'url', + target: 'https://github.com/backstage/blob/master/w.yaml', + }, x: { type: 'url', target: 'https://github.com/a/b/blob/master/x.yaml', @@ -216,5 +220,18 @@ describe('DefaultCatalogRulesEnforcer', () => { expect(enforcer.isAllowed(entity.component, location.z)).toBe(false); expect(enforcer.isAllowed(entity.location, location.z)).toBe(false); }); + + it('should only allow sources that are specified in sources', () => { + const enforcer = DefaultCatalogRulesEnforcer.fromConfig( + new ConfigReader({ + catalog: { + rules: [{ allow: ['Component'], sources: ['github.com/backstage'] }], + }, + }), + ); + expect(enforcer.isAllowed(entity.component, location.w)).toBe(true); + expect(enforcer.isAllowed(entity.component, location.y)).toBe(false); + expect(enforcer.isAllowed(entity.component, location.z)).toBe(false); + }); }); }); From e8303e99ff5bf7518673b0410feabce67c461825 Mon Sep 17 00:00:00 2001 From: Lucas De Souza Date: Mon, 21 Nov 2022 15:01:30 -0600 Subject: [PATCH 16/32] updating to use location instead of sources Co-authored-by: Zeky Abubaker Signed-off-by: Lucas De Souza --- app-config.yaml | 3 +- .../src/ingestion/CatalogRules.test.ts | 8 ++-- .../src/ingestion/CatalogRules.ts | 46 ++++++++----------- 3 files changed, 25 insertions(+), 32 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index fd9263a5e8..d68391717e 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -218,8 +218,7 @@ catalog: - System - Domain - Location - sources: - - github.com/backstage + processors: ldapOrg: diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts index 43c3cd8f2f..c9e0f7c619 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts @@ -37,7 +37,7 @@ const entity = { const location: Record = { w: { type: 'url', - target: 'https://github.com/backstage/blob/master/w.yaml', + target: 'https://github.com/b/c/blob/master/w.yaml', }, x: { type: 'url', @@ -209,7 +209,7 @@ describe('DefaultCatalogRulesEnforcer', () => { const enforcer = DefaultCatalogRulesEnforcer.fromConfig( new ConfigReader({ catalog: { - rules: [{ allow: ['Group'], locations: [{ type: 'url' }] }], + rules: [{ allow: ['Group'] }], }, }), ); @@ -221,11 +221,11 @@ describe('DefaultCatalogRulesEnforcer', () => { expect(enforcer.isAllowed(entity.location, location.z)).toBe(false); }); - it('should only allow sources that are specified in sources', () => { + it('should only allow locations that match a given regex', () => { const enforcer = DefaultCatalogRulesEnforcer.fromConfig( new ConfigReader({ catalog: { - rules: [{ allow: ['Component'], sources: ['github.com/backstage'] }], + rules: [{ allow: ['Component'], locations: [{type: 'url', match: 'https://github.com/b/*'}] }], }, }), ); diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index 398133036a..43d865c4d2 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -31,9 +31,7 @@ export type CatalogRule = { locations?: Array<{ target?: string; type: string; - }>; - sources?: Array<{ - source: string; + match?: string; }>; }; @@ -58,7 +56,6 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { static readonly defaultRules: CatalogRule[] = [ { allow: ['Component', 'API', 'Location'].map(kind => ({ kind })), - sources: [], }, ]; @@ -96,10 +93,21 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { const rules = new Array(); if (config.has('catalog.rules')) { - const globalRules = config.getConfigArray('catalog.rules').map(sub => ({ - allow: sub.getStringArray('allow').map(kind => ({ kind })), - sources: (sub.getOptionalStringArray('sources') || []).map(source => ({ source })), - })); + const globalRules = config.getConfigArray('catalog.rules').map(ruleConfig => { + const rule: CatalogRule = { + allow: ruleConfig.getStringArray('allow').map(kind => ({ kind })), + }; + + const locConf = ruleConfig.getOptionalConfigArray('locations'); + if (locConf) + rule.locations = locConf.map( locationConfig => ({ + match: locationConfig.getOptionalString('match'), + type: locationConfig.getString('type'), + target: locationConfig.getOptionalString('target') + })) + + return rule; + }); rules.push(...globalRules); } else { rules.push(...DefaultCatalogRulesEnforcer.defaultRules); @@ -139,10 +147,6 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { continue; } - if (!this.matchSources(location, rule.sources)) { - return false; - } - if (this.matchEntity(entity, rule.allow)) { return true; } @@ -153,7 +157,7 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { private matchLocation( location: LocationSpec, - matchers?: { target?: string; type: string }[], + matchers?: { target?: string; type: string, match?: string }[], ): boolean { if (!matchers) { return true; @@ -166,6 +170,9 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { if (matcher.target && matcher.target !== location?.target) { continue; } + if (matcher.match && !location?.target.match(matcher.match)) { + continue; + } return true; } @@ -187,19 +194,6 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { return false; } - - private matchSources(location: LocationSpec, matchers?: { source: string }[]): boolean { - if (!matchers || matchers.length === 0) { - return true; - } - - const filteredRegex = new RegExp(`^http[s]?://${`(?:${matchers.map((filter, i) => i === 0 ? filter.source : `|${filter.source}`)})`}`); - - if ( location.target && location.target.length > 0) { - return filteredRegex.test(location.target); - } - return false; - } } function resolveTarget(type: string, target: string): string { From c13544e033179e4da2c6bf75c884b285eedb371d Mon Sep 17 00:00:00 2001 From: Lucas Desouza Date: Mon, 21 Nov 2022 15:15:57 -0600 Subject: [PATCH 17/32] remove extra new line Co-authored-by: Zeky Abubaker Signed-off-by: Lucas Desouza Signed-off-by: Lucas De Souza --- app-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/app-config.yaml b/app-config.yaml index d68391717e..dd72051189 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -219,7 +219,6 @@ catalog: - Domain - Location - processors: ldapOrg: ### Example for how to add your enterprise LDAP server From 250b6f04f804317459ed098d139062466e5ceca8 Mon Sep 17 00:00:00 2001 From: Lucas De Souza Date: Mon, 21 Nov 2022 15:40:56 -0600 Subject: [PATCH 18/32] run prettier Signed-off-by: Lucas De Souza --- app-config.yaml | 12 ++++++------ .../src/ingestion/CatalogRules.test.ts | 2 +- .../src/ingestion/CatalogRules.ts | 16 ++++++++-------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index dd72051189..c0f5767a45 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -212,12 +212,12 @@ catalog: pullRequestBranchName: backstage-integration rules: - allow: - - Component - - API - - Resource - - System - - Domain - - Location + - Component + - API + - Resource + - System + - Domain + - Location processors: ldapOrg: diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts index c9e0f7c619..f4b9f5ea59 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts @@ -225,7 +225,7 @@ describe('DefaultCatalogRulesEnforcer', () => { const enforcer = DefaultCatalogRulesEnforcer.fromConfig( new ConfigReader({ catalog: { - rules: [{ allow: ['Component'], locations: [{type: 'url', match: 'https://github.com/b/*'}] }], + rules: [{ allow: ['Component'], locations: [{ type: 'url', match: 'https://github.com/b/*' }] }], }, }), ); diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index 43d865c4d2..0d013dc78b 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -99,13 +99,13 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { }; const locConf = ruleConfig.getOptionalConfigArray('locations'); - if (locConf) - rule.locations = locConf.map( locationConfig => ({ - match: locationConfig.getOptionalString('match'), - type: locationConfig.getString('type'), - target: locationConfig.getOptionalString('target') - })) - + if (locConf) + rule.locations = locConf.map(locationConfig => ({ + match: locationConfig.getOptionalString('match'), + type: locationConfig.getString('type'), + target: locationConfig.getOptionalString('target') + })) + return rule; }); rules.push(...globalRules); @@ -135,7 +135,7 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { return new DefaultCatalogRulesEnforcer(rules); } - constructor(private readonly rules: CatalogRule[]) {} + constructor(private readonly rules: CatalogRule[]) { } /** * Checks whether a specific entity/location combination is allowed From b6e8b6a23f16a30f71084e02e455cbdacfff122d Mon Sep 17 00:00:00 2001 From: Lucas De Souza Date: Tue, 22 Nov 2022 10:19:45 -0600 Subject: [PATCH 19/32] run prettier Signed-off-by: Lucas De Souza --- app-config.yaml | 12 +++---- .../src/ingestion/CatalogRules.test.ts | 7 +++- .../src/ingestion/CatalogRules.ts | 32 ++++++++++--------- 3 files changed, 29 insertions(+), 22 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index c0f5767a45..dd72051189 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -212,12 +212,12 @@ catalog: pullRequestBranchName: backstage-integration rules: - allow: - - Component - - API - - Resource - - System - - Domain - - Location + - Component + - API + - Resource + - System + - Domain + - Location processors: ldapOrg: diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts index f4b9f5ea59..6cf8682129 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts @@ -225,7 +225,12 @@ describe('DefaultCatalogRulesEnforcer', () => { const enforcer = DefaultCatalogRulesEnforcer.fromConfig( new ConfigReader({ catalog: { - rules: [{ allow: ['Component'], locations: [{ type: 'url', match: 'https://github.com/b/*' }] }], + rules: [ + { + allow: ['Component'], + locations: [{ type: 'url', match: 'https://github.com/b/*' }], + }, + ], }, }), ); diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index 0d013dc78b..66fe3da54e 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -93,21 +93,23 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { const rules = new Array(); if (config.has('catalog.rules')) { - const globalRules = config.getConfigArray('catalog.rules').map(ruleConfig => { - const rule: CatalogRule = { - allow: ruleConfig.getStringArray('allow').map(kind => ({ kind })), - }; + const globalRules = config + .getConfigArray('catalog.rules') + .map(ruleConfig => { + const rule: CatalogRule = { + allow: ruleConfig.getStringArray('allow').map(kind => ({ kind })), + }; - const locConf = ruleConfig.getOptionalConfigArray('locations'); - if (locConf) - rule.locations = locConf.map(locationConfig => ({ - match: locationConfig.getOptionalString('match'), - type: locationConfig.getString('type'), - target: locationConfig.getOptionalString('target') - })) + const locConf = ruleConfig.getOptionalConfigArray('locations'); + if (locConf) + rule.locations = locConf.map(locationConfig => ({ + match: locationConfig.getOptionalString('match'), + type: locationConfig.getString('type'), + target: locationConfig.getOptionalString('target'), + })); - return rule; - }); + return rule; + }); rules.push(...globalRules); } else { rules.push(...DefaultCatalogRulesEnforcer.defaultRules); @@ -135,7 +137,7 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { return new DefaultCatalogRulesEnforcer(rules); } - constructor(private readonly rules: CatalogRule[]) { } + constructor(private readonly rules: CatalogRule[]) {} /** * Checks whether a specific entity/location combination is allowed @@ -157,7 +159,7 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { private matchLocation( location: LocationSpec, - matchers?: { target?: string; type: string, match?: string }[], + matchers?: { target?: string; type: string; match?: string }[], ): boolean { if (!matchers) { return true; From ba13ff663c930f563519c8f65bcceb01f918158a Mon Sep 17 00:00:00 2001 From: Justin De Burgo Date: Tue, 22 Nov 2022 15:29:42 -0600 Subject: [PATCH 20/32] adding changeset Signed-off-by: Justin De Burgo --- .changeset/breezy-apes-mate.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/breezy-apes-mate.md diff --git a/.changeset/breezy-apes-mate.md b/.changeset/breezy-apes-mate.md new file mode 100644 index 0000000000..ac3409bd69 --- /dev/null +++ b/.changeset/breezy-apes-mate.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Adding an optional restriction for locations added through catalog import. Restrictions can be added using the app-config.yaml From 4c84c3d6ee61ccbde9e7eb16ef855cc4527a1c1a Mon Sep 17 00:00:00 2001 From: Lucas De Souza Date: Wed, 23 Nov 2022 09:32:15 -0600 Subject: [PATCH 21/32] update config.d with new fields Signed-off-by: Lucas De Souza --- plugins/catalog-backend/config.d.ts | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index daa70656bc..16094af489 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -38,6 +38,32 @@ export interface Config { * E.g. ["Component", "API", "Template", "Location"] */ allow: Array; + /** + * Limit this rule to a specific location + * + * Example with a fixed location + * { "type": "url", "target": "https://github.com/a/b/blob/file.yaml} + * + * Example using a Regex + * { "type": "url", "match": "https://github.com/a/*} + * + */ + location?: Array<{ + /** + * The type of location, e.g. "url". + */ + type: string; + /** + * The target URL of the location, e.g. + * "https://github.com/org/repo/blob/master/users.yaml". + */ + target?: string; + /** + * The target Regex of the location, e.g. + * "https://github.com/org/*. + */ + match?: string; + }>; }>; /** From da01dfdacd6b979cebb567d0256d463aa6917c0c Mon Sep 17 00:00:00 2001 From: Lucas De Souza Date: Mon, 28 Nov 2022 12:04:28 -0600 Subject: [PATCH 22/32] use minimatch instead of regex Co-authored-by: Zeky Abubaker Signed-off-by: Lucas De Souza --- plugins/catalog-backend/config.d.ts | 6 +++--- plugins/catalog-backend/package.json | 1 + plugins/catalog-backend/src/ingestion/CatalogRules.test.ts | 2 +- plugins/catalog-backend/src/ingestion/CatalogRules.ts | 3 ++- yarn.lock | 1 + 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index 16094af489..40d423d0d4 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -48,7 +48,7 @@ export interface Config { * { "type": "url", "match": "https://github.com/a/*} * */ - location?: Array<{ + locations?: Array<{ /** * The type of location, e.g. "url". */ @@ -59,8 +59,8 @@ export interface Config { */ target?: string; /** - * The target Regex of the location, e.g. - * "https://github.com/org/*. + * The pattern allowed for the location, e.g. + * "https://github.com/org/*\/blob/master/*.yaml. */ match?: string; }>; diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index ea61ca648e..7c26d21a5b 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -59,6 +59,7 @@ "knex": "^2.0.0", "lodash": "^4.17.21", "luxon": "^3.0.0", + "minimatch": "^5.0.0", "node-fetch": "^2.6.7", "p-limit": "^3.0.2", "prom-client": "^14.0.1", diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts index 6cf8682129..9432f1df1a 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts @@ -228,7 +228,7 @@ describe('DefaultCatalogRulesEnforcer', () => { rules: [ { allow: ['Component'], - locations: [{ type: 'url', match: 'https://github.com/b/*' }], + locations: [{ type: 'url', match: 'https://github.com/b/**' }], }, ], }, diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index 66fe3da54e..3cda33533b 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -18,6 +18,7 @@ import { Config } from '@backstage/config'; import { Entity } from '@backstage/catalog-model'; import path from 'path'; import { LocationSpec } from '@backstage/plugin-catalog-common'; +import minimatch from 'minimatch'; /** * Rules to apply to catalog entities. @@ -172,7 +173,7 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { if (matcher.target && matcher.target !== location?.target) { continue; } - if (matcher.match && !location?.target.match(matcher.match)) { + if (matcher.match && !minimatch(location?.target, matcher.match, { nocase: true })) { continue; } return true; diff --git a/yarn.lock b/yarn.lock index 7537c367f6..49f1f4d5d4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5236,6 +5236,7 @@ __metadata: knex: ^2.0.0 lodash: ^4.17.21 luxon: ^3.0.0 + minimatch: ^5.0.0 msw: ^0.49.0 node-fetch: ^2.6.7 p-limit: ^3.0.2 From ac87547571f7055446e94ebf8ed0377f0a78cf7f Mon Sep 17 00:00:00 2001 From: Lucas De Souza Date: Mon, 28 Nov 2022 12:06:10 -0600 Subject: [PATCH 23/32] update regex to pattern on test name Co-authored-by: Zeky Abubaker Signed-off-by: Lucas De Souza --- plugins/catalog-backend/src/ingestion/CatalogRules.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts index 9432f1df1a..e87296e0a1 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts @@ -221,7 +221,7 @@ describe('DefaultCatalogRulesEnforcer', () => { expect(enforcer.isAllowed(entity.location, location.z)).toBe(false); }); - it('should only allow locations that match a given regex', () => { + it('should only allow locations that match a given pattern', () => { const enforcer = DefaultCatalogRulesEnforcer.fromConfig( new ConfigReader({ catalog: { From b285e53d4a747a666f4d8eb554ee70561d5fb23b Mon Sep 17 00:00:00 2001 From: Lucas De Souza Date: Mon, 28 Nov 2022 12:16:00 -0600 Subject: [PATCH 24/32] cleanup Co-authored-by: Zeky Abubaker Signed-off-by: Lucas De Souza --- .../src/ingestion/CatalogRules.ts | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index 3cda33533b..d4eda1db7e 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -96,21 +96,14 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { if (config.has('catalog.rules')) { const globalRules = config .getConfigArray('catalog.rules') - .map(ruleConfig => { - const rule: CatalogRule = { - allow: ruleConfig.getStringArray('allow').map(kind => ({ kind })), - }; - - const locConf = ruleConfig.getOptionalConfigArray('locations'); - if (locConf) - rule.locations = locConf.map(locationConfig => ({ + .map(ruleConf => ({ + allow: ruleConf.getStringArray('allow').map(kind => ({ kind })), + locations: ruleConf.getOptionalConfigArray('locations')?.map(locationConfig => ({ match: locationConfig.getOptionalString('match'), type: locationConfig.getString('type'), - target: locationConfig.getOptionalString('target'), - })); - - return rule; - }); + target: locationConfig.getOptionalString('target') + })) + })); rules.push(...globalRules); } else { rules.push(...DefaultCatalogRulesEnforcer.defaultRules); From 5cb15b7370927cf081c0bbacf0c869f00e19679d Mon Sep 17 00:00:00 2001 From: Lucas De Souza Date: Mon, 28 Nov 2022 12:19:21 -0600 Subject: [PATCH 25/32] prettier Co-authored-by: Zeky Abubaker Signed-off-by: Lucas De Souza --- .../src/ingestion/CatalogRules.ts | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index d4eda1db7e..3428eeaabc 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -97,13 +97,15 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { const globalRules = config .getConfigArray('catalog.rules') .map(ruleConf => ({ - allow: ruleConf.getStringArray('allow').map(kind => ({ kind })), - locations: ruleConf.getOptionalConfigArray('locations')?.map(locationConfig => ({ + allow: ruleConf.getStringArray('allow').map(kind => ({ kind })), + locations: ruleConf + .getOptionalConfigArray('locations') + ?.map(locationConfig => ({ match: locationConfig.getOptionalString('match'), type: locationConfig.getString('type'), - target: locationConfig.getOptionalString('target') - })) - })); + target: locationConfig.getOptionalString('target'), + })), + })); rules.push(...globalRules); } else { rules.push(...DefaultCatalogRulesEnforcer.defaultRules); @@ -166,7 +168,10 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { if (matcher.target && matcher.target !== location?.target) { continue; } - if (matcher.match && !minimatch(location?.target, matcher.match, { nocase: true })) { + if ( + matcher.match && + !minimatch(location?.target, matcher.match, { nocase: true }) + ) { continue; } return true; From cdb670ce5270f8171792eb0aa8d163dcd66090ea Mon Sep 17 00:00:00 2001 From: Lucas De Souza Date: Mon, 28 Nov 2022 14:43:55 -0600 Subject: [PATCH 26/32] ensure that match and target are not both used Co-authored-by: Zeky Abubaker Signed-off-by: Lucas De Souza --- .../src/ingestion/CatalogRules.test.ts | 22 +++++++++++++++++++ .../src/ingestion/CatalogRules.ts | 18 ++++++++++----- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts index e87296e0a1..06bb1389a9 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts @@ -54,6 +54,28 @@ const location: Record = { }; describe('DefaultCatalogRulesEnforcer', () => { + it('should throw an error if both match and target are used', () => { + expect(() => + DefaultCatalogRulesEnforcer.fromConfig( + new ConfigReader({ + catalog: { + rules: [ + { + allow: ['Component'], + locations: [ + { + type: 'url', + match: 'https://github.com/b/**', + target: 'https://github.com/a/b/blob/master/w.yaml', + }, + ], + }, + ], + }, + }), + ), + ).toThrow(/cannot have both target and match values/i); + }); it('should deny by default', () => { const enforcer = new DefaultCatalogRulesEnforcer([]); expect(enforcer.isAllowed(entity.user, location.x)).toBe(false); diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index 3428eeaabc..38561e0970 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -100,11 +100,19 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { allow: ruleConf.getStringArray('allow').map(kind => ({ kind })), locations: ruleConf .getOptionalConfigArray('locations') - ?.map(locationConfig => ({ - match: locationConfig.getOptionalString('match'), - type: locationConfig.getString('type'), - target: locationConfig.getOptionalString('target'), - })), + ?.map(locationConfig => { + const location = { + match: locationConfig.getOptionalString('match'), + type: locationConfig.getString('type'), + target: locationConfig.getOptionalString('target'), + }; + if (location.match && location.target) { + throw new Error( + 'A catalog rule location cannot have both target and match values', + ); + } + return location; + }), })); rules.push(...globalRules); } else { From 0b153881a72a1ed5c6e8c82e22ac1fd3e3f5f0cd Mon Sep 17 00:00:00 2001 From: Lucas De Souza Date: Wed, 30 Nov 2022 09:42:42 -0600 Subject: [PATCH 27/32] updating terms Signed-off-by: Lucas De Souza --- plugins/catalog-backend/config.d.ts | 11 ++++---- .../src/ingestion/CatalogRules.test.ts | 12 ++++---- .../src/ingestion/CatalogRules.ts | 28 +++++++++++-------- 3 files changed, 29 insertions(+), 22 deletions(-) diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index 40d423d0d4..5da7dd8df9 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -42,11 +42,12 @@ export interface Config { * Limit this rule to a specific location * * Example with a fixed location - * { "type": "url", "target": "https://github.com/a/b/blob/file.yaml} + * { "type": "url", "exact": "https://github.com/a/b/blob/file.yaml"} * * Example using a Regex - * { "type": "url", "match": "https://github.com/a/*} + * { "type": "url", "pattern": "https://github.com/org/*\/blob/master/*.yaml"} * + * Using both exact and pattern will result in an error starting the application */ locations?: Array<{ /** @@ -54,15 +55,15 @@ export interface Config { */ type: string; /** - * The target URL of the location, e.g. + * The exact location, e.g. * "https://github.com/org/repo/blob/master/users.yaml". */ - target?: string; + exact?: string; /** * The pattern allowed for the location, e.g. * "https://github.com/org/*\/blob/master/*.yaml. */ - match?: string; + pattern?: string; }>; }>; diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts index 06bb1389a9..34e89fad29 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.test.ts @@ -54,7 +54,7 @@ const location: Record = { }; describe('DefaultCatalogRulesEnforcer', () => { - it('should throw an error if both match and target are used', () => { + it('should throw an error if both pattern and exact are used', () => { expect(() => DefaultCatalogRulesEnforcer.fromConfig( new ConfigReader({ @@ -65,8 +65,8 @@ describe('DefaultCatalogRulesEnforcer', () => { locations: [ { type: 'url', - match: 'https://github.com/b/**', - target: 'https://github.com/a/b/blob/master/w.yaml', + pattern: 'https://github.com/b/**', + exact: 'https://github.com/a/b/blob/master/w.yaml', }, ], }, @@ -74,7 +74,7 @@ describe('DefaultCatalogRulesEnforcer', () => { }, }), ), - ).toThrow(/cannot have both target and match values/i); + ).toThrow(/cannot have both exact and pattern values/i); }); it('should deny by default', () => { const enforcer = new DefaultCatalogRulesEnforcer([]); @@ -250,7 +250,9 @@ describe('DefaultCatalogRulesEnforcer', () => { rules: [ { allow: ['Component'], - locations: [{ type: 'url', match: 'https://github.com/b/**' }], + locations: [ + { type: 'url', pattern: 'https://github.com/b/**' }, + ], }, ], }, diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index 38561e0970..9f06a8f0d7 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -30,9 +30,9 @@ export type CatalogRule = { kind: string; }>; locations?: Array<{ - target?: string; + exact?: string; type: string; - match?: string; + pattern?: string; }>; }; @@ -78,6 +78,10 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { * catalog: * rules: * - allow: [Component, API] + * - allow: [Template] + * locations: + * - type: url + * pattern: https://github.com/org/*\/blob/master/template.yaml * * locations: * - type: url @@ -102,13 +106,13 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { .getOptionalConfigArray('locations') ?.map(locationConfig => { const location = { - match: locationConfig.getOptionalString('match'), + pattern: locationConfig.getOptionalString('pattern'), type: locationConfig.getString('type'), - target: locationConfig.getOptionalString('target'), + exact: locationConfig.getOptionalString('exact'), }; - if (location.match && location.target) { + if (location.pattern && location.exact) { throw new Error( - 'A catalog rule location cannot have both target and match values', + 'A catalog rule location cannot have both exact and pattern values', ); } return location; @@ -127,11 +131,11 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { return []; } const type = locConf.getString('type'); - const target = resolveTarget(type, locConf.getString('target')); + const exact = resolveTarget(type, locConf.getString('target')); return locConf.getConfigArray('rules').map(ruleConf => ({ allow: ruleConf.getStringArray('allow').map(kind => ({ kind })), - locations: [{ type, target }], + locations: [{ type, exact }], })); }); @@ -163,7 +167,7 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { private matchLocation( location: LocationSpec, - matchers?: { target?: string; type: string; match?: string }[], + matchers?: { exact?: string; type: string; pattern?: string }[], ): boolean { if (!matchers) { return true; @@ -173,12 +177,12 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { if (matcher.type !== location?.type) { continue; } - if (matcher.target && matcher.target !== location?.target) { + if (matcher.exact && matcher.exact !== location?.target) { continue; } if ( - matcher.match && - !minimatch(location?.target, matcher.match, { nocase: true }) + matcher.pattern && + !minimatch(location?.target, matcher.pattern, { nocase: true }) ) { continue; } From 446d93f4aff862bba5db961890ebecfb3d176c61 Mon Sep 17 00:00:00 2001 From: Lucas De Souza Date: Wed, 30 Nov 2022 11:22:28 -0600 Subject: [PATCH 28/32] adding another example Co-authored-by: Zeky Abubaker Signed-off-by: Lucas De Souza --- plugins/catalog-backend/src/ingestion/CatalogRules.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plugins/catalog-backend/src/ingestion/CatalogRules.ts b/plugins/catalog-backend/src/ingestion/CatalogRules.ts index 9f06a8f0d7..677649d4b2 100644 --- a/plugins/catalog-backend/src/ingestion/CatalogRules.ts +++ b/plugins/catalog-backend/src/ingestion/CatalogRules.ts @@ -82,6 +82,10 @@ export class DefaultCatalogRulesEnforcer implements CatalogRulesEnforcer { * locations: * - type: url * pattern: https://github.com/org/*\/blob/master/template.yaml + * - allow: [Location] + * locations: + * - type: url + * pattern: https://github.com/org/repo/blob/master/location.yaml * * locations: * - type: url From a3f4718b9269acd64d92ed4c0354a9174682c667 Mon Sep 17 00:00:00 2001 From: Justin De Burgo <57914589+jpdeburgo@users.noreply.github.com> Date: Tue, 6 Dec 2022 12:27:54 -0600 Subject: [PATCH 29/32] Update .changeset/breezy-apes-mate.md Co-authored-by: Patrik Oldsberg Signed-off-by: Justin De Burgo <57914589+jpdeburgo@users.noreply.github.com> --- .changeset/breezy-apes-mate.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/breezy-apes-mate.md b/.changeset/breezy-apes-mate.md index ac3409bd69..65e7dc2bc4 100644 --- a/.changeset/breezy-apes-mate.md +++ b/.changeset/breezy-apes-mate.md @@ -2,4 +2,4 @@ '@backstage/plugin-catalog-backend': patch --- -Adding an optional restriction for locations added through catalog import. Restrictions can be added using the app-config.yaml +Added a new `catalog.rules[].location` configuration that makes it possible to configure catalog rules to only apply to specific locations, either via exact match or a glob pattern. From 3ff751cbbbe4a687925e9b0dc7f6b91d137af963 Mon Sep 17 00:00:00 2001 From: Justin De Burgo <57914589+jpdeburgo@users.noreply.github.com> Date: Tue, 6 Dec 2022 12:28:10 -0600 Subject: [PATCH 30/32] Update plugins/catalog-backend/config.d.ts Co-authored-by: Patrik Oldsberg Signed-off-by: Justin De Burgo <57914589+jpdeburgo@users.noreply.github.com> --- plugins/catalog-backend/config.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index 5da7dd8df9..5f0f8f1ea2 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -61,7 +61,7 @@ export interface Config { exact?: string; /** * The pattern allowed for the location, e.g. - * "https://github.com/org/*\/blob/master/*.yaml. + * "https://github.com/org/*/blob/master/*.yaml. */ pattern?: string; }>; From 5a17cc9b41a5e591b9e4e6834fb66184a54d9d7d Mon Sep 17 00:00:00 2001 From: Justin De Burgo Date: Fri, 9 Dec 2022 11:22:12 -0600 Subject: [PATCH 31/32] adding documentation for exact locations Signed-off-by: Justin De Burgo --- plugins/catalog-backend/config.d.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index 5f0f8f1ea2..cff8ee71c3 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -57,11 +57,15 @@ export interface Config { /** * The exact location, e.g. * "https://github.com/org/repo/blob/master/users.yaml". + * + * The exact location can also be used to match on locations + * that contain glob characters themselves, e.g. + * "https://github.com/org/*\/blob/master/*.yaml". */ exact?: string; /** * The pattern allowed for the location, e.g. - * "https://github.com/org/*/blob/master/*.yaml. + * "https://github.com/org/*\/blob/master/*.yaml". */ pattern?: string; }>; From 30d0090e26108ba91f831edbf7363853c7457c5c Mon Sep 17 00:00:00 2001 From: Lucas De Souza Date: Mon, 12 Dec 2022 09:17:51 -0600 Subject: [PATCH 32/32] prettier Signed-off-by: Lucas De Souza --- plugins/catalog-backend/config.d.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend/config.d.ts b/plugins/catalog-backend/config.d.ts index cff8ee71c3..5c5bfea753 100644 --- a/plugins/catalog-backend/config.d.ts +++ b/plugins/catalog-backend/config.d.ts @@ -57,8 +57,8 @@ export interface Config { /** * The exact location, e.g. * "https://github.com/org/repo/blob/master/users.yaml". - * - * The exact location can also be used to match on locations + * + * The exact location can also be used to match on locations * that contain glob characters themselves, e.g. * "https://github.com/org/*\/blob/master/*.yaml". */