diff --git a/.changeset/heavy-cameras-provide.md b/.changeset/heavy-cameras-provide.md new file mode 100644 index 0000000000..1d86f20592 --- /dev/null +++ b/.changeset/heavy-cameras-provide.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-app-api': patch +--- + +Updated the default error handling middleware to filter out certain known error types that should never be returned in responses. The errors are instead logged along with a correlation ID, which is also returned in the response. Initially only PostgreSQL protocol errors from the `pg-protocol` package are filtered out. diff --git a/.changeset/real-keys-juggle.md b/.changeset/real-keys-juggle.md new file mode 100644 index 0000000000..55bdbf4db3 --- /dev/null +++ b/.changeset/real-keys-juggle.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-azure': patch +--- + +Fixed issue where specifying a branch for discovery did not work diff --git a/.changeset/thirty-bags-try.md b/.changeset/thirty-bags-try.md new file mode 100644 index 0000000000..98d6c5620b --- /dev/null +++ b/.changeset/thirty-bags-try.md @@ -0,0 +1,33 @@ +--- +'@backstage/plugin-catalog-backend': minor +'@backstage/plugin-catalog-node': minor +--- + +Adds support for supplying field validators to the new backend's catalog plugin. If you're using entity policies, you should use the new `transformLegacyPolicyToProcessor` function to install them as processors instead. + +```ts +import { + catalogProcessingExtensionPoint, + catalogModelExtensionPoint, +} from '@backstage/plugin-catalog-node/alpha'; +import {myPolicy} from './my-policy'; + +export const catalogModulePolicyProvider = createBackendModule({ + pluginId: 'catalog', + moduleId: 'internal-policy-provider', + register(reg) { + reg.registerInit({ + deps: { + modelExtensions: catalogModelExtensionPoint, + processingExtensions: catalogProcessingExtensionPoint, + }, + async init({ modelExtensions, processingExtensions }) { + modelExtensions.setFieldValidators({ + ... + }); + processingExtensions.addProcessors(transformLegacyPolicyToProcessor(myPolicy)) + }, + }); + }, +}); +``` diff --git a/.changeset/weak-flowers-repeat.md b/.changeset/weak-flowers-repeat.md new file mode 100644 index 0000000000..d1705343bc --- /dev/null +++ b/.changeset/weak-flowers-repeat.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-scaffolder-backend-module-gitea': patch +--- + +- Fix issue for infinite loop when repository already exists +- Log the root cause of error reported by `checkGiteaOrg` diff --git a/docs/features/software-templates/writing-custom-field-extensions.md b/docs/features/software-templates/writing-custom-field-extensions.md index f22cecd22c..d53b8bdf2f 100644 --- a/docs/features/software-templates/writing-custom-field-extensions.md +++ b/docs/features/software-templates/writing-custom-field-extensions.md @@ -30,7 +30,8 @@ As an example, we will create a component that validates whether a string is in ```tsx //packages/app/src/scaffolder/ValidateKebabCase/ValidateKebabCaseExtension.tsx import React from 'react'; -import { FieldProps, FieldValidation } from '@rjsf/core'; +import { FieldExtensionComponentProps } from '@backstage/plugin-scaffolder-react'; +import type { FieldValidation } from '@rjsf/utils'; import FormControl from '@material-ui/core/FormControl'; /* This is the actual component that will get rendered in the form @@ -40,7 +41,7 @@ export const ValidateKebabCase = ({ rawErrors, required, formData, -}: FieldProps) => { +}: FieldExtensionComponentProps) => { return ( { ); }); + it('should filter out internal errors', async () => { + const app = express(); + + const grandChildLogger = { + error: jest.fn(), + }; + childLogger.child.mockReturnValue(grandChildLogger); + + class DatabaseError extends Error {} + const thrownError = new DatabaseError('some error'); + + app.use('/breaks', () => { + throw thrownError; + }); + app.use(middleware.error()); + + await request(app).get('/breaks'); + + const [{ logId }] = childLogger.child.mock.calls[0]; + + expect(logId).toMatch(/^[0-9a-f]+$/); + expect(childLogger.error).toHaveBeenCalledWith( + 'Request failed with status 500', + expect.objectContaining({ + message: expect.stringMatching( + `An internal error occurred logId=${logId}`, + ), + }), + ); + expect(grandChildLogger.error).toHaveBeenCalledWith( + expect.stringMatching( + `Filtered internal error with logId=${logId} from response`, + ), + thrownError, + ); + }); + it('does not log 400 errors', async () => { const app = express(); diff --git a/packages/backend-app-api/src/http/MiddlewareFactory.ts b/packages/backend-app-api/src/http/MiddlewareFactory.ts index bd78d77045..77136fd7e5 100644 --- a/packages/backend-app-api/src/http/MiddlewareFactory.ts +++ b/packages/backend-app-api/src/http/MiddlewareFactory.ts @@ -43,6 +43,7 @@ import { serializeError, } from '@backstage/errors'; import { NotImplementedError } from '@backstage/errors'; +import { applyInternalErrorFilter } from './applyInternalErrorFilter'; /** * Options used to create a {@link MiddlewareFactory}. @@ -209,7 +210,14 @@ export class MiddlewareFactory { type: 'errorHandler', }); - return (error: Error, req: Request, res: Response, next: NextFunction) => { + return ( + rawError: Error, + req: Request, + res: Response, + next: NextFunction, + ) => { + const error = applyInternalErrorFilter(rawError, logger); + const statusCode = getStatusCode(error); if (options.logAllErrors || statusCode >= 500) { logger.error(`Request failed with status ${statusCode}`, error); diff --git a/packages/backend-app-api/src/http/applyInternalErrorFilter.ts b/packages/backend-app-api/src/http/applyInternalErrorFilter.ts new file mode 100644 index 0000000000..d1d1e0e4d9 --- /dev/null +++ b/packages/backend-app-api/src/http/applyInternalErrorFilter.ts @@ -0,0 +1,55 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LoggerService } from '@backstage/backend-plugin-api'; +import { assertError } from '@backstage/errors'; +import { randomBytes } from 'crypto'; + +function handleBadError(error: Error, logger: LoggerService) { + const logId = randomBytes(10).toString('hex'); + logger + .child({ logId }) + .error(`Filtered internal error with logId=${logId} from response`, error); + const newError = new Error(`An internal error occurred logId=${logId}`); + delete newError.stack; // Trim the stack since it's not particularly useful + return newError; +} + +/** + * Filters out certain known error types that should never be returned in responses. + * + * @internal + */ +export function applyInternalErrorFilter( + error: unknown, + logger: LoggerService, +): Error { + try { + assertError(error); + } catch (assertionError: unknown) { + assertError(assertionError); + return handleBadError(assertionError, logger); + } + + const constructorName = error.constructor.name; + + // DatabaseError are thrown by the pg-protocol module + if (constructorName === 'DatabaseError') { + return handleBadError(error, logger); + } + + return error; +} diff --git a/packages/backend-common/src/reading/__fixtures__/awsS3/awsS3-mock-object3.yaml b/packages/backend-common/src/reading/__fixtures__/awsS3/awsS3-mock-object3.yaml new file mode 100644 index 0000000000..b622af142a --- /dev/null +++ b/packages/backend-common/src/reading/__fixtures__/awsS3/awsS3-mock-object3.yaml @@ -0,0 +1 @@ +site_name: Test3 diff --git a/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts b/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts index e2bfe1af59..7420f14d70 100644 --- a/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts +++ b/packages/backend-common/src/reading/tree/ReadableArrayResponse.test.ts @@ -31,6 +31,12 @@ const file2 = fs.readFileSync( path.resolve(__filename, '../../__fixtures__/awsS3/awsS3-mock-object2.yaml'), ); +const dir1 = 'dir1'; +const name3 = `file3.yaml`; +const file3 = fs.readFileSync( + path.resolve(__filename, '../../__fixtures__/awsS3/awsS3-mock-object3.yaml'), +); + describe('ReadableArrayResponse', () => { const sourceDir = createMockDirectory(); const targetDir = createMockDirectory(); @@ -39,6 +45,9 @@ describe('ReadableArrayResponse', () => { sourceDir.setContent({ [name1]: file1, [name2]: file2, + [dir1]: { + [name3]: file3, + }, }); targetDir.clear(); }); @@ -56,11 +65,13 @@ describe('ReadableArrayResponse', () => { const path1 = sourceDir.resolve(name1); const path2 = sourceDir.resolve(name2); + const path3 = sourceDir.resolve(`${dir1}/${name3}`); it('should read files', async () => { const arr: FromReadableArrayOptions = [ { data: createReadStream(path1), path: path1 }, { data: createReadStream(path2), path: path2 }, + { data: createReadStream(path3), path: path3 }, ]; const res = new ReadableArrayResponse(arr, targetDir.path, 'etag'); @@ -69,18 +80,21 @@ describe('ReadableArrayResponse', () => { expect(files).toEqual([ { path: path1, content: expect.any(Function) }, { path: path2, content: expect.any(Function) }, + { path: path3, content: expect.any(Function) }, ]); const contents = await Promise.all(files.map(f => f.content())); - expect(contents).toEqual([file1, file2]); + expect(contents).toEqual([file1, file2, file3]); }); it('should extract entire archive into directory', async () => { const relativePath1 = relative(sourceDir.path, path1); const relativePath2 = relative(sourceDir.path, path2); + const relativePath3 = relative(sourceDir.path, path3); const arr: FromReadableArrayOptions = [ { data: createReadStream(path1), path: relativePath1 }, { data: createReadStream(path2), path: relativePath2 }, + { data: createReadStream(path3), path: relativePath3 }, ]; const res = new ReadableArrayResponse(arr, targetDir.path, 'etag'); @@ -89,6 +103,9 @@ describe('ReadableArrayResponse', () => { expect(targetDir.content({ path: dir })).toEqual({ [name1]: file1.toString('utf8'), [name2]: file2.toString('utf8'), + [dir1]: { + [name3]: file3.toString('utf8'), + }, }); }); }); diff --git a/plugins/catalog-backend-module-azure/src/lib/azure.test.ts b/plugins/catalog-backend-module-azure/src/lib/azure.test.ts index 493f0dda51..ef00610dc0 100644 --- a/plugins/catalog-backend-module-azure/src/lib/azure.test.ts +++ b/plugins/catalog-backend-module-azure/src/lib/azure.test.ts @@ -89,6 +89,7 @@ describe('azure', () => { 'engineering', '', '/catalog-info.yaml', + '', ), ).resolves.toEqual([]); }); @@ -154,6 +155,7 @@ describe('azure', () => { 'engineering', '', '/catalog-info.yaml', + '', ), ).resolves.toEqual(response.results); }); @@ -210,6 +212,67 @@ describe('azure', () => { 'engineering', 'backstage', '/catalog-info.yaml', + '', + ), + ).resolves.toEqual(response.results); + }); + + it('searches in specific branch if parameter is set', async () => { + const response: CodeSearchResponse = { + count: 1, + results: [ + { + fileName: 'catalog-info.yaml', + path: '/catalog-info.yaml', + project: { + name: '*', + }, + repository: { + name: 'backstage', + }, + }, + ], + }; + + server.use( + rest.post( + `https://almsearch.dev.azure.com/shopify/_apis/search/codesearchresults`, + (req, res, ctx) => { + expect(req.headers.get('Authorization')).toBe('Basic OkFCQw=='); + expect(req.body).toEqual({ + searchText: + 'path:/catalog-info.yaml repo:backstage proj:engineering', + $orderBy: [ + { + field: 'path', + sortOrder: 'ASC', + }, + ], + $skip: 0, + $top: 1000, + filters: { + Branch: ['topic/catalog-info'], + }, + }); + return res(ctx.json(response)); + }, + ), + ); + + const { credentialsProvider, azureConfig } = createFixture( + 'dev.azure.com', + 'ABC', + ); + + await expect( + codeSearch( + credentialsProvider, + azureConfig, + 'shopify', + 'engineering', + 'backstage', + '/catalog-info.yaml', + 'topic/catalog-info', ), ).resolves.toEqual(response.results); }); @@ -265,6 +328,7 @@ describe('azure', () => { 'engineering', '', '/catalog-info.yaml', + '', ), ).resolves.toEqual(response.results); }); @@ -324,6 +388,7 @@ describe('azure', () => { 'engineering', 'backstage', '/catalog-info.yaml', + '', ), ).resolves.toHaveLength(totalCount); }); diff --git a/plugins/catalog-backend-module-azure/src/lib/azure.ts b/plugins/catalog-backend-module-azure/src/lib/azure.ts index c8ff8dd739..3bdff90110 100644 --- a/plugins/catalog-backend-module-azure/src/lib/azure.ts +++ b/plugins/catalog-backend-module-azure/src/lib/azure.ts @@ -37,6 +37,16 @@ export interface CodeSearchResultItem { branch?: string; } +interface CodeSearchRequest { + searchText: string; + $orderBy: Array<{ field: string; sortOrder: string }>; + $skip: number; + $top: number; + filters?: { + Branch: string[]; + }; +} + const isCloud = (host: string) => host === 'dev.azure.com'; const PAGE_SIZE = 1000; @@ -48,6 +58,7 @@ export async function codeSearch( project: string, repo: string, path: string, + branch: string, ): Promise { const searchBaseUrl = isCloud(azureConfig.host) ? 'https://almsearch.dev.azure.com' @@ -62,23 +73,29 @@ export async function codeSearch( url: `https://${azureConfig.host}/${org}`, }); + const searchRequestBody: CodeSearchRequest = { + searchText: `path:${path} repo:${repo || '*'} proj:${project || '*'}`, + $orderBy: [ + { + field: 'path', + sortOrder: 'ASC', + }, + ], + $skip: items.length, + $top: PAGE_SIZE, + }; + + if (branch) { + searchRequestBody.filters = { Branch: [branch] }; + } + const response = await fetch(searchUrl, { headers: { ...credentials?.headers, 'Content-Type': 'application/json', }, method: 'POST', - body: JSON.stringify({ - searchText: `path:${path} repo:${repo || '*'} proj:${project || '*'}`, - $orderBy: [ - { - field: 'path', - sortOrder: 'ASC', - }, - ], - $skip: items.length, - $top: PAGE_SIZE, - }), + body: JSON.stringify(searchRequestBody), }); if (response.status !== 200) { diff --git a/plugins/catalog-backend-module-azure/src/processors/AzureDevOpsDiscoveryProcessor.test.ts b/plugins/catalog-backend-module-azure/src/processors/AzureDevOpsDiscoveryProcessor.test.ts index 36dd920621..6df555f190 100644 --- a/plugins/catalog-backend-module-azure/src/processors/AzureDevOpsDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend-module-azure/src/processors/AzureDevOpsDiscoveryProcessor.test.ts @@ -35,6 +35,7 @@ describe('AzureDevOpsDiscoveryProcessor', () => { project: 'my-proj', repo: '', catalogPath: '/catalog-info.yaml', + branch: '', }); expect( @@ -47,6 +48,7 @@ describe('AzureDevOpsDiscoveryProcessor', () => { project: 'engineering', repo: 'backstage', catalogPath: '/catalog.yaml', + branch: '', }); expect( @@ -59,6 +61,33 @@ describe('AzureDevOpsDiscoveryProcessor', () => { project: 'engineering', repo: 'backstage', catalogPath: '/src/*/catalog.yaml', + branch: '', + }); + + expect( + parseUrl( + 'https://azuredevops.mycompany.com/spotify/engineering/_git/backstage?path=/src/*/catalog.yaml&version=GBtopic/catalog-info', + ), + ).toEqual({ + baseUrl: 'https://azuredevops.mycompany.com', + org: 'spotify', + project: 'engineering', + repo: 'backstage', + catalogPath: '/src/*/catalog.yaml', + branch: 'topic/catalog-info', + }); + + expect( + parseUrl( + 'https://azuredevops.mycompany.com/spotify/engineering/_git/backstage?path=/src/*/catalog.yaml&version=GBtopic%2Fcatalog-info', + ), + ).toEqual({ + baseUrl: 'https://azuredevops.mycompany.com', + org: 'spotify', + project: 'engineering', + repo: 'backstage', + catalogPath: '/src/*/catalog.yaml', + branch: 'topic/catalog-info', }); }); @@ -164,6 +193,7 @@ describe('AzureDevOpsDiscoveryProcessor', () => { 'engineering', '', '/catalog-info.yaml', + '', ); expect(emitter).toHaveBeenCalledTimes(2); expect(emitter).toHaveBeenCalledWith({ @@ -214,6 +244,7 @@ describe('AzureDevOpsDiscoveryProcessor', () => { 'engineering', 'backstage', '/catalog-info.yaml', + '', ); expect(emitter).toHaveBeenCalledTimes(1); expect(emitter).toHaveBeenCalledWith({ @@ -227,6 +258,49 @@ describe('AzureDevOpsDiscoveryProcessor', () => { }); }); + it('output locations with branch if specified in target', async () => { + const location: LocationSpec = { + type: 'azure-discovery', + target: + 'https://dev.azure.com/shopify/engineering/_git/backstage?path=/catalog-info.yaml&version=GBtopic/catalog-info', + }; + mockCodeSearch.mockResolvedValueOnce([ + { + fileName: 'catalog-info.yaml', + path: '/catalog-info.yaml', + repository: { + name: 'backstage', + }, + project: { + name: '*', + }, + }, + ]); + const emitter = jest.fn(); + + await processor.readLocation(location, false, emitter); + + expect(mockCodeSearch).toHaveBeenCalledWith( + expect.anything(), + { host: 'dev.azure.com' }, + 'shopify', + 'engineering', + 'backstage', + '/catalog-info.yaml', + 'topic/catalog-info', + ); + expect(emitter).toHaveBeenCalledTimes(1); + expect(emitter).toHaveBeenCalledWith({ + type: 'location', + location: { + type: 'url', + target: + 'https://dev.azure.com/shopify/engineering/_git/backstage?path=/catalog-info.yaml&version=GBtopic/catalog-info', + presence: 'optional', + }, + }); + }); + it('output single locations with different file name from code search', async () => { const location: LocationSpec = { type: 'azure-discovery', @@ -256,6 +330,7 @@ describe('AzureDevOpsDiscoveryProcessor', () => { 'engineering', '', '/src/*/catalog.yaml', + '', ); expect(emitter).toHaveBeenCalledTimes(1); expect(emitter).toHaveBeenCalledWith({ @@ -286,6 +361,7 @@ describe('AzureDevOpsDiscoveryProcessor', () => { 'engineering', 'backstage', '/catalog-info.yaml', + '', ); expect(emitter).not.toHaveBeenCalled(); }); diff --git a/plugins/catalog-backend-module-azure/src/processors/AzureDevOpsDiscoveryProcessor.ts b/plugins/catalog-backend-module-azure/src/processors/AzureDevOpsDiscoveryProcessor.ts index d9638e6cc7..ddb3c5f59f 100644 --- a/plugins/catalog-backend-module-azure/src/processors/AzureDevOpsDiscoveryProcessor.ts +++ b/plugins/catalog-backend-module-azure/src/processors/AzureDevOpsDiscoveryProcessor.ts @@ -92,7 +92,7 @@ export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { ); } - const { baseUrl, org, project, repo, catalogPath } = parseUrl( + const { baseUrl, org, project, repo, catalogPath, branch } = parseUrl( location.target, ); this.logger.info( @@ -106,6 +106,7 @@ export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { project, repo, catalogPath, + branch, ); this.logger.debug( @@ -113,10 +114,16 @@ export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { ); for (const file of files) { + let target = `${baseUrl}/${org}/${project}/_git/${file.repository.name}?path=${file.path}`; + + if (branch) { + target += `&version=GB${branch}`; + } + emit( processingResult.location({ type: 'url', - target: `${baseUrl}/${org}/${project}/_git/${file.repository.name}?path=${file.path}`, + target, // Not all locations may actually exist, since the user defined them as a wildcard pattern. // Thus, we emit them as optional and let the downstream processor find them while not outputting // an error if it couldn't. @@ -138,11 +145,18 @@ export function parseUrl(urlString: string): { project: string; repo: string; catalogPath: string; + branch: string; } { const url = new URL(urlString); const path = url.pathname.slice(1).split('/'); const catalogPath = url.searchParams.get('path') || '/catalog-info.yaml'; + let branch = url.searchParams.get('version') || ''; + + if (branch.startsWith('GB')) { + // DevOps prefixes branch names with 'GB' in URLs + branch = branch.slice(2); + } if (path.length === 2 && path[0].length && path[1].length) { return { @@ -151,6 +165,7 @@ export function parseUrl(urlString: string): { project: decodeURIComponent(path[1]), repo: '', catalogPath, + branch, }; } else if ( path.length === 4 && @@ -165,6 +180,7 @@ export function parseUrl(urlString: string): { project: decodeURIComponent(path[1]), repo: decodeURIComponent(path[3]), catalogPath, + branch, }; } diff --git a/plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.ts b/plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.ts index f56369a0d1..44e618ffcd 100644 --- a/plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.ts +++ b/plugins/catalog-backend-module-azure/src/providers/AzureDevOpsEntityProvider.ts @@ -158,6 +158,7 @@ export class AzureDevOpsEntityProvider implements EntityProvider { this.config.project, this.config.repository, this.config.path, + this.config.branch || '', ); logger.info(`Discovered ${files.length} catalog files`); diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index a38a964d48..92a2fbb280 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -450,6 +450,11 @@ export const processingResult: Readonly<{ // @public @deprecated (undocumented) export type ScmLocationAnalyzer = ScmLocationAnalyzer_2; +// @public +export function transformLegacyPolicyToProcessor( + policy: EntityPolicy, +): CatalogProcessor_2; + // @public (undocumented) export class UrlReaderProcessor implements CatalogProcessor_2 { constructor(options: { reader: UrlReader; logger: Logger }); diff --git a/plugins/catalog-backend/src/modules/core/index.ts b/plugins/catalog-backend/src/modules/core/index.ts index 365def0bfe..e410c3824d 100644 --- a/plugins/catalog-backend/src/modules/core/index.ts +++ b/plugins/catalog-backend/src/modules/core/index.ts @@ -24,3 +24,4 @@ export { PlaceholderProcessor } from './PlaceholderProcessor'; export type { PlaceholderProcessorOptions } from './PlaceholderProcessor'; export { UrlReaderProcessor } from './UrlReaderProcessor'; export { parseEntityYaml } from '../util/parse'; +export { transformLegacyPolicyToProcessor } from './transformLegacyPolicyToProcessor'; diff --git a/plugins/catalog-backend/src/modules/core/transformLegacyPolicyToProcessor.test.ts b/plugins/catalog-backend/src/modules/core/transformLegacyPolicyToProcessor.test.ts new file mode 100644 index 0000000000..a38074723c --- /dev/null +++ b/plugins/catalog-backend/src/modules/core/transformLegacyPolicyToProcessor.test.ts @@ -0,0 +1,90 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity, EntityPolicy } from '@backstage/catalog-model'; +import { transformLegacyPolicyToProcessor } from './transformLegacyPolicyToProcessor'; +import { clone } from 'lodash'; + +describe('transformLegacyPolicyToProcessor', () => { + const entityToProcess: Entity = { + apiVersion: 'backstage.io/v1alpha', + kind: 'Component', + metadata: { + name: 'test', + }, + }; + it('modifies the entity if the policy modifies the entity', async () => { + const policy: EntityPolicy = { + async enforce(entity) { + entity.kind = 'Group'; + return entity; + }, + }; + const processor = transformLegacyPolicyToProcessor(policy); + const clonedEntity = clone(entityToProcess); + const entity = await processor.preProcessEntity?.( + clonedEntity, + {} as any, + jest.fn(), + {} as any, + {} as any, + ); + expect(entity).toBeTruthy(); + expect(entity?.kind).toBe('Group'); + expect(entity?.apiVersion).toBe('backstage.io/v1alpha'); + expect(entity?.metadata.name).toBe('test'); + }); + + it('does not modify the entity if the policy returns undefined', async () => { + const policy: EntityPolicy = { + async enforce() { + return undefined; + }, + }; + const processor = transformLegacyPolicyToProcessor(policy); + const clonedEntity = clone(entityToProcess); + const entity = await processor.preProcessEntity?.( + clonedEntity, + {} as any, + jest.fn(), + {} as any, + {} as any, + ); + expect(entity).toBeTruthy(); + expect(entity?.kind).toBe('Component'); + expect(entity?.apiVersion).toBe('backstage.io/v1alpha'); + expect(entity?.metadata.name).toBe('test'); + }); + + it('bubbles up processor error', async () => { + const policy: EntityPolicy = { + async enforce() { + throw new TypeError('Invalid value for metadata.name'); + }, + }; + const processor = transformLegacyPolicyToProcessor(policy); + const clonedEntity = clone(entityToProcess); + await expect( + processor.preProcessEntity?.( + clonedEntity, + {} as any, + jest.fn(), + {} as any, + {} as any, + ), + ).rejects.toThrow(/Invalid value for metadata.name/); + }); +}); diff --git a/plugins/catalog-backend/src/modules/core/transformLegacyPolicyToProcessor.ts b/plugins/catalog-backend/src/modules/core/transformLegacyPolicyToProcessor.ts new file mode 100644 index 0000000000..40a3e749ab --- /dev/null +++ b/plugins/catalog-backend/src/modules/core/transformLegacyPolicyToProcessor.ts @@ -0,0 +1,42 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { EntityPolicy } from '@backstage/catalog-model'; +import { CatalogProcessor } from '@backstage/plugin-catalog-node'; + +/** + * Transform a given entity policy to an entity processor. + * @param policy - The policy to transform + * @returns A new entity processor that uses the entity policy. + * @public + */ +export function transformLegacyPolicyToProcessor( + policy: EntityPolicy, +): CatalogProcessor { + return { + getProcessorName() { + return policy.constructor.name; + }, + async preProcessEntity(entity) { + // If enforcing the policy fails, throw the policy error. + const result = await policy.enforce(entity); + if (!result) { + return entity; + } + return result; + }, + }; +} diff --git a/plugins/catalog-backend/src/service/CatalogPlugin.ts b/plugins/catalog-backend/src/service/CatalogPlugin.ts index bfe8ef5a16..255df84d58 100644 --- a/plugins/catalog-backend/src/service/CatalogPlugin.ts +++ b/plugins/catalog-backend/src/service/CatalogPlugin.ts @@ -17,7 +17,7 @@ import { createBackendPlugin, coreServices, } from '@backstage/backend-plugin-api'; -import { Entity } from '@backstage/catalog-model'; +import { Entity, Validators } from '@backstage/catalog-model'; import { CatalogBuilder, CatalogPermissionRuleInput } from './CatalogBuilder'; import { CatalogAnalysisExtensionPoint, @@ -26,6 +26,8 @@ import { catalogProcessingExtensionPoint, CatalogPermissionExtensionPoint, catalogPermissionExtensionPoint, + CatalogModelExtensionPoint, + catalogModelExtensionPoint, } from '@backstage/plugin-catalog-node/alpha'; import { CatalogProcessor, @@ -34,6 +36,7 @@ import { ScmLocationAnalyzer, } from '@backstage/plugin-catalog-node'; import { loggerToWinstonLogger } from '@backstage/backend-common'; +import { merge } from 'lodash'; class CatalogProcessingExtensionPointImpl implements CatalogProcessingExtensionPoint @@ -124,6 +127,18 @@ class CatalogPermissionExtensionPointImpl } } +class CatalogModelExtensionPointImpl implements CatalogModelExtensionPoint { + #fieldValidators: Partial = {}; + + setFieldValidators(validators: Partial): void { + merge(this.#fieldValidators, validators); + } + + get fieldValidators() { + return this.#fieldValidators; + } +} + /** * Catalog plugin * @alpha @@ -150,6 +165,9 @@ export const catalogPlugin = createBackendPlugin({ permissionExtensions, ); + const modelExtensions = new CatalogModelExtensionPointImpl(); + env.registerExtensionPoint(catalogModelExtensionPoint, modelExtensions); + env.registerInit({ deps: { logger: coreServices.logger, @@ -192,6 +210,7 @@ export const catalogPlugin = createBackendPlugin({ ); builder.addLocationAnalyzers(...analysisExtensions.locationAnalyzers); builder.addPermissionRules(...permissionExtensions.permissionRules); + builder.setFieldFormatValidators(modelExtensions.fieldValidators); const { processingEngine, router } = await builder.build(); diff --git a/plugins/catalog-node/api-report-alpha.md b/plugins/catalog-node/api-report-alpha.md index 5be4b114c0..e43d1bd49f 100644 --- a/plugins/catalog-node/api-report-alpha.md +++ b/plugins/catalog-node/api-report-alpha.md @@ -14,6 +14,7 @@ import { PermissionRuleParams } from '@backstage/plugin-permission-common'; import { PlaceholderResolver } from '@backstage/plugin-catalog-node'; import { ScmLocationAnalyzer } from '@backstage/plugin-catalog-node'; import { ServiceRef } from '@backstage/backend-plugin-api'; +import { Validators } from '@backstage/catalog-model'; // @alpha (undocumented) export interface CatalogAnalysisExtensionPoint { @@ -24,6 +25,14 @@ export interface CatalogAnalysisExtensionPoint { // @alpha (undocumented) export const catalogAnalysisExtensionPoint: ExtensionPoint; +// @alpha (undocumented) +export interface CatalogModelExtensionPoint { + setFieldValidators(validators: Partial): void; +} + +// @alpha (undocumented) +export const catalogModelExtensionPoint: ExtensionPoint; + // @alpha (undocumented) export interface CatalogPermissionExtensionPoint { // (undocumented) diff --git a/plugins/catalog-node/src/alpha.ts b/plugins/catalog-node/src/alpha.ts index 3e958d1df1..91b5bb99b6 100644 --- a/plugins/catalog-node/src/alpha.ts +++ b/plugins/catalog-node/src/alpha.ts @@ -22,3 +22,5 @@ export { catalogAnalysisExtensionPoint } from './extensions'; export type { CatalogPermissionRuleInput } from './extensions'; export type { CatalogPermissionExtensionPoint } from './extensions'; export { catalogPermissionExtensionPoint } from './extensions'; +export type { CatalogModelExtensionPoint } from './extensions'; +export { catalogModelExtensionPoint } from './extensions'; diff --git a/plugins/catalog-node/src/extensions.ts b/plugins/catalog-node/src/extensions.ts index e2ee1b4f4b..763e934176 100644 --- a/plugins/catalog-node/src/extensions.ts +++ b/plugins/catalog-node/src/extensions.ts @@ -15,7 +15,7 @@ */ import { createExtensionPoint } from '@backstage/backend-plugin-api'; -import { Entity } from '@backstage/catalog-model'; +import { Entity, Validators } from '@backstage/catalog-model'; import { CatalogProcessor, EntitiesSearchFilter, @@ -45,6 +45,18 @@ export interface CatalogProcessingExtensionPoint { ): void; } +/** @alpha */ +export interface CatalogModelExtensionPoint { + /** + * Sets the validator function to use for one or more special fields of an + * entity. This is useful if the default rules for formatting of fields are + * not sufficient. + * + * @param validators - The (subset of) validators to set + */ + setFieldValidators(validators: Partial): void; +} + /** * @alpha */ @@ -68,6 +80,12 @@ export const catalogAnalysisExtensionPoint = id: 'catalog.analysis', }); +/** @alpha */ +export const catalogModelExtensionPoint = + createExtensionPoint({ + id: 'catalog.model', + }); + /** * @alpha */ diff --git a/plugins/scaffolder-backend-module-gitea/README.md b/plugins/scaffolder-backend-module-gitea/README.md index f13602f2d8..a1282634e0 100644 --- a/plugins/scaffolder-backend-module-gitea/README.md +++ b/plugins/scaffolder-backend-module-gitea/README.md @@ -28,8 +28,8 @@ integrations: password: '' ``` -**NOTE**: As backstage will issue HTTPS/TLS requests to the gitea instance, it is needed to configure `gitea` with a valid certificate or at least with a -self-signed certificate `gitea cert --host localhost -ca`. Don't forget to set the env var `NODE_EXTRA_CA_CERTS` to point to the CA file before launching backstage ! +**Important**: As backstage will issue HTTPS/TLS requests to the gitea instance, it is needed to configure `gitea` with a valid certificate or at least with a +self-signed certificate `gitea cert --host localhost -ca` trusted by a CA authority. Don't forget to set the env var `NODE_EXTRA_CA_CERTS` to point to the CA file before launching backstage ! When done, you can create a template which: @@ -69,18 +69,8 @@ spec: action: fetch:template input: url: ./skeleton - copyWithoutTemplating: - - .github/workflows/* values: - component_id: ${{ parameters.component_id }} - namespace: ${{ parameters.component_id }}-dev - description: ${{ parameters.description }} - group_id: ${{ parameters.group_id }} - artifact_id: ${{ parameters.artifact_id }} - java_package_name: ${{ parameters.java_package_name }} - owner: ${{ parameters.owner }} - destination: ${{ (parameters.repoUrl | parseRepoUrl).owner }}/${{ (parameters.repoUrl | parseRepoUrl).repo } - port: 8080 + name: ${{ parameters.name }} - id: publish name: Publishing to a gitea git repository diff --git a/plugins/scaffolder-backend-module-gitea/src/actions/gitea.ts b/plugins/scaffolder-backend-module-gitea/src/actions/gitea.ts index e7f62f51da..089f9b2669 100644 --- a/plugins/scaffolder-backend-module-gitea/src/actions/gitea.ts +++ b/plugins/scaffolder-backend-module-gitea/src/actions/gitea.ts @@ -80,7 +80,9 @@ const checkGiteaOrg = async ( getOptions, ); } catch (e) { - throw new Error(`Unable to get the Organization: ${owner}, ${e}`); + throw new Error( + `Unable to get the Organization: ${owner}; Error cause: ${e.cause.message}`, + ); } if (response.status !== 200) { throw new Error( @@ -178,6 +180,9 @@ async function checkAvailabilityGiteaRepository( if (response.status !== 200) { // Repository is not yet available/accessible ... await sleep(1000); + } else { + // Gitea repository exists ! + break; } } }