From a73f4958401c2f6e6a46331b53dea6403e28a9e1 Mon Sep 17 00:00:00 2001 From: Hellgren Heikki Date: Fri, 1 Aug 2025 15:45:22 +0300 Subject: [PATCH 01/20] feat(config): allow specifying env specific config files this change allows to specify loading of environment specific config files based on `BACKSTAGE_ENVIRONMENT` environment variable. relates to #30716 Signed-off-by: Hellgren Heikki --- .changeset/better-eagles-tickle.md | 5 +++++ docs/conf/index.md | 21 +++++++++++++++---- .../src/sources/ConfigSources.test.ts | 12 +++++++++++ .../src/sources/ConfigSources.ts | 14 +++++++++++++ 4 files changed, 48 insertions(+), 4 deletions(-) create mode 100644 .changeset/better-eagles-tickle.md diff --git a/.changeset/better-eagles-tickle.md b/.changeset/better-eagles-tickle.md new file mode 100644 index 0000000000..b199ae21d7 --- /dev/null +++ b/.changeset/better-eagles-tickle.md @@ -0,0 +1,5 @@ +--- +'@backstage/config-loader': patch +--- + +Allow using `BACKSTAGE_ENVIRONMENT` for loading environment specific config files diff --git a/docs/conf/index.md b/docs/conf/index.md index 659ffccfe3..7c5be174e5 100644 --- a/docs/conf/index.md +++ b/docs/conf/index.md @@ -16,10 +16,23 @@ allowing for customization. ## Supplying Configuration Configuration is stored in YAML files where the defaults are `app-config.yaml` -and `app-config.local.yaml` for local overrides. Other sets of files can by -loaded by passing `--config ` flags. The configuration files themselves -contain plain YAML, but with support for loading in data and secrets from -various sources using for example `$env` and `$file` keys. +and `app-config.local.yaml` for local overrides. Additionally, it is possible +to define environment based configuration files with `BACKSTAGE_ENVIRONMENT` +environment variable, which will load `app-config..yaml`. + +Loading order of these files is as follows: + +1. `app-config.yaml` +2. `app-config..yaml` +3. `app-config.local.yaml` + +Other sets of files can by loaded by passing `--config ` flags. +Read more about the configuration loading order in the +[Configuration Files](./writing.md#configuration-files) section. + +The configuration files themselves contain plain YAML, but with support for +loading in data and secrets from various sources using for example +`$env` and `$file` keys. It is also possible to supply configuration through environment variables, for example `APP_CONFIG_app_baseUrl=https://staging.example.com`. However these diff --git a/packages/config-loader/src/sources/ConfigSources.test.ts b/packages/config-loader/src/sources/ConfigSources.test.ts index e19ac1d51f..9b2932e439 100644 --- a/packages/config-loader/src/sources/ConfigSources.test.ts +++ b/packages/config-loader/src/sources/ConfigSources.test.ts @@ -89,6 +89,18 @@ describe('ConfigSources', () => { { name: 'FileConfigSource', path: `${root}app-config.yaml` }, { name: 'FileConfigSource', path: `${root}app-config.local.yaml` }, ]); + + process.env = Object.assign(process.env, { BACKSTAGE_ENVIRONMENT: 'test' }); + expect( + mergeSources( + ConfigSources.defaultForTargets({ rootDir: '/', targets: [] }), + ), + ).toEqual([ + { name: 'FileConfigSource', path: `${root}app-config.yaml` }, + { name: 'FileConfigSource', path: `${root}app-config.test.yaml` }, + { name: 'FileConfigSource', path: `${root}app-config.local.yaml` }, + ]); + fsSpy.mockRestore(); expect( diff --git a/packages/config-loader/src/sources/ConfigSources.ts b/packages/config-loader/src/sources/ConfigSources.ts index 37c1c6b087..0d958bb81d 100644 --- a/packages/config-loader/src/sources/ConfigSources.ts +++ b/packages/config-loader/src/sources/ConfigSources.ts @@ -182,6 +182,10 @@ export class ConfigSources { if (argSources.length === 0) { const defaultPath = resolvePath(rootDir, 'app-config.yaml'); const localPath = resolvePath(rootDir, 'app-config.local.yaml'); + const envPath = resolvePath( + rootDir, + `app-config.${process.env.BACKSTAGE_ENVIRONMENT}.yaml`, + ); const alwaysIncludeDefaultConfigSource = !options.allowMissingDefaultConfig; @@ -195,6 +199,16 @@ export class ConfigSources { ); } + if (process.env.BACKSTAGE_ENVIRONMENT && fs.pathExistsSync(envPath)) { + argSources.push( + FileConfigSource.create({ + watch: options.watch, + path: envPath, + substitutionFunc: options.substitutionFunc, + }), + ); + } + if (fs.pathExistsSync(localPath)) { argSources.push( FileConfigSource.create({ From b321dd90406223c4b8a05f462bab9b02c560c3fd Mon Sep 17 00:00:00 2001 From: Hellgren Heikki Date: Mon, 11 Aug 2025 14:46:51 +0300 Subject: [PATCH 02/20] feat: add support for local env specific config files Signed-off-by: Hellgren Heikki --- docs/conf/index.md | 1 + .../src/sources/ConfigSources.test.ts | 1 + .../config-loader/src/sources/ConfigSources.ts | 17 +++++++++++++++++ 3 files changed, 19 insertions(+) diff --git a/docs/conf/index.md b/docs/conf/index.md index 7c5be174e5..0b2240e2de 100644 --- a/docs/conf/index.md +++ b/docs/conf/index.md @@ -25,6 +25,7 @@ Loading order of these files is as follows: 1. `app-config.yaml` 2. `app-config..yaml` 3. `app-config.local.yaml` +4. `app-config..local.yaml` Other sets of files can by loaded by passing `--config ` flags. Read more about the configuration loading order in the diff --git a/packages/config-loader/src/sources/ConfigSources.test.ts b/packages/config-loader/src/sources/ConfigSources.test.ts index 9b2932e439..da56f5ccf9 100644 --- a/packages/config-loader/src/sources/ConfigSources.test.ts +++ b/packages/config-loader/src/sources/ConfigSources.test.ts @@ -99,6 +99,7 @@ describe('ConfigSources', () => { { name: 'FileConfigSource', path: `${root}app-config.yaml` }, { name: 'FileConfigSource', path: `${root}app-config.test.yaml` }, { name: 'FileConfigSource', path: `${root}app-config.local.yaml` }, + { name: 'FileConfigSource', path: `${root}app-config.test.local.yaml` }, ]); fsSpy.mockRestore(); diff --git a/packages/config-loader/src/sources/ConfigSources.ts b/packages/config-loader/src/sources/ConfigSources.ts index 0d958bb81d..1dacdc4752 100644 --- a/packages/config-loader/src/sources/ConfigSources.ts +++ b/packages/config-loader/src/sources/ConfigSources.ts @@ -186,6 +186,10 @@ export class ConfigSources { rootDir, `app-config.${process.env.BACKSTAGE_ENVIRONMENT}.yaml`, ); + const envLocalPath = resolvePath( + rootDir, + `app-config.${process.env.BACKSTAGE_ENVIRONMENT}.local.yaml`, + ); const alwaysIncludeDefaultConfigSource = !options.allowMissingDefaultConfig; @@ -218,6 +222,19 @@ export class ConfigSources { }), ); } + + if ( + process.env.BACKSTAGE_ENVIRONMENT && + fs.pathExistsSync(envLocalPath) + ) { + argSources.push( + FileConfigSource.create({ + watch: options.watch, + path: envLocalPath, + substitutionFunc: options.substitutionFunc, + }), + ); + } } return this.merge(argSources); From 58fc10884612869139adaa87cb2f6ceb66c7eaac Mon Sep 17 00:00:00 2001 From: Stephen Glass Date: Wed, 20 Aug 2025 23:16:24 -0400 Subject: [PATCH 03/20] Set a minimum height for scaffolder task log stream Signed-off-by: Stephen Glass --- .changeset/tangy-squids-film.md | 5 +++++ .../src/next/components/TaskLogStream/TaskLogStream.tsx | 1 + 2 files changed, 6 insertions(+) create mode 100644 .changeset/tangy-squids-film.md diff --git a/.changeset/tangy-squids-film.md b/.changeset/tangy-squids-film.md new file mode 100644 index 0000000000..acb5546031 --- /dev/null +++ b/.changeset/tangy-squids-film.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-react': patch +--- + +Fix scaffolder task log stream not having a minimum height diff --git a/plugins/scaffolder-react/src/next/components/TaskLogStream/TaskLogStream.tsx b/plugins/scaffolder-react/src/next/components/TaskLogStream/TaskLogStream.tsx index a0c4ce94cd..b88d96510e 100644 --- a/plugins/scaffolder-react/src/next/components/TaskLogStream/TaskLogStream.tsx +++ b/plugins/scaffolder-react/src/next/components/TaskLogStream/TaskLogStream.tsx @@ -21,6 +21,7 @@ const useStyles = makeStyles({ width: '100%', height: '100%', position: 'relative', + minHeight: 240, }, }); From 85c5e045c9778623644e86716dd0aa76cf670ee6 Mon Sep 17 00:00:00 2001 From: MT Lewis Date: Thu, 21 Aug 2025 16:22:44 +0100 Subject: [PATCH 04/20] catalog: fix incorrect defaultTarget in createComponentRouteRef The previous value doesn't exist in the scaffolder. I believe the logical place for this to point to is the root of the scaffolder plugin, which renders the list of templates. Signed-off-by: MT Lewis --- .changeset/warm-emus-itch.md | 5 +++++ packages/app-next/app-config.yaml | 1 - plugins/catalog/src/routes.ts | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 .changeset/warm-emus-itch.md diff --git a/.changeset/warm-emus-itch.md b/.changeset/warm-emus-itch.md new file mode 100644 index 0000000000..3275098425 --- /dev/null +++ b/.changeset/warm-emus-itch.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog': patch +--- + +Fix incorrect defaultTarget on `createComponentRouteRef`. diff --git a/packages/app-next/app-config.yaml b/packages/app-next/app-config.yaml index f83e4d4ba0..35380c961f 100644 --- a/packages/app-next/app-config.yaml +++ b/packages/app-next/app-config.yaml @@ -4,7 +4,6 @@ app: routes: bindings: catalog.viewTechDoc: techdocs.docRoot - catalog.createComponent: catalog-import.importPage org.catalogIndex: catalog.catalogIndex pluginOverrides: diff --git a/plugins/catalog/src/routes.ts b/plugins/catalog/src/routes.ts index 4f7f0427b1..6043ecd1a1 100644 --- a/plugins/catalog/src/routes.ts +++ b/plugins/catalog/src/routes.ts @@ -22,7 +22,7 @@ import { export const createComponentRouteRef = createExternalRouteRef({ id: 'create-component', optional: true, - defaultTarget: 'scaffolder.createComponent', + defaultTarget: 'scaffolder.root', }); export const viewTechDocRouteRef = createExternalRouteRef({ From 0d415ae01409b022b4d2fe8457cc8d301d2fadcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Fankh=C3=A4nel?= Date: Tue, 26 Aug 2025 15:08:55 +0200 Subject: [PATCH 05/20] fix(scaffolder): render TechDocs link on Template List page for TechDocs annotations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Show “View TechDocs” link when template has backstage.io/techdocs-ref or backstage.io/techdocs-entity. Append backstage.io/techdocs-entity-path when set. Use buildTechDocsURL from @backstage/plugin-techdocs-react. Add tests covering both annotations and path handling. Update dependencies to include @backstage/plugin-techdocs-common and @backstage/plugin-techdocs-react. Add sample TechDocs scaffolding to notifications-demo template. Closes #29076. Signed-off-by: David Fankhänel --- .changeset/itchy-moons-start.md | 5 + plugins/scaffolder/package.json | 3 + .../TemplateListPage.test.tsx | 129 +++++++++++++++++- .../TemplateListPage/TemplateListPage.tsx | 17 ++- yarn.lock | 3 + 5 files changed, 150 insertions(+), 7 deletions(-) create mode 100644 .changeset/itchy-moons-start.md diff --git a/.changeset/itchy-moons-start.md b/.changeset/itchy-moons-start.md new file mode 100644 index 0000000000..e5768b58b2 --- /dev/null +++ b/.changeset/itchy-moons-start.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder': patch +--- + +Render a TechDocs link on the Scaffolder Template List page when templates include either `backstage.io/techdocs-ref` or `backstage.io/techdocs-entity` annotations, using the shared `buildTechDocsURL` helper. Also adds tests to verify both annotations and optional `backstage.io/techdocs-entity-path` are respected. diff --git a/plugins/scaffolder/package.json b/plugins/scaffolder/package.json index ff56a19175..df75a65656 100644 --- a/plugins/scaffolder/package.json +++ b/plugins/scaffolder/package.json @@ -72,6 +72,8 @@ "@backstage/plugin-permission-react": "workspace:^", "@backstage/plugin-scaffolder-common": "workspace:^", "@backstage/plugin-scaffolder-react": "workspace:^", + "@backstage/plugin-techdocs-common": "workspace:^", + "@backstage/plugin-techdocs-react": "workspace:^", "@backstage/types": "workspace:^", "@codemirror/language": "^6.0.0", "@codemirror/legacy-modes": "^6.1.0", @@ -109,6 +111,7 @@ "@backstage/dev-utils": "workspace:^", "@backstage/plugin-catalog": "workspace:^", "@backstage/plugin-permission-common": "workspace:^", + "@backstage/plugin-techdocs": "workspace:^", "@backstage/test-utils": "workspace:^", "@testing-library/dom": "^10.0.0", "@testing-library/jest-dom": "^6.0.0", diff --git a/plugins/scaffolder/src/alpha/components/TemplateListPage/TemplateListPage.test.tsx b/plugins/scaffolder/src/alpha/components/TemplateListPage/TemplateListPage.test.tsx index a16474a4cf..da544cb2c0 100644 --- a/plugins/scaffolder/src/alpha/components/TemplateListPage/TemplateListPage.test.tsx +++ b/plugins/scaffolder/src/alpha/components/TemplateListPage/TemplateListPage.test.tsx @@ -27,13 +27,19 @@ import { TestApiProvider, mockApis, } from '@backstage/test-utils'; -import { rootRouteRef } from '../../../routes'; +import { rootRouteRef, viewTechDocRouteRef } from '../../../routes'; import { TemplateListPage } from './TemplateListPage'; +import { + TECHDOCS_ANNOTATION, + TECHDOCS_EXTERNAL_ANNOTATION, + TECHDOCS_EXTERNAL_PATH_ANNOTATION, +} from '@backstage/plugin-techdocs-common'; const mountedRoutes = { mountedRoutes: { '/': rootRouteRef, '/catalog/:namespace/:kind/:name': entityRouteRef, + '/docs/:namespace/:kind/:name': viewTechDocRouteRef, }, }; @@ -51,6 +57,127 @@ describe('TemplateListPage', () => { ], }); + describe('TechDocs link rendering', () => { + it('shows TechDocs link when template has backstage.io/techdocs-ref', async () => { + const mockCatalogApiWithDocs = catalogApiMock({ + entities: [ + { + apiVersion: 'scaffolder.backstage.io/v1beta3', + kind: 'Template', + metadata: { + name: 'tmpl-a', + annotations: { [TECHDOCS_ANNOTATION]: 'dir:.' }, + }, + spec: { type: 'service' }, + }, + ], + }); + + const { findByText } = await renderInTestApp( + + + , + mountedRoutes, + ); + + expect(await findByText('View TechDocs')).toBeInTheDocument(); + }); + + it('shows TechDocs link when template has backstage.io/techdocs-entity', async () => { + const mockCatalogApiWithExternal = catalogApiMock({ + entities: [ + { + apiVersion: 'scaffolder.backstage.io/v1beta3', + kind: 'Template', + metadata: { + name: 'tmpl-b', + annotations: { + [TECHDOCS_EXTERNAL_ANNOTATION]: 'component:default/other', + }, + }, + spec: { type: 'service' }, + }, + ], + }); + + const { findByText } = await renderInTestApp( + + + , + mountedRoutes, + ); + + expect(await findByText('View TechDocs')).toBeInTheDocument(); + }); + + it('appends path when backstage.io/techdocs-entity-path is set', async () => { + const mockCatalogApiWithPath = catalogApiMock({ + entities: [ + { + apiVersion: 'scaffolder.backstage.io/v1beta3', + kind: 'Template', + metadata: { + name: 'tmpl-c', + annotations: { + [TECHDOCS_EXTERNAL_ANNOTATION]: 'component:default/other', + [TECHDOCS_EXTERNAL_PATH_ANNOTATION]: '/guides/start', + }, + }, + spec: { type: 'service' }, + }, + ], + }); + + const { findByText } = await renderInTestApp( + + + , + mountedRoutes, + ); + + const link = (await findByText('View TechDocs')).closest('a')!; + expect(link).toHaveAttribute( + 'href', + expect.stringMatching( + /\/docs\/default\/component\/other\/?(index\.html)?#?\/guides\/start|\/docs\/default\/component\/other\/guides\/start/, + ), + ); + }); + }); + it('should render the search bar for templates', async () => { const { getByPlaceholderText } = await renderInTestApp( { const additionalLinksForEntity = useCallback( (template: TemplateEntityV1beta3) => { - const { kind, namespace, name } = parseEntityRef( - stringifyEntityRef(template), - ); - return template.metadata.annotations?.['backstage.io/techdocs-ref'] && - viewTechDocsLink + const hasTechDocs = + !!template.metadata.annotations?.[TECHDOCS_ANNOTATION] || + !!template.metadata.annotations?.[TECHDOCS_EXTERNAL_ANNOTATION]; + + return hasTechDocs && viewTechDocsLink ? [ { icon: app.getSystemIcon('docs') ?? DocsIcon, text: t( 'templateListPage.additionalLinksForEntity.viewTechDocsTitle', ), - url: viewTechDocsLink({ kind, namespace, name }), + url: buildTechDocsURL(template, viewTechDocsLink), }, ] : []; diff --git a/yarn.lock b/yarn.lock index 5487f23662..aae8a0ef0f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6573,6 +6573,9 @@ __metadata: "@backstage/plugin-permission-react": "workspace:^" "@backstage/plugin-scaffolder-common": "workspace:^" "@backstage/plugin-scaffolder-react": "workspace:^" + "@backstage/plugin-techdocs": "workspace:^" + "@backstage/plugin-techdocs-common": "workspace:^" + "@backstage/plugin-techdocs-react": "workspace:^" "@backstage/test-utils": "workspace:^" "@backstage/types": "workspace:^" "@codemirror/language": "npm:^6.0.0" From 5b6416938412da34b1459a484d9d6827c92c8800 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Fankh=C3=A4nel?= Date: Wed, 27 Aug 2025 10:28:28 +0200 Subject: [PATCH 06/20] fix(TemplateListPage): fix tsc error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Fankhänel --- .../TemplateListPage/TemplateListPage.tsx | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/plugins/scaffolder/src/alpha/components/TemplateListPage/TemplateListPage.tsx b/plugins/scaffolder/src/alpha/components/TemplateListPage/TemplateListPage.tsx index 0a7a3df888..37a50ed904 100644 --- a/plugins/scaffolder/src/alpha/components/TemplateListPage/TemplateListPage.tsx +++ b/plugins/scaffolder/src/alpha/components/TemplateListPage/TemplateListPage.tsx @@ -149,18 +149,25 @@ export const TemplateListPage = (props: TemplateListPageProps) => { const additionalLinksForEntity = useCallback( (template: TemplateEntityV1beta3) => { - const hasTechDocs = - !!template.metadata.annotations?.[TECHDOCS_ANNOTATION] || - !!template.metadata.annotations?.[TECHDOCS_EXTERNAL_ANNOTATION]; + if ( + !( + template.metadata.annotations?.[TECHDOCS_ANNOTATION] || + template.metadata.annotations?.[TECHDOCS_EXTERNAL_ANNOTATION] + ) || + !viewTechDocsLink + ) { + return []; + } - return hasTechDocs && viewTechDocsLink + const url = buildTechDocsURL(template, viewTechDocsLink); + return url ? [ { icon: app.getSystemIcon('docs') ?? DocsIcon, text: t( 'templateListPage.additionalLinksForEntity.viewTechDocsTitle', ), - url: buildTechDocsURL(template, viewTechDocsLink), + url, }, ] : []; From 2204f5b77edff470b212b6c4ff2a2e58c41e0e44 Mon Sep 17 00:00:00 2001 From: Andreas Berger Date: Wed, 3 Sep 2025 14:53:40 +0200 Subject: [PATCH 07/20] Prevent deadlock in catalog deferred stitching resolves #30843 Signed-off-by: Andreas Berger --- .changeset/late-swans-press.md | 5 + .../stitcher/markForStitching.test.ts | 139 ++++++++++++++++++ .../operations/stitcher/markForStitching.ts | 106 +++++++++---- 3 files changed, 219 insertions(+), 31 deletions(-) create mode 100644 .changeset/late-swans-press.md diff --git a/.changeset/late-swans-press.md b/.changeset/late-swans-press.md new file mode 100644 index 0000000000..987672bd77 --- /dev/null +++ b/.changeset/late-swans-press.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Prevent deadlock in catalog deferred stitching diff --git a/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.test.ts b/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.test.ts index 1afa0b9c30..33712b99f4 100644 --- a/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.test.ts +++ b/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.test.ts @@ -435,4 +435,143 @@ describe('markForStitching', () => { } }, ); + + const deadlockTestDatabases = TestDatabases.create({ + ids: ['POSTGRES_17', 'POSTGRES_16', 'SQLITE_3'], + disableDocker: false, + }); + it.each(deadlockTestDatabases.eachSupportedId())( + 'reproduces deadlock scenario when concurrent transactions update overlapping entity sets %p', + async databaseId => { + const knex = await deadlockTestDatabases.init(databaseId); + await applyDatabaseMigrations(knex); + + // Setup test data with multiple entities + const entityRefs = [ + 'k:ns/entity-a', + 'k:ns/entity-b', + 'k:ns/entity-c', + 'k:ns/entity-d', + 'k:ns/entity-e', + 'k:ns/entity-f', + ]; + + await knex('refresh_state').insert( + entityRefs.map((ref, i) => ({ + entity_id: `${i + 1}`, + entity_ref: ref, + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '[]', + next_update_at: knex.fn.now(), + last_discovery_at: knex.fn.now(), + next_stitch_at: null, + next_stitch_ticket: null, + })), + ); + + // This test attempts to reproduce the deadlock by running concurrent transactions + // that update overlapping sets of entities in different orders + const errorResults = []; + + for (let attempt = 0; attempt < 10; attempt++) { + // Transaction 1: Update entities A, B, C, D, E + const transaction1 = knex.transaction(async trx => { + await markForStitching({ + knex: trx, + strategy: { + mode: 'deferred', + pollingInterval: { seconds: 1 }, + stitchTimeout: { seconds: 1 }, + }, + entityRefs: [ + 'k:ns/entity-a', + 'k:ns/entity-b', + 'k:ns/entity-c', + 'k:ns/entity-d', + 'k:ns/entity-e', + ], + }); + + // Add a small delay to increase chance of collision + await new Promise(resolve => setTimeout(resolve, 10)); + + await markForStitching({ + knex: trx, + strategy: { + mode: 'deferred', + pollingInterval: { seconds: 1 }, + stitchTimeout: { seconds: 1 }, + }, + entityRefs: ['k:ns/entity-f'], + }); + }); + + // Transaction 2: Update entities F, E, D, C, B (reverse order) + const transaction2 = knex.transaction(async trx => { + await markForStitching({ + knex: trx, + strategy: { + mode: 'deferred', + pollingInterval: { seconds: 1 }, + stitchTimeout: { seconds: 1 }, + }, + entityRefs: [ + 'k:ns/entity-f', + 'k:ns/entity-e', + 'k:ns/entity-d', + 'k:ns/entity-c', + 'k:ns/entity-b', + ], + }); + + // Add a small delay to increase chance of collision + await new Promise(resolve => setTimeout(resolve, 10)); + + await markForStitching({ + knex: trx, + strategy: { + mode: 'deferred', + pollingInterval: { seconds: 1 }, + stitchTimeout: { seconds: 1 }, + }, + entityRefs: ['k:ns/entity-a'], + }); + }); + + // Run both transactions concurrently to create potential deadlock + errorResults.push( + Promise.allSettled([transaction1, transaction2]).then(results => + results + .filter(r => r.status === 'rejected') + .map(r => (r as PromiseRejectedResult).reason), + ), + ); + } + + const allResults = await Promise.all(errorResults); + + const deadlockErrors = allResults + .flat() + .filter( + error => + error?.code === '40P01' || + error?.message?.includes('deadlock detected') || + error?.message?.includes('deadlock'), + ); + expect(deadlockErrors.length).toEqual(0); + + // Verify final state - all entities should have been marked for stitching + const finalState = await knex('refresh_state') + .select('entity_ref', 'next_stitch_at', 'next_stitch_ticket') + .whereNotNull('next_stitch_at') + .orderBy('entity_ref'); + + expect(finalState.length).toBeGreaterThan(0); + finalState.forEach(row => { + expect(row.next_stitch_at).not.toBeNull(); + expect(row.next_stitch_ticket).not.toBeNull(); + }); + }, + ); }); diff --git a/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.ts b/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.ts index ecc364a9cc..a3d63778c3 100644 --- a/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.ts +++ b/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.ts @@ -20,6 +20,11 @@ import { v4 as uuid } from 'uuid'; import { StitchingStrategy } from '../../../stitching/types'; import { DbFinalEntitiesRow, DbRefreshStateRow } from '../../tables'; +const UPDATE_CHUNK_SIZE = 100; // Smaller chunks reduce contention +const DEADLOCK_SQLSTATE = '40P01'; +const DEADLOCK_RETRY_ATTEMPTS = 3; +const DEADLOCK_BASE_DELAY_MS = 25; + /** * Marks a number of entities for stitching some time in the near * future. @@ -32,9 +37,9 @@ export async function markForStitching(options: { entityRefs?: Iterable; entityIds?: Iterable; }): Promise { - // Splitting to chunks just to cover pathological cases that upset the db - const entityRefs = split(options.entityRefs); - const entityIds = split(options.entityIds); + // Sort inputs to ensure consistent lock order across concurrent writers + const entityRefs = split(options.entityRefs, true); + const entityIds = split(options.entityIds, true); const knex = options.knex; const mode = options.strategy.mode; @@ -51,13 +56,15 @@ export async function markForStitching(options: { .select('entity_id') .whereIn('entity_ref', chunk), ); - await knex - .table('refresh_state') - .update({ - result_hash: 'force-stitching', - next_update_at: knex.fn.now(), - }) - .whereIn('entity_ref', chunk); + await retryOnDeadlock(async () => { + await knex + .table('refresh_state') + .update({ + result_hash: 'force-stitching', + next_update_at: knex.fn.now(), + }) + .whereIn('entity_ref', chunk); + }); } for (const chunk of entityIds) { @@ -67,44 +74,81 @@ export async function markForStitching(options: { hash: 'force-stitching', }) .whereIn('entity_id', chunk); - await knex - .table('refresh_state') - .update({ - result_hash: 'force-stitching', - next_update_at: knex.fn.now(), - }) - .whereIn('entity_id', chunk); + await retryOnDeadlock(async () => { + await knex + .table('refresh_state') + .update({ + result_hash: 'force-stitching', + next_update_at: knex.fn.now(), + }) + .whereIn('entity_id', chunk); + }); } } else if (mode === 'deferred') { // It's OK that this is shared across refresh state rows; it just needs to // be uniquely generated for every new stitch request. const ticket = uuid(); + // Update by primary key in deterministic order to avoid deadlocks for (const chunk of entityRefs) { - await knex('refresh_state') - .update({ - next_stitch_at: knex.fn.now(), - next_stitch_ticket: ticket, - }) - .whereIn('entity_ref', chunk); + await retryOnDeadlock(async () => { + await knex('refresh_state') + .update({ + next_stitch_at: knex.fn.now(), + next_stitch_ticket: ticket, + }) + .whereIn('entity_ref', chunk); + }); } for (const chunk of entityIds) { - await knex('refresh_state') - .update({ - next_stitch_at: knex.fn.now(), - next_stitch_ticket: ticket, - }) - .whereIn('entity_id', chunk); + // Ensure ids are sorted for deterministic lock order + + await retryOnDeadlock(async () => { + await knex('refresh_state') + .update({ + next_stitch_at: knex.fn.now(), + next_stitch_ticket: ticket, + }) + .whereIn('entity_id', chunk); + }); } } else { throw new Error(`Unknown stitching strategy mode ${mode}`); } } -function split(input: Iterable | undefined): string[][] { +function split(input: Iterable | undefined, sort = false): string[][] { if (!input) { return []; } - return splitToChunks(Array.isArray(input) ? input : [...input], 200); + const array = Array.isArray(input) ? input.slice() : [...input]; + if (sort) { + array.sort(); + } + return splitToChunks(array, UPDATE_CHUNK_SIZE); +} + +async function retryOnDeadlock( + fn: () => Promise, + retries = DEADLOCK_RETRY_ATTEMPTS, + baseMs = DEADLOCK_BASE_DELAY_MS, +): Promise { + let attempt = 0; + for (;;) { + try { + return await fn(); + } catch (e: any) { + if (e?.code === DEADLOCK_SQLSTATE && attempt < retries) { + await sleep(baseMs * Math.pow(2, attempt)); + attempt++; + continue; + } + throw e; + } + } +} + +function sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); } From 681c726ecc08e443158e0cbbd7786ca1ce7e4c59 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Sep 2025 05:10:36 +0000 Subject: [PATCH 08/20] chore(deps): bump form-data from 2.5.1 to 2.5.5 Bumps [form-data](https://github.com/form-data/form-data) from 2.5.1 to 2.5.5. - [Release notes](https://github.com/form-data/form-data/releases) - [Changelog](https://github.com/form-data/form-data/blob/v2.5.5/CHANGELOG.md) - [Commits](https://github.com/form-data/form-data/compare/v2.5.1...v2.5.5) --- updated-dependencies: - dependency-name: form-data dependency-version: 2.5.5 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- yarn.lock | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/yarn.lock b/yarn.lock index 853749b99e..cf287c28d8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -26265,7 +26265,7 @@ __metadata: languageName: node linkType: hard -"combined-stream@npm:^1.0.6, combined-stream@npm:^1.0.8": +"combined-stream@npm:^1.0.8": version: 1.0.8 resolution: "combined-stream@npm:1.0.8" dependencies: @@ -31054,13 +31054,16 @@ __metadata: linkType: hard "form-data@npm:^2.3.2, form-data@npm:^2.5.0": - version: 2.5.1 - resolution: "form-data@npm:2.5.1" + version: 2.5.5 + resolution: "form-data@npm:2.5.5" dependencies: asynckit: "npm:^0.4.0" - combined-stream: "npm:^1.0.6" - mime-types: "npm:^2.1.12" - checksum: 10/2e2e5e927979ba3623f9b4c4bcc939275fae3f2dea9dafc8db3ca656a3d75476605de2c80f0e6f1487987398e056f0b4c738972d6e1edd83392d5686d0952eed + combined-stream: "npm:^1.0.8" + es-set-tostringtag: "npm:^2.1.0" + hasown: "npm:^2.0.2" + mime-types: "npm:^2.1.35" + safe-buffer: "npm:^5.2.1" + checksum: 10/4b6a8d07bb67089da41048e734215f68317a8e29dd5385a972bf5c458a023313c69d3b5d6b8baafbb7f808fa9881e0e2e030ffe61e096b3ddc894c516401271d languageName: node linkType: hard From 2b208f1963ef4e135233a2b2f0d5a6dfd096b6a0 Mon Sep 17 00:00:00 2001 From: Andreas Berger Date: Thu, 4 Sep 2025 10:40:22 +0200 Subject: [PATCH 09/20] Adjustments after review Signed-off-by: Andreas Berger --- .../stitcher/markForStitching.test.ts | 10 ++--- .../operations/stitcher/markForStitching.ts | 41 ++++++++++++------- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.test.ts b/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.test.ts index 33712b99f4..ff450e9ccf 100644 --- a/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.test.ts +++ b/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.test.ts @@ -436,14 +436,10 @@ describe('markForStitching', () => { }, ); - const deadlockTestDatabases = TestDatabases.create({ - ids: ['POSTGRES_17', 'POSTGRES_16', 'SQLITE_3'], - disableDocker: false, - }); - it.each(deadlockTestDatabases.eachSupportedId())( + it.each(databases.eachSupportedId())( 'reproduces deadlock scenario when concurrent transactions update overlapping entity sets %p', async databaseId => { - const knex = await deadlockTestDatabases.init(databaseId); + const knex = await databases.init(databaseId); await applyDatabaseMigrations(knex); // Setup test data with multiple entities @@ -559,7 +555,7 @@ describe('markForStitching', () => { error?.message?.includes('deadlock detected') || error?.message?.includes('deadlock'), ); - expect(deadlockErrors.length).toEqual(0); + expect(deadlockErrors).toEqual([]); // Verify final state - all entities should have been marked for stitching const finalState = await knex('refresh_state') diff --git a/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.ts b/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.ts index a3d63778c3..913936202f 100644 --- a/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.ts +++ b/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.ts @@ -21,10 +21,25 @@ import { StitchingStrategy } from '../../../stitching/types'; import { DbFinalEntitiesRow, DbRefreshStateRow } from '../../tables'; const UPDATE_CHUNK_SIZE = 100; // Smaller chunks reduce contention -const DEADLOCK_SQLSTATE = '40P01'; const DEADLOCK_RETRY_ATTEMPTS = 3; const DEADLOCK_BASE_DELAY_MS = 25; +// PostgreSQL deadlock error code +const POSTGRES_DEADLOCK_SQLSTATE = '40P01'; + +/** + * Checks if the given error is a deadlock error for the database engine in use. + */ +function isDeadlockError(knex: Knex | Knex.Transaction, e: unknown): boolean { + if (knex.client.config.client.includes('pg')) { + // PostgreSQL deadlock detection + return (e as any)?.code === POSTGRES_DEADLOCK_SQLSTATE; + } + + // Add more database engine checks here as needed + return false; +} + /** * Marks a number of entities for stitching some time in the near * future. @@ -37,9 +52,8 @@ export async function markForStitching(options: { entityRefs?: Iterable; entityIds?: Iterable; }): Promise { - // Sort inputs to ensure consistent lock order across concurrent writers - const entityRefs = split(options.entityRefs, true); - const entityIds = split(options.entityIds, true); + const entityRefs = sortSplit(options.entityRefs); + const entityIds = sortSplit(options.entityIds); const knex = options.knex; const mode = options.strategy.mode; @@ -64,7 +78,7 @@ export async function markForStitching(options: { next_update_at: knex.fn.now(), }) .whereIn('entity_ref', chunk); - }); + }, knex); } for (const chunk of entityIds) { @@ -82,7 +96,7 @@ export async function markForStitching(options: { next_update_at: knex.fn.now(), }) .whereIn('entity_id', chunk); - }); + }, knex); } } else if (mode === 'deferred') { // It's OK that this is shared across refresh state rows; it just needs to @@ -98,12 +112,10 @@ export async function markForStitching(options: { next_stitch_ticket: ticket, }) .whereIn('entity_ref', chunk); - }); + }, knex); } for (const chunk of entityIds) { - // Ensure ids are sorted for deterministic lock order - await retryOnDeadlock(async () => { await knex('refresh_state') .update({ @@ -111,26 +123,25 @@ export async function markForStitching(options: { next_stitch_ticket: ticket, }) .whereIn('entity_id', chunk); - }); + }, knex); } } else { throw new Error(`Unknown stitching strategy mode ${mode}`); } } -function split(input: Iterable | undefined, sort = false): string[][] { +function sortSplit(input: Iterable | undefined): string[][] { if (!input) { return []; } const array = Array.isArray(input) ? input.slice() : [...input]; - if (sort) { - array.sort(); - } + array.sort(); return splitToChunks(array, UPDATE_CHUNK_SIZE); } async function retryOnDeadlock( fn: () => Promise, + knex: Knex | Knex.Transaction, retries = DEADLOCK_RETRY_ATTEMPTS, baseMs = DEADLOCK_BASE_DELAY_MS, ): Promise { @@ -139,7 +150,7 @@ async function retryOnDeadlock( try { return await fn(); } catch (e: any) { - if (e?.code === DEADLOCK_SQLSTATE && attempt < retries) { + if (isDeadlockError(knex, e) && attempt < retries) { await sleep(baseMs * Math.pow(2, attempt)); attempt++; continue; From 5798d2566f2d6898863568793de556cbc58c3e2d Mon Sep 17 00:00:00 2001 From: Hellgren Heikki Date: Mon, 8 Sep 2025 08:47:51 +0300 Subject: [PATCH 10/20] chore: change to BACKSTAGE_ENV as per review Signed-off-by: Hellgren Heikki --- .changeset/better-eagles-tickle.md | 2 +- docs/conf/index.md | 8 ++++---- .../config-loader/src/sources/ConfigSources.test.ts | 2 +- packages/config-loader/src/sources/ConfigSources.ts | 11 ++++------- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/.changeset/better-eagles-tickle.md b/.changeset/better-eagles-tickle.md index b199ae21d7..418f0bcae0 100644 --- a/.changeset/better-eagles-tickle.md +++ b/.changeset/better-eagles-tickle.md @@ -2,4 +2,4 @@ '@backstage/config-loader': patch --- -Allow using `BACKSTAGE_ENVIRONMENT` for loading environment specific config files +Allow using `BACKSTAGE_ENV` for loading environment specific config files diff --git a/docs/conf/index.md b/docs/conf/index.md index 0b2240e2de..029c5b88fd 100644 --- a/docs/conf/index.md +++ b/docs/conf/index.md @@ -17,15 +17,15 @@ allowing for customization. Configuration is stored in YAML files where the defaults are `app-config.yaml` and `app-config.local.yaml` for local overrides. Additionally, it is possible -to define environment based configuration files with `BACKSTAGE_ENVIRONMENT` -environment variable, which will load `app-config..yaml`. +to define environment based configuration files with `BACKSTAGE_ENV` +environment variable, which will load `app-config..yaml`. Loading order of these files is as follows: 1. `app-config.yaml` -2. `app-config..yaml` +2. `app-config..yaml` 3. `app-config.local.yaml` -4. `app-config..local.yaml` +4. `app-config..local.yaml` Other sets of files can by loaded by passing `--config ` flags. Read more about the configuration loading order in the diff --git a/packages/config-loader/src/sources/ConfigSources.test.ts b/packages/config-loader/src/sources/ConfigSources.test.ts index da56f5ccf9..a6b0648889 100644 --- a/packages/config-loader/src/sources/ConfigSources.test.ts +++ b/packages/config-loader/src/sources/ConfigSources.test.ts @@ -90,7 +90,7 @@ describe('ConfigSources', () => { { name: 'FileConfigSource', path: `${root}app-config.local.yaml` }, ]); - process.env = Object.assign(process.env, { BACKSTAGE_ENVIRONMENT: 'test' }); + process.env = Object.assign(process.env, { BACKSTAGE_ENV: 'test' }); expect( mergeSources( ConfigSources.defaultForTargets({ rootDir: '/', targets: [] }), diff --git a/packages/config-loader/src/sources/ConfigSources.ts b/packages/config-loader/src/sources/ConfigSources.ts index 1dacdc4752..18d0443986 100644 --- a/packages/config-loader/src/sources/ConfigSources.ts +++ b/packages/config-loader/src/sources/ConfigSources.ts @@ -184,11 +184,11 @@ export class ConfigSources { const localPath = resolvePath(rootDir, 'app-config.local.yaml'); const envPath = resolvePath( rootDir, - `app-config.${process.env.BACKSTAGE_ENVIRONMENT}.yaml`, + `app-config.${process.env.BACKSTAGE_ENV}.yaml`, ); const envLocalPath = resolvePath( rootDir, - `app-config.${process.env.BACKSTAGE_ENVIRONMENT}.local.yaml`, + `app-config.${process.env.BACKSTAGE_ENV}.local.yaml`, ); const alwaysIncludeDefaultConfigSource = !options.allowMissingDefaultConfig; @@ -203,7 +203,7 @@ export class ConfigSources { ); } - if (process.env.BACKSTAGE_ENVIRONMENT && fs.pathExistsSync(envPath)) { + if (process.env.BACKSTAGE_ENV && fs.pathExistsSync(envPath)) { argSources.push( FileConfigSource.create({ watch: options.watch, @@ -223,10 +223,7 @@ export class ConfigSources { ); } - if ( - process.env.BACKSTAGE_ENVIRONMENT && - fs.pathExistsSync(envLocalPath) - ) { + if (process.env.BACKSTAGE_ENV && fs.pathExistsSync(envLocalPath)) { argSources.push( FileConfigSource.create({ watch: options.watch, From d23bab525d9380ca43b2e322d546bec5299ed054 Mon Sep 17 00:00:00 2001 From: gyan Date: Mon, 8 Sep 2025 15:29:54 +0530 Subject: [PATCH 11/20] remove extra theme string from the tool-tip Signed-off-by: gyan --- plugins/user-settings/src/translation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/user-settings/src/translation.ts b/plugins/user-settings/src/translation.ts index 0161508d14..7445a415f2 100644 --- a/plugins/user-settings/src/translation.ts +++ b/plugins/user-settings/src/translation.ts @@ -28,7 +28,7 @@ export const userSettingsTranslationRef = createTranslationRef({ themeToggle: { title: 'Theme', description: 'Change the theme mode', - select: 'Select theme {{theme}}', + select: 'Select {{theme}}', selectAuto: 'Select Auto Theme', names: { light: 'Light', From b713b543564445dd18908b5ffa791da55d2a5686 Mon Sep 17 00:00:00 2001 From: gyan Date: Mon, 8 Sep 2025 15:45:56 +0530 Subject: [PATCH 12/20] add changeset configuration Signed-off-by: gyan --- .changeset/red-shrimps-fall.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/red-shrimps-fall.md diff --git a/.changeset/red-shrimps-fall.md b/.changeset/red-shrimps-fall.md new file mode 100644 index 0000000000..1ee3aca7ed --- /dev/null +++ b/.changeset/red-shrimps-fall.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-user-settings': patch +--- + +Tool-tip text correction for the Theme selection in settings page From 4eda590b6a5e0309fcc23a279fc2931471f5b5c0 Mon Sep 17 00:00:00 2001 From: Shijun Wang Date: Tue, 29 Jul 2025 15:53:19 +0300 Subject: [PATCH 13/20] add logic to construct namespace from provided namespace and plugin id Signed-off-by: Shijun Wang --- .changeset/big-cameras-turn.md | 5 + docs/backend-system/core-services/cache.md | 33 ++++ .../entrypoints/cache/CacheManager.test.ts | 143 ++++++++++++++++++ .../src/entrypoints/cache/CacheManager.ts | 23 ++- 4 files changed, 202 insertions(+), 2 deletions(-) create mode 100644 .changeset/big-cameras-turn.md diff --git a/.changeset/big-cameras-turn.md b/.changeset/big-cameras-turn.md new file mode 100644 index 0000000000..6226b17806 --- /dev/null +++ b/.changeset/big-cameras-turn.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-defaults': patch +--- + +Fixed cache namespace and key prefix separator configuration to properly use configured values instead of hardcoded plugin ID. The cache manager now correctly combines the configured namespace with plugin IDs using the configured separator for Redis and Valkey. Memcache and memory store continue to use plugin ID as namespace. diff --git a/docs/backend-system/core-services/cache.md b/docs/backend-system/core-services/cache.md index 0e26e8e1f5..ad4d04b50a 100644 --- a/docs/backend-system/core-services/cache.md +++ b/docs/backend-system/core-services/cache.md @@ -7,6 +7,39 @@ description: Documentation for the Cache service This service lets your plugin interact with a cache. It is bound to your plugin too, so that you will only set and get values in your plugin's private namespace. +## Configuration + +The cache service can be configured using the `backend.cache` section in your `app-config.yaml`: + +```yaml +backend: + cache: + store: redis # or 'valkey', 'memcache', 'memory' + connection: redis://localhost:6379 + + # Store-specific configuration (Redis/Valkey only) + redis: + client: + # Optional: Global namespace prefix for all cache keys + namespace: 'my-app' + # Optional: Separator used between namespace and plugin ID (default: ':') + keyPrefixSeparator: ':' + # Other Redis-specific options... + clearBatchSize: 1000 + useUnlink: false +``` + +### Namespace Configuration + +For Redis and Valkey stores, you can configure a global namespace that will be prefixed to all cache keys: + +- **Without namespace**: Cache keys use only the plugin ID (e.g., `catalog:some-key`) +- **With namespace**: Cache keys use the format `namespace:pluginId:key` (e.g., `my-app:catalog:some-key`) + +The `keyPrefixSeparator` controls what character is used between the namespace and plugin ID (defaults to `:`). + +**Note**: Memory and Memcache stores do not support namespace configuration and will always use the plugin ID directly. + ## Using the service The following example shows how to get a cache client in your `example` backend plugin and setting and getting values from the cache. diff --git a/packages/backend-defaults/src/entrypoints/cache/CacheManager.test.ts b/packages/backend-defaults/src/entrypoints/cache/CacheManager.test.ts index 4e9475b66e..a308caa222 100644 --- a/packages/backend-defaults/src/entrypoints/cache/CacheManager.test.ts +++ b/packages/backend-defaults/src/entrypoints/cache/CacheManager.test.ts @@ -333,4 +333,147 @@ describe('CacheManager store options', () => { keyPrefixSeparator: '!', }); }); + + it('correctly applies namespace configuration to redis and valkey stores', () => { + const testCases = [ + { store: 'redis', namespace: 'test1', separator: ':' }, + { store: 'valkey', namespace: 'test2', separator: '!' }, + ]; + + testCases.forEach(({ store, namespace, separator }) => { + const manager = CacheManager.fromConfig( + mockServices.rootConfig({ + data: { + backend: { + cache: { + store, + connection: 'redis://localhost:6379', + [store]: { + client: { + namespace, + keyPrefixSeparator: separator, + }, + }, + }, + }, + }, + }), + ); + + manager.forPlugin('testPlugin'); + + if (store === 'redis') { + // eslint-disable-next-line jest/no-conditional-expect + expect(KeyvRedis).toHaveBeenCalledWith('redis://localhost:6379', { + namespace, + keyPrefixSeparator: separator, + }); + } else if (store === 'valkey') { + // eslint-disable-next-line jest/no-conditional-expect + expect(KeyvValkey).toHaveBeenCalledWith('redis://localhost:6379', { + namespace, + keyPrefixSeparator: separator, + }); + } + }); + }); + + it('falls back to pluginId when no namespace is configured', () => { + const manager = CacheManager.fromConfig( + mockServices.rootConfig({ + data: { + backend: { + cache: { + store: 'redis', + connection: 'redis://localhost:6379', + }, + }, + }, + }), + ); + + manager.forPlugin('testPlugin'); + + expect(KeyvRedis).toHaveBeenCalledWith('redis://localhost:6379', { + keyPrefixSeparator: ':', + }); + }); + + describe('Namespace construction', () => { + it('returns pluginId when no store options are provided', () => { + const result = (CacheManager as any).constructNamespace( + 'testPlugin', + undefined, + ); + expect(result).toBe('testPlugin'); + }); + + it('returns pluginId when store options have no namespace', () => { + const storeOptions = { + client: { + keyPrefixSeparator: ':', + }, + }; + const result = (CacheManager as any).constructNamespace( + 'testPlugin', + storeOptions, + ); + expect(result).toBe('testPlugin'); + }); + + it('combines namespace and pluginId with default separator', () => { + const storeOptions = { + client: { + namespace: 'my-app', + keyPrefixSeparator: ':', + }, + }; + const result = (CacheManager as any).constructNamespace( + 'testPlugin', + storeOptions, + ); + expect(result).toBe('my-app:testPlugin'); + }); + + it('combines namespace and pluginId with custom separator', () => { + const storeOptions = { + client: { + namespace: 'my-app', + keyPrefixSeparator: '-', + }, + }; + const result = (CacheManager as any).constructNamespace( + 'testPlugin', + storeOptions, + ); + expect(result).toBe('my-app-testPlugin'); + }); + + it('uses default separator when keyPrefixSeparator is not provided', () => { + const storeOptions = { + client: { + namespace: 'my-app', + }, + }; + const result = (CacheManager as any).constructNamespace( + 'testPlugin', + storeOptions, + ); + expect(result).toBe('my-app:testPlugin'); + }); + + it('handles empty namespace by falling back to pluginId', () => { + const storeOptions = { + client: { + namespace: '', + keyPrefixSeparator: ':', + }, + }; + const result = (CacheManager as any).constructNamespace( + 'testPlugin', + storeOptions, + ); + expect(result).toBe('testPlugin'); + }); + }); }); diff --git a/packages/backend-defaults/src/entrypoints/cache/CacheManager.ts b/packages/backend-defaults/src/entrypoints/cache/CacheManager.ts index 1c26083525..29bea42273 100644 --- a/packages/backend-defaults/src/entrypoints/cache/CacheManager.ts +++ b/packages/backend-defaults/src/entrypoints/cache/CacheManager.ts @@ -213,6 +213,25 @@ export class CacheManager { return redisOptions; } + /** + * Construct the full namespace based on the options and pluginId. + * + * @param pluginId - The plugin ID to namespace + * @param storeOptions - Optional cache store configuration options + * @returns The constructed namespace string combining the configured namespace with pluginId + */ + private static constructNamespace( + pluginId: string, + storeOptions: CacheStoreOptions | undefined, + ): string { + if (storeOptions?.client?.namespace) { + const separator = storeOptions.client.keyPrefixSeparator ?? ':'; + return `${storeOptions.client.namespace}${separator}${pluginId}`; + } + + return pluginId; + } + /** @internal */ constructor( store: string, @@ -286,7 +305,7 @@ export class CacheManager { }); } return new Keyv({ - namespace: pluginId, + namespace: CacheManager.constructNamespace(pluginId, this.storeOptions), ttl: defaultTtl, store: stores[pluginId], emitErrors: false, @@ -326,7 +345,7 @@ export class CacheManager { }); } return new Keyv({ - namespace: pluginId, + namespace: CacheManager.constructNamespace(pluginId, this.storeOptions), ttl: defaultTtl, store: stores[pluginId], emitErrors: false, From 0b9278c6235374ac5df7f2deed24dcb5e677a080 Mon Sep 17 00:00:00 2001 From: Shijun Wang Date: Mon, 8 Sep 2025 13:32:43 +0300 Subject: [PATCH 14/20] fix type errors Signed-off-by: Shijun Wang --- .../src/entrypoints/cache/CacheManager.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/packages/backend-defaults/src/entrypoints/cache/CacheManager.ts b/packages/backend-defaults/src/entrypoints/cache/CacheManager.ts index 29bea42273..06b2b0c57b 100644 --- a/packages/backend-defaults/src/entrypoints/cache/CacheManager.ts +++ b/packages/backend-defaults/src/entrypoints/cache/CacheManager.ts @@ -222,14 +222,15 @@ export class CacheManager { */ private static constructNamespace( pluginId: string, - storeOptions: CacheStoreOptions | undefined, + storeOptions: RedisCacheStoreOptions | undefined, ): string { - if (storeOptions?.client?.namespace) { - const separator = storeOptions.client.keyPrefixSeparator ?? ':'; - return `${storeOptions.client.namespace}${separator}${pluginId}`; - } + const prefix = storeOptions?.client?.namespace + ? `${storeOptions.client.namespace}${ + storeOptions.client.keyPrefixSeparator ?? ':' + }` + : ''; - return pluginId; + return `${prefix}${pluginId}`; } /** @internal */ From 929c55adbc7095a6431512cbb37a22513b94062c Mon Sep 17 00:00:00 2001 From: Shijun Wang Date: Mon, 8 Sep 2025 15:07:15 +0300 Subject: [PATCH 15/20] wait for storage to become ready Signed-off-by: Shijun Wang --- .changeset/heavy-cats-unite.md | 6 ++++++ plugins/home/report.api.md | 2 +- .../CustomHomepage/CustomHomepageGrid.tsx | 14 +++++++++++--- 3 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 .changeset/heavy-cats-unite.md diff --git a/.changeset/heavy-cats-unite.md b/.changeset/heavy-cats-unite.md new file mode 100644 index 0000000000..4875093b0a --- /dev/null +++ b/.changeset/heavy-cats-unite.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-home': patch +--- + +Fixed race condition in CustomHomepageGrid by waiting for storage to load before rendering custom layout to prevent +rendering of the default content. diff --git a/plugins/home/report.api.md b/plugins/home/report.api.md index dd33b952ee..73c6d035cc 100644 --- a/plugins/home/report.api.md +++ b/plugins/home/report.api.md @@ -84,7 +84,7 @@ export const createCardExtension: typeof createCardExtension_2; // @public export const CustomHomepageGrid: ( props: CustomHomepageGridProps, -) => JSX_2.Element; +) => JSX_2.Element | null; // @public export type CustomHomepageGridProps = { diff --git a/plugins/home/src/components/CustomHomepage/CustomHomepageGrid.tsx b/plugins/home/src/components/CustomHomepage/CustomHomepageGrid.tsx index 5d85e834e8..b4b674be1b 100644 --- a/plugins/home/src/components/CustomHomepage/CustomHomepageGrid.tsx +++ b/plugins/home/src/components/CustomHomepage/CustomHomepageGrid.tsx @@ -90,7 +90,7 @@ const useStyles = makeStyles((theme: Theme) => function useHomeStorage( defaultWidgets: GridWidget[], -): [GridWidget[], (value: GridWidget[]) => void] { +): [GridWidget[], (value: GridWidget[]) => void, boolean] { const key = 'home'; const storageApi = useApi(storageApiRef).forBucket('home.customHomepage'); // TODO: Support multiple home pages @@ -110,6 +110,9 @@ function useHomeStorage( storageApi.observe$(key), storageApi.snapshot(key), ); + + const isStorageLoading = homeSnapshot.presence === 'unknown' || !homeSnapshot; + const widgets: GridWidget[] = useMemo(() => { if (homeSnapshot.presence === 'absent') { return defaultWidgets; @@ -122,7 +125,7 @@ function useHomeStorage( } }, [homeSnapshot, defaultWidgets]); - return [widgets, setWidgets]; + return [widgets, setWidgets, isStorageLoading]; } const convertConfigToDefaultWidgets = ( @@ -213,7 +216,7 @@ export const CustomHomepageGrid = (props: CustomHomepageGridProps) => { ? convertConfigToDefaultWidgets(props.config, availableWidgets) : []; }, [props.config, availableWidgets]); - const [widgets, setWidgets] = useHomeStorage(defaultLayout); + const [widgets, setWidgets, isStorageLoading] = useHomeStorage(defaultLayout); const [addWidgetDialogOpen, setAddWidgetDialogOpen] = useState(false); const editModeOn = widgets.find(w => w.layout.isResizable) !== undefined; const [editMode, setEditMode] = useState(editModeOn); @@ -322,6 +325,11 @@ export const CustomHomepageGrid = (props: CustomHomepageGridProps) => { ); }; + // Don't render anything while storage is loading + if (isStorageLoading) { + return null; + } + return ( <> From a2be46e16b8f820fae952e149cb69afe3e20527d Mon Sep 17 00:00:00 2001 From: Shijun Wang Date: Tue, 9 Sep 2025 08:33:25 +0300 Subject: [PATCH 16/20] return progress bar instead of null Signed-off-by: Shijun Wang --- plugins/home/report.api.md | 2 +- .../src/components/CustomHomepage/CustomHomepageGrid.tsx | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/plugins/home/report.api.md b/plugins/home/report.api.md index 73c6d035cc..dd33b952ee 100644 --- a/plugins/home/report.api.md +++ b/plugins/home/report.api.md @@ -84,7 +84,7 @@ export const createCardExtension: typeof createCardExtension_2; // @public export const CustomHomepageGrid: ( props: CustomHomepageGridProps, -) => JSX_2.Element | null; +) => JSX_2.Element; // @public export type CustomHomepageGridProps = { diff --git a/plugins/home/src/components/CustomHomepage/CustomHomepageGrid.tsx b/plugins/home/src/components/CustomHomepage/CustomHomepageGrid.tsx index b4b674be1b..fec7895170 100644 --- a/plugins/home/src/components/CustomHomepage/CustomHomepageGrid.tsx +++ b/plugins/home/src/components/CustomHomepage/CustomHomepageGrid.tsx @@ -34,7 +34,11 @@ import { } from '@material-ui/core/styles'; import { compact } from 'lodash'; import useObservable from 'react-use/esm/useObservable'; -import { ContentHeader, ErrorBoundary } from '@backstage/core-components'; +import { + ContentHeader, + ErrorBoundary, + Progress, +} from '@backstage/core-components'; import Typography from '@material-ui/core/Typography'; import { WidgetSettingsOverlay } from './WidgetSettingsOverlay'; import { AddWidgetDialog } from './AddWidgetDialog'; @@ -325,9 +329,8 @@ export const CustomHomepageGrid = (props: CustomHomepageGridProps) => { ); }; - // Don't render anything while storage is loading if (isStorageLoading) { - return null; + return ; } return ( From 70a332477f5f707f9be89d5263faf2758a3f7e48 Mon Sep 17 00:00:00 2001 From: gyan Date: Tue, 9 Sep 2025 11:09:48 +0530 Subject: [PATCH 17/20] report-alpha.api.md Signed-off-by: gyan --- plugins/user-settings/report-alpha.api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/user-settings/report-alpha.api.md b/plugins/user-settings/report-alpha.api.md index 5a28aacc02..1c8e844d05 100644 --- a/plugins/user-settings/report-alpha.api.md +++ b/plugins/user-settings/report-alpha.api.md @@ -123,7 +123,7 @@ export const userSettingsTranslationRef: TranslationRef< readonly 'languageToggle.select': 'Select language {{language}}'; readonly 'languageToggle.title': 'Language'; readonly 'languageToggle.description': 'Change the language'; - readonly 'themeToggle.select': 'Select theme {{theme}}'; + readonly 'themeToggle.select': 'Select {{theme}}'; readonly 'themeToggle.title': 'Theme'; readonly 'themeToggle.description': 'Change the theme mode'; readonly 'themeToggle.names.auto': 'Auto'; From a79d7cbe36e701d88e024b698ac0b0fe11fcd0f5 Mon Sep 17 00:00:00 2001 From: benjdlambert Date: Tue, 9 Sep 2025 10:32:05 +0200 Subject: [PATCH 18/20] chore: cleanup a little bit Signed-off-by: benjdlambert --- .../operations/stitcher/markForStitching.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.ts b/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.ts index 913936202f..484d6e4ed6 100644 --- a/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.ts +++ b/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.ts @@ -17,7 +17,9 @@ import { Knex } from 'knex'; import splitToChunks from 'lodash/chunk'; import { v4 as uuid } from 'uuid'; +import { ErrorLike, isError } from '@backstage/errors'; import { StitchingStrategy } from '../../../stitching/types'; +import { setTimeout as sleep } from 'timers/promises'; import { DbFinalEntitiesRow, DbRefreshStateRow } from '../../tables'; const UPDATE_CHUNK_SIZE = 100; // Smaller chunks reduce contention @@ -30,10 +32,13 @@ const POSTGRES_DEADLOCK_SQLSTATE = '40P01'; /** * Checks if the given error is a deadlock error for the database engine in use. */ -function isDeadlockError(knex: Knex | Knex.Transaction, e: unknown): boolean { +function isDeadlockError( + knex: Knex | Knex.Transaction, + e: unknown, +): e is ErrorLike { if (knex.client.config.client.includes('pg')) { // PostgreSQL deadlock detection - return (e as any)?.code === POSTGRES_DEADLOCK_SQLSTATE; + return isError(e) && e.code === POSTGRES_DEADLOCK_SQLSTATE; } // Add more database engine checks here as needed @@ -149,7 +154,7 @@ async function retryOnDeadlock( for (;;) { try { return await fn(); - } catch (e: any) { + } catch (e: unknown) { if (isDeadlockError(knex, e) && attempt < retries) { await sleep(baseMs * Math.pow(2, attempt)); attempt++; @@ -159,7 +164,3 @@ async function retryOnDeadlock( } } } - -function sleep(ms: number): Promise { - return new Promise(resolve => setTimeout(resolve, ms)); -} From 1d15f6b01baaefdd8b0b3c4c047382c7a5f84270 Mon Sep 17 00:00:00 2001 From: Andre Wanlin Date: Thu, 7 Aug 2025 06:20:45 -0500 Subject: [PATCH 19/20] Removed references to Lerna as it's not used Signed-off-by: Andre Wanlin --- docs/contribute/project-structure.md | 4 ---- docs/getting-started/index.md | 4 ++-- docs/references/glossary.md | 2 +- lerna.json | 6 ------ packages/config-loader/src/schema/collect.test.ts | 10 ---------- 5 files changed, 3 insertions(+), 23 deletions(-) delete mode 100644 lerna.json diff --git a/docs/contribute/project-structure.md b/docs/contribute/project-structure.md index 5b86b655b2..f58afedee8 100644 --- a/docs/contribute/project-structure.md +++ b/docs/contribute/project-structure.md @@ -221,7 +221,3 @@ future. - [`catalog-info.yaml`](https://github.com/backstage/backstage/tree/master/catalog-info.yaml) - Description of Backstage in the Backstage Entity format. - -- [`lerna.json`](https://github.com/backstage/backstage/tree/master/lerna.json) - - [Lerna](https://github.com/lerna/lerna) monorepo config. We are using - `yarn workspaces`, so this will only be used for executing scripts. diff --git a/docs/getting-started/index.md b/docs/getting-started/index.md index cd86aa3343..9fb29f693e 100644 --- a/docs/getting-started/index.md +++ b/docs/getting-started/index.md @@ -89,8 +89,8 @@ app - **package.json**: Root package.json for the project. _Note: Be sure that you don't add any npm dependencies here as they probably should be installed in the intended workspace rather than in the root._ -- **packages/**: Lerna leaf packages or "workspaces". Everything here is going - to be a separate package, managed by lerna. +- **packages/**: Yarn workspaces, everything here is going + to be a separate package, managed by Yarn. - **packages/app/**: A fully functioning Backstage frontend app that acts as a good starting point for you to get to know Backstage. - **packages/backend/**: We include a backend that helps power features such as diff --git a/docs/references/glossary.md b/docs/references/glossary.md index c955d9a5ae..ad9e2bbee5 100644 --- a/docs/references/glossary.md +++ b/docs/references/glossary.md @@ -208,7 +208,7 @@ One of the [packages](#package) within a [monorepo](#monorepo). A package may or 1. A single repository for a collection of related software projects, such as all projects belonging to an organization. -2. A project layout that consists of multiple [packages](#package) within a single project, where packages are able to have local dependencies on each other. Often enabled through tooling such as [lerna](https://lerna.js.org/) and [yarn workspaces](https://classic.yarnpkg.com/en/docs/workspaces/) +2. A project layout that consists of multiple [packages](#package) within a single project, where packages are able to have local dependencies on each other. Often enabled through tooling such as [yarn workspaces](https://classic.yarnpkg.com/en/docs/workspaces/) ## Name diff --git a/lerna.json b/lerna.json deleted file mode 100644 index 4621689664..0000000000 --- a/lerna.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "packages": ["packages/*", "plugins/*"], - "npmClient": "yarn", - "useWorkspaces": true, - "version": "0.0.0" -} diff --git a/packages/config-loader/src/schema/collect.test.ts b/packages/config-loader/src/schema/collect.test.ts index 53db5c6da7..d48c98fbe4 100644 --- a/packages/config-loader/src/schema/collect.test.ts +++ b/packages/config-loader/src/schema/collect.test.ts @@ -41,16 +41,6 @@ describe('collectConfigSchemas', () => { mockDir.clear(); }); - it('should not find any schemas without packages', async () => { - mockDir.setContent({ - 'lerna.json': JSON.stringify({ - packages: ['packages/*'], - }), - }); - - await expect(collectConfigSchemas([], [])).resolves.toEqual([]); - }); - it('should find schema in a local package', async () => { mockDir.setContent({ node_modules: { From 675814c0962df46a37abc30664360bc74c938dd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Tue, 9 Sep 2025 13:29:58 +0200 Subject: [PATCH 20/20] Update .changeset/warm-emus-itch.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/warm-emus-itch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/warm-emus-itch.md b/.changeset/warm-emus-itch.md index 3275098425..a7bd51abdb 100644 --- a/.changeset/warm-emus-itch.md +++ b/.changeset/warm-emus-itch.md @@ -2,4 +2,4 @@ '@backstage/plugin-catalog': patch --- -Fix incorrect defaultTarget on `createComponentRouteRef`. +Fix incorrect `defaultTarget` on `createComponentRouteRef`.