diff --git a/.changeset/better-eagles-tickle.md b/.changeset/better-eagles-tickle.md new file mode 100644 index 0000000000..418f0bcae0 --- /dev/null +++ b/.changeset/better-eagles-tickle.md @@ -0,0 +1,5 @@ +--- +'@backstage/config-loader': patch +--- + +Allow using `BACKSTAGE_ENV` for loading environment specific config files diff --git a/.changeset/big-cameras-turn.md b/.changeset/big-cameras-turn.md new file mode 100644 index 0000000000..6226b17806 --- /dev/null +++ b/.changeset/big-cameras-turn.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-defaults': patch +--- + +Fixed cache namespace and key prefix separator configuration to properly use configured values instead of hardcoded plugin ID. The cache manager now correctly combines the configured namespace with plugin IDs using the configured separator for Redis and Valkey. Memcache and memory store continue to use plugin ID as namespace. diff --git a/.changeset/heavy-cats-unite.md b/.changeset/heavy-cats-unite.md new file mode 100644 index 0000000000..4875093b0a --- /dev/null +++ b/.changeset/heavy-cats-unite.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-home': patch +--- + +Fixed race condition in CustomHomepageGrid by waiting for storage to load before rendering custom layout to prevent +rendering of the default content. diff --git a/.changeset/itchy-moons-start.md b/.changeset/itchy-moons-start.md new file mode 100644 index 0000000000..e5768b58b2 --- /dev/null +++ b/.changeset/itchy-moons-start.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder': patch +--- + +Render a TechDocs link on the Scaffolder Template List page when templates include either `backstage.io/techdocs-ref` or `backstage.io/techdocs-entity` annotations, using the shared `buildTechDocsURL` helper. Also adds tests to verify both annotations and optional `backstage.io/techdocs-entity-path` are respected. diff --git a/.changeset/late-swans-press.md b/.changeset/late-swans-press.md new file mode 100644 index 0000000000..987672bd77 --- /dev/null +++ b/.changeset/late-swans-press.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Prevent deadlock in catalog deferred stitching diff --git a/.changeset/red-shrimps-fall.md b/.changeset/red-shrimps-fall.md new file mode 100644 index 0000000000..1ee3aca7ed --- /dev/null +++ b/.changeset/red-shrimps-fall.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-user-settings': patch +--- + +Tool-tip text correction for the Theme selection in settings page diff --git a/.changeset/tangy-squids-film.md b/.changeset/tangy-squids-film.md new file mode 100644 index 0000000000..acb5546031 --- /dev/null +++ b/.changeset/tangy-squids-film.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-react': patch +--- + +Fix scaffolder task log stream not having a minimum height diff --git a/.changeset/warm-emus-itch.md b/.changeset/warm-emus-itch.md new file mode 100644 index 0000000000..a7bd51abdb --- /dev/null +++ b/.changeset/warm-emus-itch.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog': patch +--- + +Fix incorrect `defaultTarget` on `createComponentRouteRef`. diff --git a/docs/backend-system/core-services/cache.md b/docs/backend-system/core-services/cache.md index 0e26e8e1f5..ad4d04b50a 100644 --- a/docs/backend-system/core-services/cache.md +++ b/docs/backend-system/core-services/cache.md @@ -7,6 +7,39 @@ description: Documentation for the Cache service This service lets your plugin interact with a cache. It is bound to your plugin too, so that you will only set and get values in your plugin's private namespace. +## Configuration + +The cache service can be configured using the `backend.cache` section in your `app-config.yaml`: + +```yaml +backend: + cache: + store: redis # or 'valkey', 'memcache', 'memory' + connection: redis://localhost:6379 + + # Store-specific configuration (Redis/Valkey only) + redis: + client: + # Optional: Global namespace prefix for all cache keys + namespace: 'my-app' + # Optional: Separator used between namespace and plugin ID (default: ':') + keyPrefixSeparator: ':' + # Other Redis-specific options... + clearBatchSize: 1000 + useUnlink: false +``` + +### Namespace Configuration + +For Redis and Valkey stores, you can configure a global namespace that will be prefixed to all cache keys: + +- **Without namespace**: Cache keys use only the plugin ID (e.g., `catalog:some-key`) +- **With namespace**: Cache keys use the format `namespace:pluginId:key` (e.g., `my-app:catalog:some-key`) + +The `keyPrefixSeparator` controls what character is used between the namespace and plugin ID (defaults to `:`). + +**Note**: Memory and Memcache stores do not support namespace configuration and will always use the plugin ID directly. + ## Using the service The following example shows how to get a cache client in your `example` backend plugin and setting and getting values from the cache. diff --git a/docs/conf/index.md b/docs/conf/index.md index 659ffccfe3..029c5b88fd 100644 --- a/docs/conf/index.md +++ b/docs/conf/index.md @@ -16,10 +16,24 @@ allowing for customization. ## Supplying Configuration Configuration is stored in YAML files where the defaults are `app-config.yaml` -and `app-config.local.yaml` for local overrides. Other sets of files can by -loaded by passing `--config ` flags. The configuration files themselves -contain plain YAML, but with support for loading in data and secrets from -various sources using for example `$env` and `$file` keys. +and `app-config.local.yaml` for local overrides. Additionally, it is possible +to define environment based configuration files with `BACKSTAGE_ENV` +environment variable, which will load `app-config..yaml`. + +Loading order of these files is as follows: + +1. `app-config.yaml` +2. `app-config..yaml` +3. `app-config.local.yaml` +4. `app-config..local.yaml` + +Other sets of files can by loaded by passing `--config ` flags. +Read more about the configuration loading order in the +[Configuration Files](./writing.md#configuration-files) section. + +The configuration files themselves contain plain YAML, but with support for +loading in data and secrets from various sources using for example +`$env` and `$file` keys. It is also possible to supply configuration through environment variables, for example `APP_CONFIG_app_baseUrl=https://staging.example.com`. However these diff --git a/docs/contribute/project-structure.md b/docs/contribute/project-structure.md index 5b86b655b2..f58afedee8 100644 --- a/docs/contribute/project-structure.md +++ b/docs/contribute/project-structure.md @@ -221,7 +221,3 @@ future. - [`catalog-info.yaml`](https://github.com/backstage/backstage/tree/master/catalog-info.yaml) - Description of Backstage in the Backstage Entity format. - -- [`lerna.json`](https://github.com/backstage/backstage/tree/master/lerna.json) - - [Lerna](https://github.com/lerna/lerna) monorepo config. We are using - `yarn workspaces`, so this will only be used for executing scripts. diff --git a/docs/getting-started/index.md b/docs/getting-started/index.md index 611002bf02..252884dc54 100644 --- a/docs/getting-started/index.md +++ b/docs/getting-started/index.md @@ -89,8 +89,8 @@ app - **package.json**: Root package.json for the project. _Note: Be sure that you don't add any npm dependencies here as they probably should be installed in the intended workspace rather than in the root._ -- **packages/**: Lerna leaf packages or "workspaces". Everything here is going - to be a separate package managed by Lerna. +- **packages/**: Yarn workspaces, everything here is going + to be a separate package, managed by Yarn. - **packages/app/**: A fully functioning Backstage frontend app that acts as a good starting point for you to get to know Backstage. - **packages/backend/**: We include a backend that helps power features such as diff --git a/docs/references/glossary.md b/docs/references/glossary.md index c955d9a5ae..ad9e2bbee5 100644 --- a/docs/references/glossary.md +++ b/docs/references/glossary.md @@ -208,7 +208,7 @@ One of the [packages](#package) within a [monorepo](#monorepo). A package may or 1. A single repository for a collection of related software projects, such as all projects belonging to an organization. -2. A project layout that consists of multiple [packages](#package) within a single project, where packages are able to have local dependencies on each other. Often enabled through tooling such as [lerna](https://lerna.js.org/) and [yarn workspaces](https://classic.yarnpkg.com/en/docs/workspaces/) +2. A project layout that consists of multiple [packages](#package) within a single project, where packages are able to have local dependencies on each other. Often enabled through tooling such as [yarn workspaces](https://classic.yarnpkg.com/en/docs/workspaces/) ## Name diff --git a/lerna.json b/lerna.json deleted file mode 100644 index 4621689664..0000000000 --- a/lerna.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "packages": ["packages/*", "plugins/*"], - "npmClient": "yarn", - "useWorkspaces": true, - "version": "0.0.0" -} diff --git a/packages/app-next/app-config.yaml b/packages/app-next/app-config.yaml index f83e4d4ba0..35380c961f 100644 --- a/packages/app-next/app-config.yaml +++ b/packages/app-next/app-config.yaml @@ -4,7 +4,6 @@ app: routes: bindings: catalog.viewTechDoc: techdocs.docRoot - catalog.createComponent: catalog-import.importPage org.catalogIndex: catalog.catalogIndex pluginOverrides: diff --git a/packages/backend-defaults/src/entrypoints/cache/CacheManager.test.ts b/packages/backend-defaults/src/entrypoints/cache/CacheManager.test.ts index 4e9475b66e..a308caa222 100644 --- a/packages/backend-defaults/src/entrypoints/cache/CacheManager.test.ts +++ b/packages/backend-defaults/src/entrypoints/cache/CacheManager.test.ts @@ -333,4 +333,147 @@ describe('CacheManager store options', () => { keyPrefixSeparator: '!', }); }); + + it('correctly applies namespace configuration to redis and valkey stores', () => { + const testCases = [ + { store: 'redis', namespace: 'test1', separator: ':' }, + { store: 'valkey', namespace: 'test2', separator: '!' }, + ]; + + testCases.forEach(({ store, namespace, separator }) => { + const manager = CacheManager.fromConfig( + mockServices.rootConfig({ + data: { + backend: { + cache: { + store, + connection: 'redis://localhost:6379', + [store]: { + client: { + namespace, + keyPrefixSeparator: separator, + }, + }, + }, + }, + }, + }), + ); + + manager.forPlugin('testPlugin'); + + if (store === 'redis') { + // eslint-disable-next-line jest/no-conditional-expect + expect(KeyvRedis).toHaveBeenCalledWith('redis://localhost:6379', { + namespace, + keyPrefixSeparator: separator, + }); + } else if (store === 'valkey') { + // eslint-disable-next-line jest/no-conditional-expect + expect(KeyvValkey).toHaveBeenCalledWith('redis://localhost:6379', { + namespace, + keyPrefixSeparator: separator, + }); + } + }); + }); + + it('falls back to pluginId when no namespace is configured', () => { + const manager = CacheManager.fromConfig( + mockServices.rootConfig({ + data: { + backend: { + cache: { + store: 'redis', + connection: 'redis://localhost:6379', + }, + }, + }, + }), + ); + + manager.forPlugin('testPlugin'); + + expect(KeyvRedis).toHaveBeenCalledWith('redis://localhost:6379', { + keyPrefixSeparator: ':', + }); + }); + + describe('Namespace construction', () => { + it('returns pluginId when no store options are provided', () => { + const result = (CacheManager as any).constructNamespace( + 'testPlugin', + undefined, + ); + expect(result).toBe('testPlugin'); + }); + + it('returns pluginId when store options have no namespace', () => { + const storeOptions = { + client: { + keyPrefixSeparator: ':', + }, + }; + const result = (CacheManager as any).constructNamespace( + 'testPlugin', + storeOptions, + ); + expect(result).toBe('testPlugin'); + }); + + it('combines namespace and pluginId with default separator', () => { + const storeOptions = { + client: { + namespace: 'my-app', + keyPrefixSeparator: ':', + }, + }; + const result = (CacheManager as any).constructNamespace( + 'testPlugin', + storeOptions, + ); + expect(result).toBe('my-app:testPlugin'); + }); + + it('combines namespace and pluginId with custom separator', () => { + const storeOptions = { + client: { + namespace: 'my-app', + keyPrefixSeparator: '-', + }, + }; + const result = (CacheManager as any).constructNamespace( + 'testPlugin', + storeOptions, + ); + expect(result).toBe('my-app-testPlugin'); + }); + + it('uses default separator when keyPrefixSeparator is not provided', () => { + const storeOptions = { + client: { + namespace: 'my-app', + }, + }; + const result = (CacheManager as any).constructNamespace( + 'testPlugin', + storeOptions, + ); + expect(result).toBe('my-app:testPlugin'); + }); + + it('handles empty namespace by falling back to pluginId', () => { + const storeOptions = { + client: { + namespace: '', + keyPrefixSeparator: ':', + }, + }; + const result = (CacheManager as any).constructNamespace( + 'testPlugin', + storeOptions, + ); + expect(result).toBe('testPlugin'); + }); + }); }); diff --git a/packages/backend-defaults/src/entrypoints/cache/CacheManager.ts b/packages/backend-defaults/src/entrypoints/cache/CacheManager.ts index 1c26083525..06b2b0c57b 100644 --- a/packages/backend-defaults/src/entrypoints/cache/CacheManager.ts +++ b/packages/backend-defaults/src/entrypoints/cache/CacheManager.ts @@ -213,6 +213,26 @@ export class CacheManager { return redisOptions; } + /** + * Construct the full namespace based on the options and pluginId. + * + * @param pluginId - The plugin ID to namespace + * @param storeOptions - Optional cache store configuration options + * @returns The constructed namespace string combining the configured namespace with pluginId + */ + private static constructNamespace( + pluginId: string, + storeOptions: RedisCacheStoreOptions | undefined, + ): string { + const prefix = storeOptions?.client?.namespace + ? `${storeOptions.client.namespace}${ + storeOptions.client.keyPrefixSeparator ?? ':' + }` + : ''; + + return `${prefix}${pluginId}`; + } + /** @internal */ constructor( store: string, @@ -286,7 +306,7 @@ export class CacheManager { }); } return new Keyv({ - namespace: pluginId, + namespace: CacheManager.constructNamespace(pluginId, this.storeOptions), ttl: defaultTtl, store: stores[pluginId], emitErrors: false, @@ -326,7 +346,7 @@ export class CacheManager { }); } return new Keyv({ - namespace: pluginId, + namespace: CacheManager.constructNamespace(pluginId, this.storeOptions), ttl: defaultTtl, store: stores[pluginId], emitErrors: false, diff --git a/packages/config-loader/src/schema/collect.test.ts b/packages/config-loader/src/schema/collect.test.ts index 53db5c6da7..d48c98fbe4 100644 --- a/packages/config-loader/src/schema/collect.test.ts +++ b/packages/config-loader/src/schema/collect.test.ts @@ -41,16 +41,6 @@ describe('collectConfigSchemas', () => { mockDir.clear(); }); - it('should not find any schemas without packages', async () => { - mockDir.setContent({ - 'lerna.json': JSON.stringify({ - packages: ['packages/*'], - }), - }); - - await expect(collectConfigSchemas([], [])).resolves.toEqual([]); - }); - it('should find schema in a local package', async () => { mockDir.setContent({ node_modules: { diff --git a/packages/config-loader/src/sources/ConfigSources.test.ts b/packages/config-loader/src/sources/ConfigSources.test.ts index e19ac1d51f..a6b0648889 100644 --- a/packages/config-loader/src/sources/ConfigSources.test.ts +++ b/packages/config-loader/src/sources/ConfigSources.test.ts @@ -89,6 +89,19 @@ describe('ConfigSources', () => { { name: 'FileConfigSource', path: `${root}app-config.yaml` }, { name: 'FileConfigSource', path: `${root}app-config.local.yaml` }, ]); + + process.env = Object.assign(process.env, { BACKSTAGE_ENV: 'test' }); + expect( + mergeSources( + ConfigSources.defaultForTargets({ rootDir: '/', targets: [] }), + ), + ).toEqual([ + { name: 'FileConfigSource', path: `${root}app-config.yaml` }, + { name: 'FileConfigSource', path: `${root}app-config.test.yaml` }, + { name: 'FileConfigSource', path: `${root}app-config.local.yaml` }, + { name: 'FileConfigSource', path: `${root}app-config.test.local.yaml` }, + ]); + fsSpy.mockRestore(); expect( diff --git a/packages/config-loader/src/sources/ConfigSources.ts b/packages/config-loader/src/sources/ConfigSources.ts index 37c1c6b087..18d0443986 100644 --- a/packages/config-loader/src/sources/ConfigSources.ts +++ b/packages/config-loader/src/sources/ConfigSources.ts @@ -182,6 +182,14 @@ export class ConfigSources { if (argSources.length === 0) { const defaultPath = resolvePath(rootDir, 'app-config.yaml'); const localPath = resolvePath(rootDir, 'app-config.local.yaml'); + const envPath = resolvePath( + rootDir, + `app-config.${process.env.BACKSTAGE_ENV}.yaml`, + ); + const envLocalPath = resolvePath( + rootDir, + `app-config.${process.env.BACKSTAGE_ENV}.local.yaml`, + ); const alwaysIncludeDefaultConfigSource = !options.allowMissingDefaultConfig; @@ -195,6 +203,16 @@ export class ConfigSources { ); } + if (process.env.BACKSTAGE_ENV && fs.pathExistsSync(envPath)) { + argSources.push( + FileConfigSource.create({ + watch: options.watch, + path: envPath, + substitutionFunc: options.substitutionFunc, + }), + ); + } + if (fs.pathExistsSync(localPath)) { argSources.push( FileConfigSource.create({ @@ -204,6 +222,16 @@ export class ConfigSources { }), ); } + + if (process.env.BACKSTAGE_ENV && fs.pathExistsSync(envLocalPath)) { + argSources.push( + FileConfigSource.create({ + watch: options.watch, + path: envLocalPath, + substitutionFunc: options.substitutionFunc, + }), + ); + } } return this.merge(argSources); diff --git a/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.test.ts b/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.test.ts index 1afa0b9c30..ff450e9ccf 100644 --- a/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.test.ts +++ b/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.test.ts @@ -435,4 +435,139 @@ describe('markForStitching', () => { } }, ); + + it.each(databases.eachSupportedId())( + 'reproduces deadlock scenario when concurrent transactions update overlapping entity sets %p', + async databaseId => { + const knex = await databases.init(databaseId); + await applyDatabaseMigrations(knex); + + // Setup test data with multiple entities + const entityRefs = [ + 'k:ns/entity-a', + 'k:ns/entity-b', + 'k:ns/entity-c', + 'k:ns/entity-d', + 'k:ns/entity-e', + 'k:ns/entity-f', + ]; + + await knex('refresh_state').insert( + entityRefs.map((ref, i) => ({ + entity_id: `${i + 1}`, + entity_ref: ref, + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '[]', + next_update_at: knex.fn.now(), + last_discovery_at: knex.fn.now(), + next_stitch_at: null, + next_stitch_ticket: null, + })), + ); + + // This test attempts to reproduce the deadlock by running concurrent transactions + // that update overlapping sets of entities in different orders + const errorResults = []; + + for (let attempt = 0; attempt < 10; attempt++) { + // Transaction 1: Update entities A, B, C, D, E + const transaction1 = knex.transaction(async trx => { + await markForStitching({ + knex: trx, + strategy: { + mode: 'deferred', + pollingInterval: { seconds: 1 }, + stitchTimeout: { seconds: 1 }, + }, + entityRefs: [ + 'k:ns/entity-a', + 'k:ns/entity-b', + 'k:ns/entity-c', + 'k:ns/entity-d', + 'k:ns/entity-e', + ], + }); + + // Add a small delay to increase chance of collision + await new Promise(resolve => setTimeout(resolve, 10)); + + await markForStitching({ + knex: trx, + strategy: { + mode: 'deferred', + pollingInterval: { seconds: 1 }, + stitchTimeout: { seconds: 1 }, + }, + entityRefs: ['k:ns/entity-f'], + }); + }); + + // Transaction 2: Update entities F, E, D, C, B (reverse order) + const transaction2 = knex.transaction(async trx => { + await markForStitching({ + knex: trx, + strategy: { + mode: 'deferred', + pollingInterval: { seconds: 1 }, + stitchTimeout: { seconds: 1 }, + }, + entityRefs: [ + 'k:ns/entity-f', + 'k:ns/entity-e', + 'k:ns/entity-d', + 'k:ns/entity-c', + 'k:ns/entity-b', + ], + }); + + // Add a small delay to increase chance of collision + await new Promise(resolve => setTimeout(resolve, 10)); + + await markForStitching({ + knex: trx, + strategy: { + mode: 'deferred', + pollingInterval: { seconds: 1 }, + stitchTimeout: { seconds: 1 }, + }, + entityRefs: ['k:ns/entity-a'], + }); + }); + + // Run both transactions concurrently to create potential deadlock + errorResults.push( + Promise.allSettled([transaction1, transaction2]).then(results => + results + .filter(r => r.status === 'rejected') + .map(r => (r as PromiseRejectedResult).reason), + ), + ); + } + + const allResults = await Promise.all(errorResults); + + const deadlockErrors = allResults + .flat() + .filter( + error => + error?.code === '40P01' || + error?.message?.includes('deadlock detected') || + error?.message?.includes('deadlock'), + ); + expect(deadlockErrors).toEqual([]); + + // Verify final state - all entities should have been marked for stitching + const finalState = await knex('refresh_state') + .select('entity_ref', 'next_stitch_at', 'next_stitch_ticket') + .whereNotNull('next_stitch_at') + .orderBy('entity_ref'); + + expect(finalState.length).toBeGreaterThan(0); + finalState.forEach(row => { + expect(row.next_stitch_at).not.toBeNull(); + expect(row.next_stitch_ticket).not.toBeNull(); + }); + }, + ); }); diff --git a/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.ts b/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.ts index ecc364a9cc..484d6e4ed6 100644 --- a/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.ts +++ b/plugins/catalog-backend/src/database/operations/stitcher/markForStitching.ts @@ -17,9 +17,34 @@ import { Knex } from 'knex'; import splitToChunks from 'lodash/chunk'; import { v4 as uuid } from 'uuid'; +import { ErrorLike, isError } from '@backstage/errors'; import { StitchingStrategy } from '../../../stitching/types'; +import { setTimeout as sleep } from 'timers/promises'; import { DbFinalEntitiesRow, DbRefreshStateRow } from '../../tables'; +const UPDATE_CHUNK_SIZE = 100; // Smaller chunks reduce contention +const DEADLOCK_RETRY_ATTEMPTS = 3; +const DEADLOCK_BASE_DELAY_MS = 25; + +// PostgreSQL deadlock error code +const POSTGRES_DEADLOCK_SQLSTATE = '40P01'; + +/** + * Checks if the given error is a deadlock error for the database engine in use. + */ +function isDeadlockError( + knex: Knex | Knex.Transaction, + e: unknown, +): e is ErrorLike { + if (knex.client.config.client.includes('pg')) { + // PostgreSQL deadlock detection + return isError(e) && e.code === POSTGRES_DEADLOCK_SQLSTATE; + } + + // Add more database engine checks here as needed + return false; +} + /** * Marks a number of entities for stitching some time in the near * future. @@ -32,9 +57,8 @@ export async function markForStitching(options: { entityRefs?: Iterable; entityIds?: Iterable; }): Promise { - // Splitting to chunks just to cover pathological cases that upset the db - const entityRefs = split(options.entityRefs); - const entityIds = split(options.entityIds); + const entityRefs = sortSplit(options.entityRefs); + const entityIds = sortSplit(options.entityIds); const knex = options.knex; const mode = options.strategy.mode; @@ -51,13 +75,15 @@ export async function markForStitching(options: { .select('entity_id') .whereIn('entity_ref', chunk), ); - await knex - .table('refresh_state') - .update({ - result_hash: 'force-stitching', - next_update_at: knex.fn.now(), - }) - .whereIn('entity_ref', chunk); + await retryOnDeadlock(async () => { + await knex + .table('refresh_state') + .update({ + result_hash: 'force-stitching', + next_update_at: knex.fn.now(), + }) + .whereIn('entity_ref', chunk); + }, knex); } for (const chunk of entityIds) { @@ -67,44 +93,74 @@ export async function markForStitching(options: { hash: 'force-stitching', }) .whereIn('entity_id', chunk); - await knex - .table('refresh_state') - .update({ - result_hash: 'force-stitching', - next_update_at: knex.fn.now(), - }) - .whereIn('entity_id', chunk); + await retryOnDeadlock(async () => { + await knex + .table('refresh_state') + .update({ + result_hash: 'force-stitching', + next_update_at: knex.fn.now(), + }) + .whereIn('entity_id', chunk); + }, knex); } } else if (mode === 'deferred') { // It's OK that this is shared across refresh state rows; it just needs to // be uniquely generated for every new stitch request. const ticket = uuid(); + // Update by primary key in deterministic order to avoid deadlocks for (const chunk of entityRefs) { - await knex('refresh_state') - .update({ - next_stitch_at: knex.fn.now(), - next_stitch_ticket: ticket, - }) - .whereIn('entity_ref', chunk); + await retryOnDeadlock(async () => { + await knex('refresh_state') + .update({ + next_stitch_at: knex.fn.now(), + next_stitch_ticket: ticket, + }) + .whereIn('entity_ref', chunk); + }, knex); } for (const chunk of entityIds) { - await knex('refresh_state') - .update({ - next_stitch_at: knex.fn.now(), - next_stitch_ticket: ticket, - }) - .whereIn('entity_id', chunk); + await retryOnDeadlock(async () => { + await knex('refresh_state') + .update({ + next_stitch_at: knex.fn.now(), + next_stitch_ticket: ticket, + }) + .whereIn('entity_id', chunk); + }, knex); } } else { throw new Error(`Unknown stitching strategy mode ${mode}`); } } -function split(input: Iterable | undefined): string[][] { +function sortSplit(input: Iterable | undefined): string[][] { if (!input) { return []; } - return splitToChunks(Array.isArray(input) ? input : [...input], 200); + const array = Array.isArray(input) ? input.slice() : [...input]; + array.sort(); + return splitToChunks(array, UPDATE_CHUNK_SIZE); +} + +async function retryOnDeadlock( + fn: () => Promise, + knex: Knex | Knex.Transaction, + retries = DEADLOCK_RETRY_ATTEMPTS, + baseMs = DEADLOCK_BASE_DELAY_MS, +): Promise { + let attempt = 0; + for (;;) { + try { + return await fn(); + } catch (e: unknown) { + if (isDeadlockError(knex, e) && attempt < retries) { + await sleep(baseMs * Math.pow(2, attempt)); + attempt++; + continue; + } + throw e; + } + } } diff --git a/plugins/catalog/src/routes.ts b/plugins/catalog/src/routes.ts index 4f7f0427b1..6043ecd1a1 100644 --- a/plugins/catalog/src/routes.ts +++ b/plugins/catalog/src/routes.ts @@ -22,7 +22,7 @@ import { export const createComponentRouteRef = createExternalRouteRef({ id: 'create-component', optional: true, - defaultTarget: 'scaffolder.createComponent', + defaultTarget: 'scaffolder.root', }); export const viewTechDocRouteRef = createExternalRouteRef({ diff --git a/plugins/home/src/components/CustomHomepage/CustomHomepageGrid.tsx b/plugins/home/src/components/CustomHomepage/CustomHomepageGrid.tsx index 5d85e834e8..fec7895170 100644 --- a/plugins/home/src/components/CustomHomepage/CustomHomepageGrid.tsx +++ b/plugins/home/src/components/CustomHomepage/CustomHomepageGrid.tsx @@ -34,7 +34,11 @@ import { } from '@material-ui/core/styles'; import { compact } from 'lodash'; import useObservable from 'react-use/esm/useObservable'; -import { ContentHeader, ErrorBoundary } from '@backstage/core-components'; +import { + ContentHeader, + ErrorBoundary, + Progress, +} from '@backstage/core-components'; import Typography from '@material-ui/core/Typography'; import { WidgetSettingsOverlay } from './WidgetSettingsOverlay'; import { AddWidgetDialog } from './AddWidgetDialog'; @@ -90,7 +94,7 @@ const useStyles = makeStyles((theme: Theme) => function useHomeStorage( defaultWidgets: GridWidget[], -): [GridWidget[], (value: GridWidget[]) => void] { +): [GridWidget[], (value: GridWidget[]) => void, boolean] { const key = 'home'; const storageApi = useApi(storageApiRef).forBucket('home.customHomepage'); // TODO: Support multiple home pages @@ -110,6 +114,9 @@ function useHomeStorage( storageApi.observe$(key), storageApi.snapshot(key), ); + + const isStorageLoading = homeSnapshot.presence === 'unknown' || !homeSnapshot; + const widgets: GridWidget[] = useMemo(() => { if (homeSnapshot.presence === 'absent') { return defaultWidgets; @@ -122,7 +129,7 @@ function useHomeStorage( } }, [homeSnapshot, defaultWidgets]); - return [widgets, setWidgets]; + return [widgets, setWidgets, isStorageLoading]; } const convertConfigToDefaultWidgets = ( @@ -213,7 +220,7 @@ export const CustomHomepageGrid = (props: CustomHomepageGridProps) => { ? convertConfigToDefaultWidgets(props.config, availableWidgets) : []; }, [props.config, availableWidgets]); - const [widgets, setWidgets] = useHomeStorage(defaultLayout); + const [widgets, setWidgets, isStorageLoading] = useHomeStorage(defaultLayout); const [addWidgetDialogOpen, setAddWidgetDialogOpen] = useState(false); const editModeOn = widgets.find(w => w.layout.isResizable) !== undefined; const [editMode, setEditMode] = useState(editModeOn); @@ -322,6 +329,10 @@ export const CustomHomepageGrid = (props: CustomHomepageGridProps) => { ); }; + if (isStorageLoading) { + return ; + } + return ( <> diff --git a/plugins/scaffolder-react/src/next/components/TaskLogStream/TaskLogStream.tsx b/plugins/scaffolder-react/src/next/components/TaskLogStream/TaskLogStream.tsx index a0c4ce94cd..b88d96510e 100644 --- a/plugins/scaffolder-react/src/next/components/TaskLogStream/TaskLogStream.tsx +++ b/plugins/scaffolder-react/src/next/components/TaskLogStream/TaskLogStream.tsx @@ -21,6 +21,7 @@ const useStyles = makeStyles({ width: '100%', height: '100%', position: 'relative', + minHeight: 240, }, }); diff --git a/plugins/scaffolder/package.json b/plugins/scaffolder/package.json index bad13ef8d2..d9a63c7661 100644 --- a/plugins/scaffolder/package.json +++ b/plugins/scaffolder/package.json @@ -72,6 +72,8 @@ "@backstage/plugin-permission-react": "workspace:^", "@backstage/plugin-scaffolder-common": "workspace:^", "@backstage/plugin-scaffolder-react": "workspace:^", + "@backstage/plugin-techdocs-common": "workspace:^", + "@backstage/plugin-techdocs-react": "workspace:^", "@backstage/types": "workspace:^", "@codemirror/language": "^6.0.0", "@codemirror/legacy-modes": "^6.1.0", @@ -109,6 +111,7 @@ "@backstage/dev-utils": "workspace:^", "@backstage/plugin-catalog": "workspace:^", "@backstage/plugin-permission-common": "workspace:^", + "@backstage/plugin-techdocs": "workspace:^", "@backstage/test-utils": "workspace:^", "@testing-library/dom": "^10.0.0", "@testing-library/jest-dom": "^6.0.0", diff --git a/plugins/scaffolder/src/alpha/components/TemplateListPage/TemplateListPage.test.tsx b/plugins/scaffolder/src/alpha/components/TemplateListPage/TemplateListPage.test.tsx index a16474a4cf..da544cb2c0 100644 --- a/plugins/scaffolder/src/alpha/components/TemplateListPage/TemplateListPage.test.tsx +++ b/plugins/scaffolder/src/alpha/components/TemplateListPage/TemplateListPage.test.tsx @@ -27,13 +27,19 @@ import { TestApiProvider, mockApis, } from '@backstage/test-utils'; -import { rootRouteRef } from '../../../routes'; +import { rootRouteRef, viewTechDocRouteRef } from '../../../routes'; import { TemplateListPage } from './TemplateListPage'; +import { + TECHDOCS_ANNOTATION, + TECHDOCS_EXTERNAL_ANNOTATION, + TECHDOCS_EXTERNAL_PATH_ANNOTATION, +} from '@backstage/plugin-techdocs-common'; const mountedRoutes = { mountedRoutes: { '/': rootRouteRef, '/catalog/:namespace/:kind/:name': entityRouteRef, + '/docs/:namespace/:kind/:name': viewTechDocRouteRef, }, }; @@ -51,6 +57,127 @@ describe('TemplateListPage', () => { ], }); + describe('TechDocs link rendering', () => { + it('shows TechDocs link when template has backstage.io/techdocs-ref', async () => { + const mockCatalogApiWithDocs = catalogApiMock({ + entities: [ + { + apiVersion: 'scaffolder.backstage.io/v1beta3', + kind: 'Template', + metadata: { + name: 'tmpl-a', + annotations: { [TECHDOCS_ANNOTATION]: 'dir:.' }, + }, + spec: { type: 'service' }, + }, + ], + }); + + const { findByText } = await renderInTestApp( + + + , + mountedRoutes, + ); + + expect(await findByText('View TechDocs')).toBeInTheDocument(); + }); + + it('shows TechDocs link when template has backstage.io/techdocs-entity', async () => { + const mockCatalogApiWithExternal = catalogApiMock({ + entities: [ + { + apiVersion: 'scaffolder.backstage.io/v1beta3', + kind: 'Template', + metadata: { + name: 'tmpl-b', + annotations: { + [TECHDOCS_EXTERNAL_ANNOTATION]: 'component:default/other', + }, + }, + spec: { type: 'service' }, + }, + ], + }); + + const { findByText } = await renderInTestApp( + + + , + mountedRoutes, + ); + + expect(await findByText('View TechDocs')).toBeInTheDocument(); + }); + + it('appends path when backstage.io/techdocs-entity-path is set', async () => { + const mockCatalogApiWithPath = catalogApiMock({ + entities: [ + { + apiVersion: 'scaffolder.backstage.io/v1beta3', + kind: 'Template', + metadata: { + name: 'tmpl-c', + annotations: { + [TECHDOCS_EXTERNAL_ANNOTATION]: 'component:default/other', + [TECHDOCS_EXTERNAL_PATH_ANNOTATION]: '/guides/start', + }, + }, + spec: { type: 'service' }, + }, + ], + }); + + const { findByText } = await renderInTestApp( + + + , + mountedRoutes, + ); + + const link = (await findByText('View TechDocs')).closest('a')!; + expect(link).toHaveAttribute( + 'href', + expect.stringMatching( + /\/docs\/default\/component\/other\/?(index\.html)?#?\/guides\/start|\/docs\/default\/component\/other\/guides\/start/, + ), + ); + }); + }); + it('should render the search bar for templates', async () => { const { getByPlaceholderText } = await renderInTestApp( { const additionalLinksForEntity = useCallback( (template: TemplateEntityV1beta3) => { - const { kind, namespace, name } = parseEntityRef( - stringifyEntityRef(template), - ); - return template.metadata.annotations?.['backstage.io/techdocs-ref'] && - viewTechDocsLink + if ( + !( + template.metadata.annotations?.[TECHDOCS_ANNOTATION] || + template.metadata.annotations?.[TECHDOCS_EXTERNAL_ANNOTATION] + ) || + !viewTechDocsLink + ) { + return []; + } + + const url = buildTechDocsURL(template, viewTechDocsLink); + return url ? [ { icon: app.getSystemIcon('docs') ?? DocsIcon, text: t( 'templateListPage.additionalLinksForEntity.viewTechDocsTitle', ), - url: viewTechDocsLink({ kind, namespace, name }), + url, }, ] : []; diff --git a/plugins/user-settings/report-alpha.api.md b/plugins/user-settings/report-alpha.api.md index 5a28aacc02..1c8e844d05 100644 --- a/plugins/user-settings/report-alpha.api.md +++ b/plugins/user-settings/report-alpha.api.md @@ -123,7 +123,7 @@ export const userSettingsTranslationRef: TranslationRef< readonly 'languageToggle.select': 'Select language {{language}}'; readonly 'languageToggle.title': 'Language'; readonly 'languageToggle.description': 'Change the language'; - readonly 'themeToggle.select': 'Select theme {{theme}}'; + readonly 'themeToggle.select': 'Select {{theme}}'; readonly 'themeToggle.title': 'Theme'; readonly 'themeToggle.description': 'Change the theme mode'; readonly 'themeToggle.names.auto': 'Auto'; diff --git a/plugins/user-settings/src/translation.ts b/plugins/user-settings/src/translation.ts index 0161508d14..7445a415f2 100644 --- a/plugins/user-settings/src/translation.ts +++ b/plugins/user-settings/src/translation.ts @@ -28,7 +28,7 @@ export const userSettingsTranslationRef = createTranslationRef({ themeToggle: { title: 'Theme', description: 'Change the theme mode', - select: 'Select theme {{theme}}', + select: 'Select {{theme}}', selectAuto: 'Select Auto Theme', names: { light: 'Light', diff --git a/yarn.lock b/yarn.lock index 853749b99e..3f719b4756 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6574,6 +6574,9 @@ __metadata: "@backstage/plugin-permission-react": "workspace:^" "@backstage/plugin-scaffolder-common": "workspace:^" "@backstage/plugin-scaffolder-react": "workspace:^" + "@backstage/plugin-techdocs": "workspace:^" + "@backstage/plugin-techdocs-common": "workspace:^" + "@backstage/plugin-techdocs-react": "workspace:^" "@backstage/test-utils": "workspace:^" "@backstage/types": "workspace:^" "@codemirror/language": "npm:^6.0.0" @@ -26265,7 +26268,7 @@ __metadata: languageName: node linkType: hard -"combined-stream@npm:^1.0.6, combined-stream@npm:^1.0.8": +"combined-stream@npm:^1.0.8": version: 1.0.8 resolution: "combined-stream@npm:1.0.8" dependencies: @@ -31054,13 +31057,16 @@ __metadata: linkType: hard "form-data@npm:^2.3.2, form-data@npm:^2.5.0": - version: 2.5.1 - resolution: "form-data@npm:2.5.1" + version: 2.5.5 + resolution: "form-data@npm:2.5.5" dependencies: asynckit: "npm:^0.4.0" - combined-stream: "npm:^1.0.6" - mime-types: "npm:^2.1.12" - checksum: 10/2e2e5e927979ba3623f9b4c4bcc939275fae3f2dea9dafc8db3ca656a3d75476605de2c80f0e6f1487987398e056f0b4c738972d6e1edd83392d5686d0952eed + combined-stream: "npm:^1.0.8" + es-set-tostringtag: "npm:^2.1.0" + hasown: "npm:^2.0.2" + mime-types: "npm:^2.1.35" + safe-buffer: "npm:^5.2.1" + checksum: 10/4b6a8d07bb67089da41048e734215f68317a8e29dd5385a972bf5c458a023313c69d3b5d6b8baafbb7f808fa9881e0e2e030ffe61e096b3ddc894c516401271d languageName: node linkType: hard