From 3b8c0557c6aef126e4604fb73e480f2f29f0c4e8 Mon Sep 17 00:00:00 2001 From: Aramis Sennyey <159921952+aramissennyeydd@users.noreply.github.com> Date: Wed, 22 Apr 2026 10:12:51 -0400 Subject: [PATCH] golden-path: backend plugin persistence guide (#33540) * docs: backend plugin persistence guide Signed-off-by: aramissennyeydd * fix prettier Signed-off-by: aramissennyeydd * add dto section Signed-off-by: aramissennyeydd * Apply suggestion from @aramissennyeydd Signed-off-by: Aramis Sennyey <159921952+aramissennyeydd@users.noreply.github.com> * address feedback Signed-off-by: aramissennyeydd * test against real scaffolding Signed-off-by: aramissennyeydd * Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Aramis Sennyey <159921952+aramissennyeydd@users.noreply.github.com> * fix knex migrate:make command to specify migrations directory Without --migrations-directory, knex cannot resolve the config and errors with "Failed to resolve config file". Co-Authored-By: Claude Opus 4.6 (1M context) Signed-off-by: aramissennyeydd * address copilot review feedback - Fix file paths in code snippets to match scaffolded layout (src/ prefix) - Add missing semicolons in toDatabaseRow/fromDatabaseRow return objects - Change knex from devDependency to regular dependency for type imports - Add missing customize-your-instance to adoption sidebar Co-Authored-By: Claude Opus 4.6 (1M context) Signed-off-by: aramissennyeydd --------- Signed-off-by: aramissennyeydd Signed-off-by: Aramis Sennyey <159921952+aramissennyeydd@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 (1M context) --- .../plugins/backend/003-persistence.md | 317 +++++++++++++++++- ...-tracked.md => 004-reading-from-source.md} | 2 +- microsite/sidebars.ts | 27 +- 3 files changed, 342 insertions(+), 4 deletions(-) rename docs/golden-path/plugins/backend/{004-source-tracked.md => 004-reading-from-source.md} (93%) diff --git a/docs/golden-path/plugins/backend/003-persistence.md b/docs/golden-path/plugins/backend/003-persistence.md index 60d275d7ec..a164d89995 100644 --- a/docs/golden-path/plugins/backend/003-persistence.md +++ b/docs/golden-path/plugins/backend/003-persistence.md @@ -13,8 +13,323 @@ You may have noticed that your list of TODOs disappears after you restart your B SQLite is the default database for local development. It runs in memory (and can also run from a file on disk). It supports quick iteration cycles and can be easily deleted if anything goes wrong. +### What does our data look like at rest? + +Writing to a database requires a table, which requires us to chat quickly about what we want to store. Our TODO object with `title`, `id`, `createdBy` and `createdAt` keys is a good fit to map 1:1 with our database schema. + ## Adding the `databaseService` to your plugin - +### The plumbing + +To start, let's just plumb through the general `databaseService` usage we expect. + +First, add a new service dependency on `databaseService`, + +```diff file="src/services/TodoListService.ts" + +export const todoListServiceRef = createServiceRef>({ + id: 'todo.list', + defaultFactory: async service => + createServiceFactory({ + service, + deps: { + logger: coreServices.logger, + catalog: catalogServiceRef, ++ database: coreServices.database, + }, + async factory(deps) { + return TodoListService.create(deps); + }, + }), +}); +``` + +We then need to add it to our service, + +```diff file="src/services/TodoListService.ts" ++import type { Knex } from 'knex'; +import { + coreServices, + createServiceFactory, + createServiceRef, + LoggerService, ++ DatabaseService, +} from '@backstage/backend-plugin-api'; + +export class TodoListService { ++ readonly #database: Knex; + +- readonly #storedTodos = new Array(); + +- static create(options: { ++ static async create(options: { + logger: LoggerService; + catalog: typeof catalogServiceRef.T; ++ database: DatabaseService; + }) { + const knex = await options.database.getClient(); +- return new TodoListService(options.logger, options.catalog); ++ return new TodoListService(options.logger, options.catalog, knex); + } + + private constructor( + logger: LoggerService, + catalog: typeof catalogServiceRef.T, ++ database: Knex, + ) { + this.#logger = logger; + this.#catalog = catalog; ++ this.#database = database; + } +``` + +And with that, we have an isolated `knex` client to communicate with our database! + +### Creating your table + +Unfortunately, without tables in our database, our `knex` client is not doing much. We need to create a _migration_. Knex stores migrations as JavaScript/TypeScript files that get executed as part of a call to `knex.migrate.latest()`. By default, these are stored in a `migrations/` directory. + +Let's get started. First, we need to install `knex` as a dependency so both its CLI and imported `Knex` types are available, + +```bash +yarn workspace @internal/plugin-todo-backend add knex +``` + +Now, running this command will scaffold a file in that `migrations/` directory for us. + +```bash +yarn workspace @internal/plugin-todo-backend knex migrate:make init --migrations-directory ./migrations +``` + +This should spit out a message like + +```bash +Created Migration: ~/Projects/backstage/backstage/plugins/todo-backend/migrations/20260323130057_init.js +``` + +Let's open that file, + +```js +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.up = async function up(knex) { + // await knex.schema... +}; + +/** + * @param { import("knex").Knex } knex + * @returns { Promise } + */ +exports.down = async function down(knex) { + // await knex.schema... +}; +``` + +You can see two functions, `up` and `down`. `up` is called to apply a migration and `down` is used to undo a previous migration. These should be reversible - if you call `up` and then `down` the database should generally be in the same state if those commands hadn't been run. + +Let's create our table, + +```diff +exports.up = async function up(knex) { ++ await knex.schema.createTable('todo', table => { ++ table.uuid('id').primary(); ++ table.string('created_by', 255).notNullable(); ++ table.string('title').notNullable(); ++ table.datetime('created_at').defaultTo(knex.fn.now()).notNullable(); + ++ table.index(['created_by'], 'todo_user_idx'); + }); +}; +``` + +You'll notice that we use `snake_case` instead of `camelCase` - that's how SQL is conventionally written. + +Let's make sure that we don't forget to add a `down` migration as well! + +```diff +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { ++ await knex.schema.dropTable('todo'); +}; +``` + +Now, we need to actually tell our `knex` client to automatically apply these migrations. We'll add the `database` service to our plugin's `init` function, + +```diff file="src/plugin.ts" +import { + coreServices, + createBackendPlugin, ++ resolvePackagePath, +} from '@backstage/backend-plugin-api'; + +// ... + + deps: { + httpAuth: coreServices.httpAuth, + httpRouter: coreServices.httpRouter, ++ logger: coreServices.logger, ++ database: coreServices.database, + todoList: todoListServiceRef, + }, +- async init({ httpAuth, httpRouter, todoList }) { ++ async init({ httpAuth, logger, httpRouter, database, todoList }) { ++ const knex = await database.getClient(); ++ ++ if (!database.migrations?.skip) { ++ logger.info('Running database migrations...'); ++ ++ const migrationsDir = resolvePackagePath( ++ '@internal/plugin-todo-backend', ++ 'migrations', ++ ); ++ ++ await knex.migrate.latest({ ++ directory: migrationsDir, ++ }); ++ } + + httpRouter.use( + await createRouter({ + httpAuth, + todoList, + }), + ); +``` + +Walking through what we've written - + +1. `database.migrations?.skip` - convention for migrations to allow them to be skipped through config. +1. `const migrationsDir = resolvePackagePath` - ensure the correct migrations directory is passed regardless of environment. +1. `await knex.migrate.latest(` - actually run the migration, calls our `up` method we wrote above. + +We also need to do 1 more thing, + +```diff file="package.json" +"files": [ +- "dist" ++ "dist", ++ "migrations" + ], +``` + +This will make sure the migrations in our plugin work for all users. + +For those who want more details, the full [Knex migration docs](https://knexjs.org/guide/migrations.html#migration-cli) are very informative! + +### Defining our types + +Now that we have our table, we need to add types for it to protect against runtime incompatibilities. For now, these are hand written. + +```diff title="src/services/TodoListService.ts" ++export interface TodoDatabaseRow { ++ title: string; ++ id: string; ++ created_by: string; ++ created_at: string; ++} + +export interface TodoItem { + title: string; + id: string; + createdBy: string; + createdAt: string; +} +``` + +Notice the change to snake case as it has to match the database schema we have above. Now we need to transform `TodoItem` to `TodoDatabaseRow` for writes and `TodoDatabaseRow` to `TodoItem` for reads. + +```diff title="src/services/TodoListService.ts" + private constructor( + logger: LoggerService, + catalog: typeof catalogServiceRef.T, ++ database: Knex, + ) { + this.#logger = logger; + this.#catalog = catalog; ++ this.#database = database; + } + ++ private toDatabaseRow(todo: TodoItem): TodoDatabaseRow { ++ return { ++ id: todo.id, ++ title: todo.title, ++ created_by: todo.createdBy, ++ created_at: todo.createdAt, ++ }; ++ } + ++ private fromDatabaseRow(row: TodoDatabaseRow): TodoItem { ++ return { ++ id: row.id, ++ title: row.title, ++ createdBy: row.created_by, ++ createdAt: row.created_at, ++ }; ++ } +``` + +And that's it! You're now set up to actually read from and write to your database. + +### Writing to your table + +Creating your table was a solid chunk of work - thankfully, writing to it is going to be much easier! + +```diff title="src/services/TodoListService.ts" + + async createTodo( + // ... + const id = crypto.randomUUID(); + const createdBy = options.credentials.principal.userEntityRef; + const newTodo = { + title, + id, + createdBy, + createdAt: new Date().toISOString(), + }; + +- this.#storedTodos.push(newTodo); ++ await this.#database ++ .insert(this.toDatabaseRow(newTodo)) ++ .into('todo'); + + return newTodo; + } +``` + +We've basically just updated our service call to use `this.#database` instead of `this.#storedTodos`. + +### Reading from your table + +Now that we have things in our database, how do we actually get them back out again? + +```diff title="src/services/TodoListService.ts" + + async listTodos(): Promise<{ items: TodoItem[] }> { +- return { items: Array.from(this.#storedTodos) }; ++ const rows = await this.#database('todo').select(); ++ return { items: rows.map(row => this.fromDatabaseRow(row)) }; + } + + async getTodo(request: { id: string }): Promise { +- const todo = this.#storedTodos.find(item => item.id === request.id); ++ const item = await this.#database('todo').where({ id: request.id }).first(); +- if (!todo) { ++ if (!item) { + throw new NotFoundError(`No todo found with id '${request.id}'`); + } +- return todo; ++ return this.fromDatabaseRow(item); + } +``` + +And we're done! ## Testing your changes + +To validate this flow, let's use the same commands that we ran in [the last section of this guide](./002-poking-around.md#testing-locally). + +If everything is working correctly, you will see the same response that you did last time. diff --git a/docs/golden-path/plugins/backend/004-source-tracked.md b/docs/golden-path/plugins/backend/004-reading-from-source.md similarity index 93% rename from docs/golden-path/plugins/backend/004-source-tracked.md rename to docs/golden-path/plugins/backend/004-reading-from-source.md index 54013ccead..87ca4b210e 100644 --- a/docs/golden-path/plugins/backend/004-source-tracked.md +++ b/docs/golden-path/plugins/backend/004-reading-from-source.md @@ -1,5 +1,5 @@ --- -id: source-tracked +id: reading-from-source sidebar_label: 004 - Integrating with SCMs title: 004 - Git-tracked TODOs description: How to ingest TODOs from source code repositories into your plugin diff --git a/microsite/sidebars.ts b/microsite/sidebars.ts index df39e911ff..565c09feb4 100644 --- a/microsite/sidebars.ts +++ b/microsite/sidebars.ts @@ -108,9 +108,32 @@ export default { 'golden-path/plugins/why-build-plugins', 'golden-path/plugins/sustainable-plugin-development', sidebarElementWithIndex({ label: 'Backend Plugins' }, [ - 'golden-path/plugins/backend/001-first-steps', - 'golden-path/plugins/backend/002-poking-around', + 'golden-path/plugins/backend/first-steps', + 'golden-path/plugins/backend/poking-around', + 'golden-path/plugins/backend/persistence', + 'golden-path/plugins/backend/reading-from-source', + 'golden-path/plugins/backend/testing', ]), + sidebarElementWithIndex({ label: 'Frontend Plugins' }, [ + 'golden-path/plugins/frontend/first-steps', + 'golden-path/plugins/frontend/poking-around', + 'golden-path/plugins/frontend/dynamic-config', + 'golden-path/plugins/frontend/http-client', + 'golden-path/plugins/frontend/testing', + ]), + ]), + sidebarElementWithIndex({ label: '003 - Deployment' }, [ + 'golden-path/deployment/index', + ]), + sidebarElementWithIndex({ label: '004 - Adoption' }, [ + 'golden-path/adoption/getting-started', + 'golden-path/adoption/leadership-buy-in', + 'golden-path/adoption/setting-up-a-poc', + 'golden-path/adoption/first-stakeholder-feedback', + 'golden-path/adoption/customize-your-instance', + 'golden-path/adoption/preparing-for-ga', + 'golden-path/adoption/plugin-ownership', + 'golden-path/adoption/full-catalog', ]), ]), ]