From 9461f73643f9b5cee593420505bfeacd8de3009d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Sat, 5 Mar 2022 16:58:28 +0100 Subject: [PATCH 01/40] try a convenience thing for scheduling providers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/new-books-protect.md | 30 ++++ .changeset/silent-cats-kneel.md | 2 +- .changeset/ten-queens-dance.md | 7 + docs/integrations/ldap/org.md | 68 +++------- packages/backend-tasks/api-report.md | 34 +++-- .../src/tasks/PluginTaskSchedulerImpl.test.ts | 27 ++++ .../src/tasks/PluginTaskSchedulerImpl.ts | 19 ++- packages/backend-tasks/src/tasks/index.ts | 4 +- packages/backend-tasks/src/tasks/types.ts | 81 ++++++++--- .../catalog-backend-module-ldap/api-report.md | 21 +-- .../catalog-backend-module-ldap/package.json | 2 + .../src/processors/LdapOrgEntityProvider.ts | 128 +++++++++++++----- .../src/processors/index.ts | 1 + 13 files changed, 301 insertions(+), 123 deletions(-) create mode 100644 .changeset/new-books-protect.md create mode 100644 .changeset/ten-queens-dance.md diff --git a/.changeset/new-books-protect.md b/.changeset/new-books-protect.md new file mode 100644 index 0000000000..2a561b5e5c --- /dev/null +++ b/.changeset/new-books-protect.md @@ -0,0 +1,30 @@ +--- +'@backstage/plugin-catalog-backend-module-ldap': minor +--- + +**BREAKING**: Added a `schedule` field to `LdapOrgEntityProvider.fromConfig`, which is required. If you want to retain the old behavior of scheduling the provider manually, you can set it to the string value `'manual'`. But you may want to leverage the ability to instead pass in the recurring task schedule information directly. This will allow you to simplify your backend setup code to not need an intermediate variable and separate scheduling code at the bottom. + +All things said, a typical setup might now look as follows: + +```diff + // packages/backend/src/plugins/catalog.ts ++import { Duration } from 'luxon'; ++import { LdapOrgEntityProvider } from '@backstage/plugin-catalog-backend-module-ldap'; + export default async function createPlugin( + env: PluginEnvironment, + ): Promise { + const builder = await CatalogBuilder.create(env); ++ // The target parameter below needs to match the ldap.providers.target ++ // value specified in your app-config. ++ builder.addEntityProvider( ++ LdapOrgEntityProvider.fromConfig(env.config, { ++ id: 'our-ldap-master', ++ target: 'ldaps://ds.example.net', ++ logger: env.logger, ++ schedule: env.scheduler.createTaskSchedule({ ++ frequency: Duration.fromObject({ minutes: 60 }), ++ timeout: Duration.fromObject({ minutes: 15 }), ++ }), ++ }), ++ ); +``` diff --git a/.changeset/silent-cats-kneel.md b/.changeset/silent-cats-kneel.md index f34fc98afc..9bd9cce0e0 100644 --- a/.changeset/silent-cats-kneel.md +++ b/.changeset/silent-cats-kneel.md @@ -2,4 +2,4 @@ '@backstage/plugin-catalog-backend-module-gitlab': minor --- -Added package, moving out gitlab specific functionality from the catalog-backend +Added package, moving out GitLab specific functionality from the catalog-backend diff --git a/.changeset/ten-queens-dance.md b/.changeset/ten-queens-dance.md new file mode 100644 index 0000000000..947675aa46 --- /dev/null +++ b/.changeset/ten-queens-dance.md @@ -0,0 +1,7 @@ +--- +'@backstage/backend-tasks': minor +--- + +**BREAKING**: The `TaskDefinition` type has been removed, and replaced by the equal pair `TaskScheduleDefinition` and `TaskInvocationDefinition`. The interface for `PluginTaskScheduler.scheduleTask` stays effectively unchanged, so this only affects you if you use the actual types directly. + +Added the method `PluginTaskScheduler.createTaskSchedule`, which returns a `TaskSchedule` wrapper that is convenient to pass down into classes that want to control their task invocations while the caller wants to retain control of the actual schedule chosen. diff --git a/docs/integrations/ldap/org.md b/docs/integrations/ldap/org.md index 645524493b..b0fa2ca76d 100644 --- a/docs/integrations/ldap/org.md +++ b/docs/integrations/ldap/org.md @@ -32,55 +32,29 @@ yarn add @backstage/plugin-catalog-backend-module-ldap Update the catalog plugin initialization in your backend to add the provider and schedule it: -```ts -// packages/backend/src/plugins/catalog.ts -import { CatalogBuilder } from '@backstage/plugin-catalog-backend'; -import { Router } from 'express'; -import { PluginEnvironment } from '../types'; -import { Duration } from 'luxon'; -import { LdapOrgEntityProvider } from '@backstage/plugin-catalog-backend-module-ldap'; +```diff + // packages/backend/src/plugins/catalog.ts ++import { Duration } from 'luxon'; ++import { LdapOrgEntityProvider } from '@backstage/plugin-catalog-backend-module-ldap'; -export default async function createPlugin( - env: PluginEnvironment, -): Promise { - // The target parameter below needs to match the ldap.providers.target - // value specified in your app-config - const ldapEntityProvider = LdapOrgEntityProvider.fromConfig(env.config, { - id: 'our-ldap-master', - target: 'ldaps://ds.example.net', - logger: env.logger, - }); + export default async function createPlugin( + env: PluginEnvironment, + ): Promise { + const builder = await CatalogBuilder.create(env); - const builder = await CatalogBuilder.create(env); - builder.addEntityProvider(ldapEntityProvider); - - // You can change the refresh interval for the other catalog entries - // independently, or just leave the line below out to use the default - // refresh interval. Note that this interval does NOT at all affect - // the LDAP refresh when using the provider method, which is good! - builder.setRefreshIntervalSeconds(100); - - const { processingEngine, router } = await builder.build(); - await processingEngine.start(); - - // Only perform this scheduling after starting the processing engine - await env.scheduler.scheduleTask({ - id: 'refresh_ldap', - // frequency sets how often you want to ingest users and groups from - // LDAP, in this case every 60 minutes - frequency: Duration.fromObject({ minutes: 60 }), - timeout: Duration.fromObject({ minutes: 15 }), - fn: async () => { - try { - await ldapEntityProvider.read(); - } catch (error) { - env.logger.error(error); - } - }, - }); - - return router; -} ++ // The target parameter below needs to match the ldap.providers.target ++ // value specified in your app-config. ++ builder.addEntityProvider( ++ LdapOrgEntityProvider.fromConfig(env.config, { ++ id: 'our-ldap-master', ++ target: 'ldaps://ds.example.net', ++ logger: env.logger, ++ schedule: env.scheduler.createTaskSchedule({ ++ frequency: Duration.fromObject({ minutes: 60 }), ++ timeout: Duration.fromObject({ minutes: 15 }), ++ }), ++ }), ++ ); ``` After this, you also have to add some configuration in your app-config that diff --git a/packages/backend-tasks/api-report.md b/packages/backend-tasks/api-report.md index 82b2ca9b0c..db4dee0c65 100644 --- a/packages/backend-tasks/api-report.md +++ b/packages/backend-tasks/api-report.md @@ -11,17 +11,10 @@ import { Logger } from 'winston'; // @public export interface PluginTaskScheduler { - scheduleTask(task: TaskDefinition): Promise; -} - -// @public -export interface TaskDefinition { - fn: TaskFunction; - frequency: Duration; - id: string; - initialDelay?: Duration; - signal?: AbortSignal_2; - timeout: Duration; + createTaskSchedule(schedule: TaskScheduleDefinition): TaskSchedule; + scheduleTask( + task: TaskScheduleDefinition & TaskInvocationDefinition, + ): Promise; } // @public @@ -29,6 +22,25 @@ export type TaskFunction = | ((abortSignal: AbortSignal_2) => void | Promise) | (() => void | Promise); +// @public +export interface TaskInvocationDefinition { + fn: TaskFunction; + id: string; + signal?: AbortSignal_2; +} + +// @public +export interface TaskSchedule { + run(task: TaskInvocationDefinition): Promise; +} + +// @public +export interface TaskScheduleDefinition { + frequency: Duration; + initialDelay?: Duration; + timeout: Duration; +} + // @public export class TaskScheduler { constructor(databaseManager: DatabaseManager, logger: Logger); diff --git a/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts b/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts index e387b85413..0a811a5b88 100644 --- a/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts +++ b/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts @@ -59,4 +59,31 @@ describe('PluginTaskManagerImpl', () => { 60_000, ); }); + + // This is just to test the wrapper code; most of the actual tests are in + // TaskWorker.test.ts + describe('createTaskSchedule', () => { + it.each(databases.eachSupportedId())( + 'can run the happy path, %p', + async databaseId => { + const { manager } = await init(databaseId); + + const fn = jest.fn(); + await manager + .createTaskSchedule({ + timeout: Duration.fromMillis(5000), + frequency: Duration.fromMillis(5000), + }) + .run({ + id: 'task1', + fn, + }); + + await waitForExpect(() => { + expect(fn).toBeCalled(); + }); + }, + 60_000, + ); + }); }); diff --git a/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.ts b/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.ts index 93975bc327..b410e4d3b5 100644 --- a/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.ts +++ b/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.ts @@ -17,7 +17,12 @@ import { Knex } from 'knex'; import { Logger } from 'winston'; import { TaskWorker } from './TaskWorker'; -import { PluginTaskScheduler, TaskDefinition } from './types'; +import { + PluginTaskScheduler, + TaskInvocationDefinition, + TaskSchedule, + TaskScheduleDefinition, +} from './types'; import { validateId } from './util'; /** @@ -29,7 +34,9 @@ export class PluginTaskSchedulerImpl implements PluginTaskScheduler { private readonly logger: Logger, ) {} - async scheduleTask(task: TaskDefinition): Promise { + async scheduleTask( + task: TaskScheduleDefinition & TaskInvocationDefinition, + ): Promise { validateId(task.id); const knex = await this.databaseFactory(); @@ -47,4 +54,12 @@ export class PluginTaskSchedulerImpl implements PluginTaskScheduler { }, ); } + + createTaskSchedule(schedule: TaskScheduleDefinition): TaskSchedule { + return { + run: async task => { + await this.scheduleTask({ ...task, ...schedule }); + }, + }; + } } diff --git a/packages/backend-tasks/src/tasks/index.ts b/packages/backend-tasks/src/tasks/index.ts index 9e0a06f71c..d925f089e8 100644 --- a/packages/backend-tasks/src/tasks/index.ts +++ b/packages/backend-tasks/src/tasks/index.ts @@ -17,6 +17,8 @@ export { TaskScheduler } from './TaskScheduler'; export type { PluginTaskScheduler, - TaskDefinition, TaskFunction, + TaskInvocationDefinition, + TaskSchedule, + TaskScheduleDefinition, } from './types'; diff --git a/packages/backend-tasks/src/tasks/types.ts b/packages/backend-tasks/src/tasks/types.ts index 4693af7ef3..3e46acf55e 100644 --- a/packages/backend-tasks/src/tasks/types.ts +++ b/packages/backend-tasks/src/tasks/types.ts @@ -31,27 +31,11 @@ export type TaskFunction = | (() => void | Promise); /** - * Options that apply to the invocation of a given task. + * Options that control the scheduling of a task. * * @public */ -export interface TaskDefinition { - /** - * A unique ID (within the scope of the plugin) for the task. - */ - id: string; - - /** - * The actual task function to be invoked regularly. - */ - fn: TaskFunction; - - /** - * An abort signal that, when triggered, will stop the recurring execution of - * the task. - */ - signal?: AbortSignal; - +export interface TaskScheduleDefinition { /** * The maximum amount of time that a single task invocation can take, before * it's considered timed out and gets "released" such that a new invocation @@ -91,6 +75,43 @@ export interface TaskDefinition { initialDelay?: Duration; } +/** + * Options that apply to the invocation of a given task. + * + * @public + */ +export interface TaskInvocationDefinition { + /** + * A unique ID (within the scope of the plugin) for the task. + */ + id: string; + + /** + * The actual task function to be invoked regularly. + */ + fn: TaskFunction; + + /** + * An abort signal that, when triggered, will stop the recurring execution of + * the task. + */ + signal?: AbortSignal; +} + +/** + * A previously prepared task schedule, ready to be invoked. + * + * @public + */ +export interface TaskSchedule { + /** + * Takes the schedule and executes an actual task using it. + * + * @param task - The actual runtime properties of the task + */ + run(task: TaskInvocationDefinition): Promise; +} + /** * Deals with the scheduling of distributed tasks, for a given plugin. * @@ -99,15 +120,33 @@ export interface TaskDefinition { export interface PluginTaskScheduler { /** * Schedules a task function for coordinated exclusive invocation across - * workers. + * workers. This convenience method performs both the scheduling and + * invocation in one go. + * + * @remarks * * If the task was already scheduled since before by us or by another party, * its options are just overwritten with the given options, and things * continue from there. * - * @param definition - The task definition + * @param task - The task definition */ - scheduleTask(task: TaskDefinition): Promise; + scheduleTask( + task: TaskScheduleDefinition & TaskInvocationDefinition, + ): Promise; + + /** + * Creates a task schedule, ready to be invoked at a later time. + * + * @remarks + * + * This method is useful for pre-creating a schedule in outer code to be + * passed into an inner implementation, such that the outer code controls + * scheduling while inner code controls implementation. + * + * @param schedule - The task schedule + */ + createTaskSchedule(schedule: TaskScheduleDefinition): TaskSchedule; } function isValidOptionalDurationString(d: string | undefined): boolean { diff --git a/plugins/catalog-backend-module-ldap/api-report.md b/plugins/catalog-backend-module-ldap/api-report.md index 79646aab01..25f479f73c 100644 --- a/plugins/catalog-backend-module-ldap/api-report.md +++ b/plugins/catalog-backend-module-ldap/api-report.md @@ -15,6 +15,7 @@ import { LocationSpec } from '@backstage/plugin-catalog-backend'; import { Logger } from 'winston'; import { SearchEntry } from 'ldapjs'; import { SearchOptions } from 'ldapjs'; +import { TaskSchedule } from '@backstage/backend-tasks'; import { UserEntity } from '@backstage/catalog-model'; // @public @@ -106,17 +107,21 @@ export class LdapOrgEntityProvider implements EntityProvider { // (undocumented) static fromConfig( configRoot: Config, - options: { - id: string; - target: string; - userTransformer?: UserTransformer; - groupTransformer?: GroupTransformer; - logger: Logger; - }, + options: LdapOrgEntityProviderOptions, ): LdapOrgEntityProvider; // (undocumented) getProviderName(): string; - read(): Promise; + read(options?: { logger?: Logger }): Promise; +} + +// @public +export interface LdapOrgEntityProviderOptions { + groupTransformer?: GroupTransformer; + id: string; + logger: Logger; + schedule: 'manual' | TaskSchedule; + target: string; + userTransformer?: UserTransformer; } // @public diff --git a/plugins/catalog-backend-module-ldap/package.json b/plugins/catalog-backend-module-ldap/package.json index 8f05dceb4e..d1c9e764fd 100644 --- a/plugins/catalog-backend-module-ldap/package.json +++ b/plugins/catalog-backend-module-ldap/package.json @@ -33,6 +33,7 @@ "start": "backstage-cli package start" }, "dependencies": { + "@backstage/backend-tasks": "^0.1.10", "@backstage/catalog-model": "^0.12.0", "@backstage/config": "^0.1.15", "@backstage/errors": "^0.2.2", @@ -41,6 +42,7 @@ "@types/ldapjs": "^2.2.0", "ldapjs": "^2.2.0", "lodash": "^4.17.21", + "uuid": "^8.0.0", "winston": "^3.2.1" }, "devDependencies": { diff --git a/plugins/catalog-backend-module-ldap/src/processors/LdapOrgEntityProvider.ts b/plugins/catalog-backend-module-ldap/src/processors/LdapOrgEntityProvider.ts index 4c5bd2e3bd..9c483c7b94 100644 --- a/plugins/catalog-backend-module-ldap/src/processors/LdapOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-ldap/src/processors/LdapOrgEntityProvider.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { TaskSchedule } from '@backstage/backend-tasks'; import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, @@ -25,6 +26,7 @@ import { EntityProviderConnection, } from '@backstage/plugin-catalog-backend'; import { merge } from 'lodash'; +import * as uuid from 'uuid'; import { Logger } from 'winston'; import { GroupTransformer, @@ -36,6 +38,54 @@ import { UserTransformer, } from '../ldap'; +/** + * Options for {@link LdapOrgEntityProvider}. + * + * @public + */ +export interface LdapOrgEntityProviderOptions { + /** + * A unique, stable identifier for this provider. + * + * @example "production" + */ + id: string; + + /** + * The target that this provider should consume. + * + * Should exactly match the "target" field of one of the "ldap.providers" + * configuration entries. + * + * @example "ldaps://ds-read.example.net" + */ + target: string; + + /** + * The logger to use. + */ + logger: Logger; + + /** + * The refresh schedule to use. + * + * If you pass in 'manual', you are responsible for calling the `read` + * method manually at some interval. If not, it will be automatically + * called regularly with the given schedule using the scheduler. + */ + schedule: 'manual' | TaskSchedule; + + /** + * The function that transforms a user entry in LDAP to an entity. + */ + userTransformer?: UserTransformer; + + /** + * The function that transforms a group entry in LDAP to an entity. + */ + groupTransformer?: GroupTransformer; +} + /** * Reads user and group entries out of an LDAP service, and provides them as * User and Group entities for the catalog. @@ -49,35 +99,11 @@ import { */ export class LdapOrgEntityProvider implements EntityProvider { private connection?: EntityProviderConnection; + private scheduleFn?: () => Promise; static fromConfig( configRoot: Config, - options: { - /** - * A unique, stable identifier for this provider. - * - * @example "production" - */ - id: string; - /** - * The target that this provider should consume. - * - * Should exactly match the "target" field of one of the "ldap.providers" - * configuration entries. - * - * @example "ldaps://ds-read.example.net" - */ - target: string; - /** - * The function that transforms a user entry in LDAP to an entity. - */ - userTransformer?: UserTransformer; - /** - * The function that transforms a group entry in LDAP to an entity. - */ - groupTransformer?: GroupTransformer; - logger: Logger; - }, + options: LdapOrgEntityProviderOptions, ): LdapOrgEntityProvider { // TODO(freben): Deprecate the old catalog.processors.ldapOrg config const config = @@ -101,13 +127,17 @@ export class LdapOrgEntityProvider implements EntityProvider { target: options.target, }); - return new LdapOrgEntityProvider({ + const result = new LdapOrgEntityProvider({ id: options.id, provider, userTransformer: options.userTransformer, groupTransformer: options.groupTransformer, logger, }); + + result.schedule(options.schedule); + + return result; } constructor( @@ -128,18 +158,20 @@ export class LdapOrgEntityProvider implements EntityProvider { /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */ async connect(connection: EntityProviderConnection) { this.connection = connection; + await this.scheduleFn?.(); } /** - * Runs one complete ingestion loop. Call this method regularly at some - * appropriate cadence. + * Runs one single complete ingestion. This is only necessary if you use + * manual scheduling. */ - async read() { + async read(options?: { logger?: Logger }) { if (!this.connection) { throw new Error('Not initialized'); } - const { markReadComplete } = trackProgress(this.options.logger); + const logger = options?.logger ?? this.options.logger; + const { markReadComplete } = trackProgress(logger); // Be lazy and create the client each time; even though it's pretty // inefficient, we usually only do this once per entire refresh loop and @@ -157,7 +189,7 @@ export class LdapOrgEntityProvider implements EntityProvider { { groupTransformer: this.options.groupTransformer, userTransformer: this.options.userTransformer, - logger: this.options.logger, + logger, }, ); @@ -173,6 +205,38 @@ export class LdapOrgEntityProvider implements EntityProvider { markCommitComplete(); } + + private schedule(schedule: LdapOrgEntityProviderOptions['schedule']) { + if (schedule === 'manual') { + return; + } + + this.scheduleFn = async () => { + const id = this.getScheduledTaskId(); + await schedule.run({ + id, + fn: async () => { + const logger = this.options.logger.child({ + class: LdapOrgEntityProvider.prototype.constructor.name, + taskId: id, + taskInstanceId: uuid.v4(), + }); + + try { + await this.read({ logger }); + } catch (error) { + logger.error(error); + } + }, + }); + }; + } + + // Gets a suitable scheduler task ID for this provider instance + private getScheduledTaskId(): string { + const rawId = `refresh_${this.getProviderName()}`; + return rawId.toLocaleLowerCase('en-US').replace(/[^a-z0-9]/g, '_'); + } } // Helps wrap the timing and logging behaviors diff --git a/plugins/catalog-backend-module-ldap/src/processors/index.ts b/plugins/catalog-backend-module-ldap/src/processors/index.ts index 96e1a49cbb..5ed0095c3e 100644 --- a/plugins/catalog-backend-module-ldap/src/processors/index.ts +++ b/plugins/catalog-backend-module-ldap/src/processors/index.ts @@ -15,4 +15,5 @@ */ export { LdapOrgEntityProvider } from './LdapOrgEntityProvider'; +export type { LdapOrgEntityProviderOptions } from './LdapOrgEntityProvider'; export { LdapOrgReaderProcessor } from './LdapOrgReaderProcessor'; From 74663277fb371b01e989e08bc33a352b93ac2930 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Mon, 7 Mar 2022 15:32:13 +0100 Subject: [PATCH 02/40] rename to createScheduledTaskRunner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/new-books-protect.md | 2 +- docs/integrations/ldap/org.md | 2 +- packages/backend-tasks/api-report.md | 4 ++-- .../src/tasks/PluginTaskSchedulerImpl.test.ts | 4 ++-- .../src/tasks/PluginTaskSchedulerImpl.ts | 4 ++-- packages/backend-tasks/src/tasks/index.ts | 2 +- packages/backend-tasks/src/tasks/types.ts | 7 ++++--- plugins/catalog-backend-module-ldap/api-report.md | 4 ++-- .../src/processors/LdapOrgEntityProvider.ts | 15 ++++++++++----- 9 files changed, 25 insertions(+), 19 deletions(-) diff --git a/.changeset/new-books-protect.md b/.changeset/new-books-protect.md index 2a561b5e5c..568710ccd0 100644 --- a/.changeset/new-books-protect.md +++ b/.changeset/new-books-protect.md @@ -21,7 +21,7 @@ All things said, a typical setup might now look as follows: + id: 'our-ldap-master', + target: 'ldaps://ds.example.net', + logger: env.logger, -+ schedule: env.scheduler.createTaskSchedule({ ++ schedule: env.scheduler.createScheduledTaskRunner({ + frequency: Duration.fromObject({ minutes: 60 }), + timeout: Duration.fromObject({ minutes: 15 }), + }), diff --git a/docs/integrations/ldap/org.md b/docs/integrations/ldap/org.md index b0fa2ca76d..f7f4e25b1a 100644 --- a/docs/integrations/ldap/org.md +++ b/docs/integrations/ldap/org.md @@ -49,7 +49,7 @@ schedule it: + id: 'our-ldap-master', + target: 'ldaps://ds.example.net', + logger: env.logger, -+ schedule: env.scheduler.createTaskSchedule({ ++ schedule: env.scheduler.createScheduledTaskRunner({ + frequency: Duration.fromObject({ minutes: 60 }), + timeout: Duration.fromObject({ minutes: 15 }), + }), diff --git a/packages/backend-tasks/api-report.md b/packages/backend-tasks/api-report.md index db4dee0c65..98199a55b0 100644 --- a/packages/backend-tasks/api-report.md +++ b/packages/backend-tasks/api-report.md @@ -11,7 +11,7 @@ import { Logger } from 'winston'; // @public export interface PluginTaskScheduler { - createTaskSchedule(schedule: TaskScheduleDefinition): TaskSchedule; + createScheduledTaskRunner(schedule: TaskScheduleDefinition): TaskRunner; scheduleTask( task: TaskScheduleDefinition & TaskInvocationDefinition, ): Promise; @@ -30,7 +30,7 @@ export interface TaskInvocationDefinition { } // @public -export interface TaskSchedule { +export interface TaskRunner { run(task: TaskInvocationDefinition): Promise; } diff --git a/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts b/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts index 0a811a5b88..993041bed4 100644 --- a/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts +++ b/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.test.ts @@ -62,7 +62,7 @@ describe('PluginTaskManagerImpl', () => { // This is just to test the wrapper code; most of the actual tests are in // TaskWorker.test.ts - describe('createTaskSchedule', () => { + describe('createScheduledTaskRunner', () => { it.each(databases.eachSupportedId())( 'can run the happy path, %p', async databaseId => { @@ -70,7 +70,7 @@ describe('PluginTaskManagerImpl', () => { const fn = jest.fn(); await manager - .createTaskSchedule({ + .createScheduledTaskRunner({ timeout: Duration.fromMillis(5000), frequency: Duration.fromMillis(5000), }) diff --git a/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.ts b/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.ts index b410e4d3b5..8d50298ffd 100644 --- a/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.ts +++ b/packages/backend-tasks/src/tasks/PluginTaskSchedulerImpl.ts @@ -20,7 +20,7 @@ import { TaskWorker } from './TaskWorker'; import { PluginTaskScheduler, TaskInvocationDefinition, - TaskSchedule, + TaskRunner, TaskScheduleDefinition, } from './types'; import { validateId } from './util'; @@ -55,7 +55,7 @@ export class PluginTaskSchedulerImpl implements PluginTaskScheduler { ); } - createTaskSchedule(schedule: TaskScheduleDefinition): TaskSchedule { + createScheduledTaskRunner(schedule: TaskScheduleDefinition): TaskRunner { return { run: async task => { await this.scheduleTask({ ...task, ...schedule }); diff --git a/packages/backend-tasks/src/tasks/index.ts b/packages/backend-tasks/src/tasks/index.ts index d925f089e8..f6695a8d73 100644 --- a/packages/backend-tasks/src/tasks/index.ts +++ b/packages/backend-tasks/src/tasks/index.ts @@ -19,6 +19,6 @@ export type { PluginTaskScheduler, TaskFunction, TaskInvocationDefinition, - TaskSchedule, + TaskRunner, TaskScheduleDefinition, } from './types'; diff --git a/packages/backend-tasks/src/tasks/types.ts b/packages/backend-tasks/src/tasks/types.ts index 3e46acf55e..35b5598700 100644 --- a/packages/backend-tasks/src/tasks/types.ts +++ b/packages/backend-tasks/src/tasks/types.ts @@ -103,7 +103,7 @@ export interface TaskInvocationDefinition { * * @public */ -export interface TaskSchedule { +export interface TaskRunner { /** * Takes the schedule and executes an actual task using it. * @@ -136,7 +136,8 @@ export interface PluginTaskScheduler { ): Promise; /** - * Creates a task schedule, ready to be invoked at a later time. + * Creates a scheduled but dormant recurring task, ready to be launched at a + * later time. * * @remarks * @@ -146,7 +147,7 @@ export interface PluginTaskScheduler { * * @param schedule - The task schedule */ - createTaskSchedule(schedule: TaskScheduleDefinition): TaskSchedule; + createScheduledTaskRunner(schedule: TaskScheduleDefinition): TaskRunner; } function isValidOptionalDurationString(d: string | undefined): boolean { diff --git a/plugins/catalog-backend-module-ldap/api-report.md b/plugins/catalog-backend-module-ldap/api-report.md index 25f479f73c..edeef4cb82 100644 --- a/plugins/catalog-backend-module-ldap/api-report.md +++ b/plugins/catalog-backend-module-ldap/api-report.md @@ -15,7 +15,7 @@ import { LocationSpec } from '@backstage/plugin-catalog-backend'; import { Logger } from 'winston'; import { SearchEntry } from 'ldapjs'; import { SearchOptions } from 'ldapjs'; -import { TaskSchedule } from '@backstage/backend-tasks'; +import { TaskRunner } from '@backstage/backend-tasks'; import { UserEntity } from '@backstage/catalog-model'; // @public @@ -119,7 +119,7 @@ export interface LdapOrgEntityProviderOptions { groupTransformer?: GroupTransformer; id: string; logger: Logger; - schedule: 'manual' | TaskSchedule; + schedule: 'manual' | TaskRunner; target: string; userTransformer?: UserTransformer; } diff --git a/plugins/catalog-backend-module-ldap/src/processors/LdapOrgEntityProvider.ts b/plugins/catalog-backend-module-ldap/src/processors/LdapOrgEntityProvider.ts index 9c483c7b94..f7d07a15af 100644 --- a/plugins/catalog-backend-module-ldap/src/processors/LdapOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-ldap/src/processors/LdapOrgEntityProvider.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { TaskSchedule } from '@backstage/backend-tasks'; +import { TaskRunner } from '@backstage/backend-tasks'; import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, @@ -69,11 +69,16 @@ export interface LdapOrgEntityProviderOptions { /** * The refresh schedule to use. * - * If you pass in 'manual', you are responsible for calling the `read` - * method manually at some interval. If not, it will be automatically - * called regularly with the given schedule using the scheduler. + * @remarks + * + * If you pass in 'manual', you are responsible for calling the `read` method + * manually at some interval. + * + * But more commonly you will pass in the result of + * {@link @backstage/backend-tasks#PluginTaskScheduler.createScheduledTaskRunner} + * to enable automatic scheduling of tasks. */ - schedule: 'manual' | TaskSchedule; + schedule: 'manual' | TaskRunner; /** * The function that transforms a user entry in LDAP to an entity. From bd87e805c00b0f5bd362d0244697552bc9efb0e3 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 5 Mar 2022 18:21:27 +0100 Subject: [PATCH 03/40] techdocs-cli: tweak e2e test setup Signed-off-by: Patrik Oldsberg --- packages/techdocs-cli/e2e-test.config.js | 22 +++++++++++++++++++ .../techdocs-cli.test.ts} | 2 +- packages/techdocs-cli/package.json | 6 ++--- 3 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 packages/techdocs-cli/e2e-test.config.js rename packages/techdocs-cli/{src/e2e.test.ts => e2e-tests/techdocs-cli.test.ts} (98%) diff --git a/packages/techdocs-cli/e2e-test.config.js b/packages/techdocs-cli/e2e-test.config.js new file mode 100644 index 0000000000..2fa92462ab --- /dev/null +++ b/packages/techdocs-cli/e2e-test.config.js @@ -0,0 +1,22 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const path = require('path'); + +module.exports = require('@backstage/cli/config/jest').then(baseConfig => ({ + ...baseConfig, + rootDir: path.resolve(__dirname, 'e2e-tests'), +})); diff --git a/packages/techdocs-cli/src/e2e.test.ts b/packages/techdocs-cli/e2e-tests/techdocs-cli.test.ts similarity index 98% rename from packages/techdocs-cli/src/e2e.test.ts rename to packages/techdocs-cli/e2e-tests/techdocs-cli.test.ts index 7cdd867d0f..d9d736cb6d 100644 --- a/packages/techdocs-cli/src/e2e.test.ts +++ b/packages/techdocs-cli/e2e-tests/techdocs-cli.test.ts @@ -58,7 +58,7 @@ const timeout = 25000; jest.setTimeout(timeout * 2); describe('end-to-end', () => { - const cwd = path.resolve(__dirname, 'example-docs'); + const cwd = path.resolve(__dirname, '../src/example-docs'); afterEach(async () => { // On Windows the pid of a spawned process may be wrong diff --git a/packages/techdocs-cli/package.json b/packages/techdocs-cli/package.json index e6cfab0916..a6c7ada37a 100644 --- a/packages/techdocs-cli/package.json +++ b/packages/techdocs-cli/package.json @@ -27,9 +27,9 @@ "build": "backstage-cli package build", "clean": "backstage-cli package clean", "lint": "backstage-cli package lint", - "test": "backstage-cli package test --testPathIgnorePatterns=src/e2e.test.ts", - "test:e2e": "backstage-cli test src/e2e.test.ts", - "test:e2e:ci": "backstage-cli test --watchAll=false --ci src/e2e.test.ts", + "test": "backstage-cli package test", + "test:e2e": "backstage-cli test --config e2e-test.config.js", + "test:e2e:ci": "backstage-cli test --config e2e-test.config.js --watchAll=false --ci", "test:cypress": "cypress open", "prepack": "./scripts/prepack.sh" }, From 56e79b538863b54ce188d78a1b32830be87d2a3b Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 7 Mar 2022 21:11:36 +0000 Subject: [PATCH 04/40] chore: removing setSecret function in favour of setSecrets Signed-off-by: blam --- plugins/scaffolder/src/components/secrets/SecretsContext.tsx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/plugins/scaffolder/src/components/secrets/SecretsContext.tsx b/plugins/scaffolder/src/components/secrets/SecretsContext.tsx index f0b62f20a1..b425a72233 100644 --- a/plugins/scaffolder/src/components/secrets/SecretsContext.tsx +++ b/plugins/scaffolder/src/components/secrets/SecretsContext.tsx @@ -53,8 +53,6 @@ export const SecretsContextProvider = ({ children }: PropsWithChildren<{}>) => { * @public */ export interface ScaffolderUseTemplateSecrets { - /** @deprecated use setSecrets instead */ - setSecret: (input: Record) => void; setSecrets: (input: Record) => void; } @@ -79,5 +77,5 @@ export const useTemplateSecrets = (): ScaffolderUseTemplateSecrets => { [updateSecrets], ); - return { setSecret: setSecrets, setSecrets }; + return { setSecrets }; }; From 37027286a6488b820fc79df966e0051f00040418 Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 7 Mar 2022 21:14:00 +0000 Subject: [PATCH 05/40] chore: removing some more deprecations Signed-off-by: blam --- plugins/scaffolder/src/components/Router.tsx | 21 +++----------------- plugins/scaffolder/src/components/index.ts | 2 -- 2 files changed, 3 insertions(+), 20 deletions(-) diff --git a/plugins/scaffolder/src/components/Router.tsx b/plugins/scaffolder/src/components/Router.tsx index b675fdb357..cedeaaca0f 100644 --- a/plugins/scaffolder/src/components/Router.tsx +++ b/plugins/scaffolder/src/components/Router.tsx @@ -54,25 +54,12 @@ export type RouterProps = { }; export const Router = (props: RouterProps) => { - const { - TemplateCardComponent: legacyTemplateCardComponent, - TaskPageComponent: legacyTaskPageComponent, - groups, - components = {}, - } = props; - - if (legacyTemplateCardComponent || legacyTaskPageComponent) { - // eslint-disable-next-line no-console - console.warn( - "DEPRECATION: 'TemplateCardComponent' and 'TaskPageComponent' are deprecated when calling the 'ScaffolderPage'. Use 'components' prop to pass these component overrides instead.", - ); - } + const { groups, components = {} } = props; const { TemplateCardComponent, TaskPageComponent } = components; const outlet = useOutlet(); - const TaskPageElement = - TaskPageComponent ?? legacyTaskPageComponent ?? TaskPage; + const TaskPageElement = TaskPageComponent ?? TaskPage; const customFieldExtensions = useElementFilter(outlet, elements => elements @@ -101,9 +88,7 @@ export const Router = (props: RouterProps) => { element={ } /> diff --git a/plugins/scaffolder/src/components/index.ts b/plugins/scaffolder/src/components/index.ts index 47540ff492..d0442cb906 100644 --- a/plugins/scaffolder/src/components/index.ts +++ b/plugins/scaffolder/src/components/index.ts @@ -15,8 +15,6 @@ */ export * from './fields'; export type { RepoUrlPickerUiOptions } from './fields'; -export { TemplateList } from './TemplateList'; -export type { TemplateListProps } from './TemplateList'; export { TemplateTypePicker } from './TemplateTypePicker'; export * from './secrets'; export { TaskPage } from './TaskPage'; From cabf951c3705bf89e474296984f97f2ade494366 Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 7 Mar 2022 21:16:16 +0000 Subject: [PATCH 06/40] chore: updating the api-reports Signed-off-by: blam --- plugins/scaffolder/api-report.md | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/plugins/scaffolder/api-report.md b/plugins/scaffolder/api-report.md index cc6fc0f3df..d4a2344499 100644 --- a/plugins/scaffolder/api-report.md +++ b/plugins/scaffolder/api-report.md @@ -399,8 +399,6 @@ export type ScaffolderTaskStatus = // @public export interface ScaffolderUseTemplateSecrets { - // @deprecated (undocumented) - setSecret: (input: Record) => void; // (undocumented) setSecrets: (input: Record) => void; } @@ -413,30 +411,6 @@ export type TaskPageProps = { loadingText?: string; }; -// Warning: (ae-missing-release-tag) "TemplateList" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public @deprecated (undocumented) -export const TemplateList: ({ - TemplateCardComponent, - group, -}: TemplateListProps) => JSX.Element | null; - -// Warning: (ae-missing-release-tag) "TemplateListProps" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public @deprecated (undocumented) -export type TemplateListProps = { - TemplateCardComponent?: - | ComponentType<{ - template: TemplateEntityV1beta3; - }> - | undefined; - group?: { - title?: React_2.ReactNode; - titleComponent?: React_2.ReactNode; - filter: (entity: Entity) => boolean; - }; -}; - // Warning: (ae-missing-release-tag) "TemplateParameterSchema" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) From e63e5a9452988f218596df2413096df7dd5f7586 Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 7 Mar 2022 21:26:47 +0000 Subject: [PATCH 07/40] chore: added changeset Signed-off-by: blam --- .changeset/nine-frogs-yell.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .changeset/nine-frogs-yell.md diff --git a/.changeset/nine-frogs-yell.md b/.changeset/nine-frogs-yell.md new file mode 100644 index 0000000000..e34f00fec8 --- /dev/null +++ b/.changeset/nine-frogs-yell.md @@ -0,0 +1,11 @@ +--- +'@backstage/plugin-scaffolder': minor +--- + +Removed the following previously deprecated exports: + +- **BREAKING**: Removed the deprecated `TemplateList` component and the `TemplateListProps` type. Please use the `TemplateCard` to create your own list component instead to render these lists. + +- **BREAKING**: Removed the deprecated `setSecret` method, please use `setSecrets` instead. + +- **BREAKING**: Removed the deprecated `TemplateCardComponent` and `TaskPageComponent` props from the `ScaffolderPage` component. These are now provided using the `components` prop with the shape `{{ TemplateCardComponent: () => JSX.Element, TaskPageComponent: () => JSX.Element }}` From 2c1a2eeccee1b9c8ae6159c961b44fbf83d71ac7 Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 7 Mar 2022 21:53:29 +0000 Subject: [PATCH 08/40] chore: removed the last of the deprecations and fixed up the api-RepoUrlPickerFieldExtension Signed-off-by: blam --- plugins/scaffolder/api-report.md | 111 ++++-------------- plugins/scaffolder/src/components/Router.tsx | 16 +-- .../TemplateTypePicker/TemplateTypePicker.tsx | 5 + .../fields/EntityPicker/EntityPicker.tsx | 11 +- .../EntityTagsPicker/EntityTagsPicker.tsx | 12 +- .../OwnedEntityPicker/OwnedEntityPicker.tsx | 11 +- .../fields/OwnerPicker/OwnerPicker.tsx | 11 +- .../fields/RepoUrlPicker/RepoUrlPicker.tsx | 11 +- .../fields/RepoUrlPicker/validation.ts | 7 ++ plugins/scaffolder/src/index.ts | 1 - plugins/scaffolder/src/plugin.ts | 35 ++++++ plugins/scaffolder/src/types.ts | 53 ++++++++- 12 files changed, 183 insertions(+), 101 deletions(-) diff --git a/plugins/scaffolder/api-report.md b/plugins/scaffolder/api-report.md index d4a2344499..90fee1fc66 100644 --- a/plugins/scaffolder/api-report.md +++ b/plugins/scaffolder/api-report.md @@ -43,25 +43,19 @@ export type CustomFieldValidator = ( }, ) => void; -// Warning: (ae-missing-release-tag) "EntityNamePickerFieldExtension" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export const EntityNamePickerFieldExtension: FieldExtensionComponent< string, {} >; -// Warning: (ae-missing-release-tag) "EntityPickerFieldExtension" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export const EntityPickerFieldExtension: FieldExtensionComponent< string, EntityPickerUiOptions >; -// Warning: (ae-missing-release-tag) "EntityPickerUiOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export interface EntityPickerUiOptions { // (undocumented) allowArbitraryValues?: boolean; @@ -77,9 +71,7 @@ export const EntityTagsPickerFieldExtension: FieldExtensionComponent< EntityTagsPickerUiOptions >; -// Warning: (ae-missing-release-tag) "EntityTagsPickerUiOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export interface EntityTagsPickerUiOptions { // (undocumented) kinds?: string[]; @@ -111,14 +103,7 @@ export type FieldExtensionOptions< validation?: CustomFieldValidator; }; -// Warning: (ae-missing-release-tag) "JobStatus" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) -export type JobStatus = 'PENDING' | 'STARTED' | 'COMPLETED' | 'FAILED'; - -// Warning: (ae-missing-release-tag) "ListActionsResponse" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export type ListActionsResponse = Array<{ id: string; description?: string; @@ -128,9 +113,7 @@ export type ListActionsResponse = Array<{ }; }>; -// Warning: (ae-missing-release-tag) "LogEvent" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export type LogEvent = { type: 'log' | 'completion'; body: { @@ -143,17 +126,13 @@ export type LogEvent = { taskId: string; }; -// Warning: (ae-missing-release-tag) "OwnedEntityPickerFieldExtension" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export const OwnedEntityPickerFieldExtension: FieldExtensionComponent< string, OwnedEntityPickerUiOptions >; -// Warning: (ae-missing-release-tag) "OwnedEntityPickerUiOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export interface OwnedEntityPickerUiOptions { // (undocumented) allowedKinds?: string[]; @@ -161,25 +140,19 @@ export interface OwnedEntityPickerUiOptions { defaultKind?: string; } -// Warning: (ae-missing-release-tag) "OwnerPickerFieldExtension" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export const OwnerPickerFieldExtension: FieldExtensionComponent< string, OwnerPickerUiOptions >; -// Warning: (ae-missing-release-tag) "OwnerPickerUiOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export interface OwnerPickerUiOptions { // (undocumented) allowedKinds?: string[]; } -// Warning: (ae-missing-release-tag) "repoPickerValidation" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export const repoPickerValidation: ( value: string, validation: FieldValidation, @@ -188,17 +161,13 @@ export const repoPickerValidation: ( }, ) => void; -// Warning: (ae-missing-release-tag) "RepoUrlPickerFieldExtension" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export const RepoUrlPickerFieldExtension: FieldExtensionComponent< string, RepoUrlPickerUiOptions >; -// Warning: (ae-missing-release-tag) "RepoUrlPickerUiOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export interface RepoUrlPickerUiOptions { // (undocumented) allowedHosts?: string[]; @@ -216,16 +185,8 @@ export interface RepoUrlPickerUiOptions { }; } -// Warning: (ae-missing-release-tag) "RouterProps" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export type RouterProps = { - TemplateCardComponent?: - | ComponentType<{ - template: TemplateEntityV1beta3; - }> - | undefined; - TaskPageComponent?: ComponentType<{}>; components?: { TemplateCardComponent?: | ComponentType<{ @@ -294,17 +255,13 @@ export class ScaffolderClient implements ScaffolderApi { // @public export const ScaffolderFieldExtensions: React_2.ComponentType; -// Warning: (ae-missing-release-tag) "ScaffolderGetIntegrationsListOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export interface ScaffolderGetIntegrationsListOptions { // (undocumented) allowedHosts: string[]; } -// Warning: (ae-missing-release-tag) "ScaffolderGetIntegrationsListResponse" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export interface ScaffolderGetIntegrationsListResponse { // (undocumented) integrations: { @@ -322,14 +279,10 @@ export type ScaffolderOutputLink = { entityRef?: string; }; -// Warning: (ae-missing-release-tag) "ScaffolderPage" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export const ScaffolderPage: (props: RouterProps) => JSX.Element; -// Warning: (ae-missing-release-tag) "scaffolderPlugin" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export const scaffolderPlugin: BackstagePlugin< { root: RouteRef; @@ -339,9 +292,7 @@ export const scaffolderPlugin: BackstagePlugin< } >; -// Warning: (ae-missing-release-tag) "ScaffolderScaffoldOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export interface ScaffolderScaffoldOptions { // (undocumented) secrets?: Record; @@ -351,17 +302,13 @@ export interface ScaffolderScaffoldOptions { values: Record; } -// Warning: (ae-missing-release-tag) "ScaffolderScaffoldResponse" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export interface ScaffolderScaffoldResponse { // (undocumented) taskId: string; } -// Warning: (ae-missing-release-tag) "ScaffolderStreamLogsOptions" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export interface ScaffolderStreamLogsOptions { // (undocumented) after?: number; @@ -369,9 +316,7 @@ export interface ScaffolderStreamLogsOptions { taskId: string; } -// Warning: (ae-missing-release-tag) "ScaffolderTask" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export type ScaffolderTask = { id: string; spec: TaskSpec; @@ -387,9 +332,7 @@ export type ScaffolderTaskOutput = { [key: string]: unknown; }; -// Warning: (ae-missing-release-tag) "ScaffolderTaskStatus" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export type ScaffolderTaskStatus = | 'open' | 'processing' @@ -411,9 +354,7 @@ export type TaskPageProps = { loadingText?: string; }; -// Warning: (ae-missing-release-tag) "TemplateParameterSchema" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export type TemplateParameterSchema = { title: string; steps: Array<{ @@ -422,9 +363,7 @@ export type TemplateParameterSchema = { }>; }; -// Warning: (ae-missing-release-tag) "TemplateTypePicker" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export const TemplateTypePicker: () => JSX.Element | null; // @public diff --git a/plugins/scaffolder/src/components/Router.tsx b/plugins/scaffolder/src/components/Router.tsx index cedeaaca0f..3c906f21d4 100644 --- a/plugins/scaffolder/src/components/Router.tsx +++ b/plugins/scaffolder/src/components/Router.tsx @@ -32,14 +32,11 @@ import { } from '../extensions'; import { useElementFilter } from '@backstage/core-plugin-api'; +/** + * The props for the entrypoint `ScaffolderPage` component the plugin. + * @public + */ export type RouterProps = { - /** @deprecated use components.TemplateCardComponent instead */ - TemplateCardComponent?: - | ComponentType<{ template: TemplateEntityV1beta3 }> - | undefined; - /** @deprecated use component.TaskPageComponent instead */ - TaskPageComponent?: ComponentType<{}>; - components?: { TemplateCardComponent?: | ComponentType<{ template: TemplateEntityV1beta3 }> @@ -53,6 +50,11 @@ export type RouterProps = { }>; }; +/** + * The main entirypoint `Router` for the `ScaffolderPlugin`. + * + * @public + */ export const Router = (props: RouterProps) => { const { groups, components = {} } = props; diff --git a/plugins/scaffolder/src/components/TemplateTypePicker/TemplateTypePicker.tsx b/plugins/scaffolder/src/components/TemplateTypePicker/TemplateTypePicker.tsx index 0f18b34bb9..e7c2c965f2 100644 --- a/plugins/scaffolder/src/components/TemplateTypePicker/TemplateTypePicker.tsx +++ b/plugins/scaffolder/src/components/TemplateTypePicker/TemplateTypePicker.tsx @@ -34,6 +34,11 @@ import { alertApiRef, useApi } from '@backstage/core-plugin-api'; const icon = ; const checkedIcon = ; +/** + * The component to select the `type` of `Template` that you will see in the table. + * + * @public + */ export const TemplateTypePicker = () => { const alertApi = useApi(alertApiRef); const { error, loading, availableTypes, selectedTypes, setSelectedTypes } = diff --git a/plugins/scaffolder/src/components/fields/EntityPicker/EntityPicker.tsx b/plugins/scaffolder/src/components/fields/EntityPicker/EntityPicker.tsx index 4a2f212fee..17c501d3a1 100644 --- a/plugins/scaffolder/src/components/fields/EntityPicker/EntityPicker.tsx +++ b/plugins/scaffolder/src/components/fields/EntityPicker/EntityPicker.tsx @@ -25,6 +25,12 @@ import React, { useCallback, useEffect } from 'react'; import useAsync from 'react-use/lib/useAsync'; import { FieldExtensionComponentProps } from '../../../extensions'; +/** + * The input props that can be specified under `ui:options` for the + * `EntityPicker` field extension. + * + * @public + */ export interface EntityPickerUiOptions { allowedKinds?: string[]; defaultKind?: string; @@ -32,7 +38,10 @@ export interface EntityPickerUiOptions { } /** - * Entity Picker + * The underling component that is rendered in the form for the `EntityPicker` + * field extension. + * + * @public */ export const EntityPicker = ( props: FieldExtensionComponentProps, diff --git a/plugins/scaffolder/src/components/fields/EntityTagsPicker/EntityTagsPicker.tsx b/plugins/scaffolder/src/components/fields/EntityTagsPicker/EntityTagsPicker.tsx index 41369a58eb..582fe57d72 100644 --- a/plugins/scaffolder/src/components/fields/EntityTagsPicker/EntityTagsPicker.tsx +++ b/plugins/scaffolder/src/components/fields/EntityTagsPicker/EntityTagsPicker.tsx @@ -24,11 +24,21 @@ import { FormControl, TextField } from '@material-ui/core'; import { Autocomplete } from '@material-ui/lab'; import { FieldExtensionComponentProps } from '../../../extensions'; +/** + * The input props that can be specified under `ui:options` for the + * `EntityTagsPicker` field extension. + * + * @public + */ export interface EntityTagsPickerUiOptions { kinds?: string[]; } + /** - * EntityTagsPicker + * The underling component that is rendered in the form for the `EntityTagsPicker` + * field extension. + * + * @public */ export const EntityTagsPicker = ( props: FieldExtensionComponentProps, diff --git a/plugins/scaffolder/src/components/fields/OwnedEntityPicker/OwnedEntityPicker.tsx b/plugins/scaffolder/src/components/fields/OwnedEntityPicker/OwnedEntityPicker.tsx index fb4a7a7234..5f314b6748 100644 --- a/plugins/scaffolder/src/components/fields/OwnedEntityPicker/OwnedEntityPicker.tsx +++ b/plugins/scaffolder/src/components/fields/OwnedEntityPicker/OwnedEntityPicker.tsx @@ -28,13 +28,22 @@ import useAsync from 'react-use/lib/useAsync'; import { FieldExtensionComponentProps } from '../../../extensions'; +/** + * The input props that can be specified under `ui:options` for the + * `OwnedEntityPicker` field extension. + * + * @public + */ export interface OwnedEntityPickerUiOptions { allowedKinds?: string[]; defaultKind?: string; } /** - * Owned Entity Picker + * The underling component that is rendered in the form for the `OwnedEntityPicker` + * field extension. + * + * @public */ export const OwnedEntityPicker = ( props: FieldExtensionComponentProps, diff --git a/plugins/scaffolder/src/components/fields/OwnerPicker/OwnerPicker.tsx b/plugins/scaffolder/src/components/fields/OwnerPicker/OwnerPicker.tsx index 1824275380..5f9a05632c 100644 --- a/plugins/scaffolder/src/components/fields/OwnerPicker/OwnerPicker.tsx +++ b/plugins/scaffolder/src/components/fields/OwnerPicker/OwnerPicker.tsx @@ -17,12 +17,21 @@ import React from 'react'; import { EntityPicker } from '../EntityPicker/EntityPicker'; import { FieldExtensionComponentProps } from '../../../extensions'; +/** + * The input props that can be specified under `ui:options` for the + * `OwnerPicker` field extension. + * + * @public + */ export interface OwnerPickerUiOptions { allowedKinds?: string[]; } /** - * Owner Picker + * The underling component that is rendered in the form for the `OwnerPicker` + * field extension. + * + * @public */ export const OwnerPicker = ( props: FieldExtensionComponentProps, diff --git a/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx b/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx index 17fd5ef665..5af0295d55 100644 --- a/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx +++ b/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx @@ -30,6 +30,12 @@ import { RepoUrlPickerState } from './types'; import useDebounce from 'react-use/lib/useDebounce'; import { useTemplateSecrets } from '../../secrets'; +/** + * The input props that can be specified under `ui:options` for the + * `RepoUrlPicker` field extension. + * + * @public + */ export interface RepoUrlPickerUiOptions { allowedHosts?: string[]; allowedOwners?: string[]; @@ -45,7 +51,10 @@ export interface RepoUrlPickerUiOptions { } /** - * Repo Url Picker + * The underling component that is rendered in the form for the `RepoUrlPicker` + * field extension. + * + * @public */ export const RepoUrlPicker = ( props: FieldExtensionComponentProps, diff --git a/plugins/scaffolder/src/components/fields/RepoUrlPicker/validation.ts b/plugins/scaffolder/src/components/fields/RepoUrlPicker/validation.ts index 24a1c2f5ce..ab5435f5d8 100644 --- a/plugins/scaffolder/src/components/fields/RepoUrlPicker/validation.ts +++ b/plugins/scaffolder/src/components/fields/RepoUrlPicker/validation.ts @@ -18,6 +18,13 @@ import { FieldValidation } from '@rjsf/core'; import { ApiHolder } from '@backstage/core-plugin-api'; import { scmIntegrationsApiRef } from '@backstage/integration-react'; +/** + * The validation function for the `repoUrl` that is returned from the + * field extension. Ensures that you have all the required fields filled for + * the different providers that exist. + * + * @public + */ export const repoPickerValidation = ( value: string, validation: FieldValidation, diff --git a/plugins/scaffolder/src/index.ts b/plugins/scaffolder/src/index.ts index 34b9155d66..bdca2a0952 100644 --- a/plugins/scaffolder/src/index.ts +++ b/plugins/scaffolder/src/index.ts @@ -22,7 +22,6 @@ export { scaffolderApiRef, ScaffolderClient } from './api'; export type { - JobStatus, ListActionsResponse, LogEvent, ScaffolderApi, diff --git a/plugins/scaffolder/src/plugin.ts b/plugins/scaffolder/src/plugin.ts index 0d44a40c80..eb8c0743a4 100644 --- a/plugins/scaffolder/src/plugin.ts +++ b/plugins/scaffolder/src/plugin.ts @@ -34,6 +34,10 @@ import { import { OwnedEntityPicker } from './components/fields/OwnedEntityPicker/OwnedEntityPicker'; import { EntityTagsPicker } from './components/fields/EntityTagsPicker/EntityTagsPicker'; +/** + * The main plugin export for the scaffolder. + * @public + */ export const scaffolderPlugin = createPlugin({ id: 'scaffolder', apis: [ @@ -60,6 +64,11 @@ export const scaffolderPlugin = createPlugin({ }, }); +/** + * A field extension for selecting an Entity that exists in the Catalog. + * + * @public + */ export const EntityPickerFieldExtension = scaffolderPlugin.provide( createScaffolderFieldExtension({ component: EntityPicker, @@ -67,6 +76,11 @@ export const EntityPickerFieldExtension = scaffolderPlugin.provide( }), ); +/** + * The field extension for selecting a name for a new Entity in the Catalog. + * + * @public + */ export const EntityNamePickerFieldExtension = scaffolderPlugin.provide( createScaffolderFieldExtension({ component: EntityNamePicker, @@ -75,6 +89,12 @@ export const EntityNamePickerFieldExtension = scaffolderPlugin.provide( }), ); +/** + * The field extension which provides the ability to select a RepositoryUrl. + * Currently this is an encoded URL that looks something like the following `github.com?repo=myRepoName&owner=backstage`. + * + * @public + */ export const RepoUrlPickerFieldExtension = scaffolderPlugin.provide( createScaffolderFieldExtension({ component: RepoUrlPicker, @@ -83,6 +103,11 @@ export const RepoUrlPickerFieldExtension = scaffolderPlugin.provide( }), ); +/** + * A field extensions for picking users and groups out of the Catalog. + * + * @public + */ export const OwnerPickerFieldExtension = scaffolderPlugin.provide( createScaffolderFieldExtension({ component: OwnerPicker, @@ -90,6 +115,11 @@ export const OwnerPickerFieldExtension = scaffolderPlugin.provide( }), ); +/** + * The Router and main entrypoint to the Scaffolder plugin. + * + * @public + */ export const ScaffolderPage = scaffolderPlugin.provide( createRoutableExtension({ name: 'ScaffolderPage', @@ -98,6 +128,11 @@ export const ScaffolderPage = scaffolderPlugin.provide( }), ); +/** + * A field extension to show all the Entities that are owned by the current logged-in User for use in templates. + * + * @public + */ export const OwnedEntityPickerFieldExtension = scaffolderPlugin.provide( createScaffolderFieldExtension({ component: OwnedEntityPicker, diff --git a/plugins/scaffolder/src/types.ts b/plugins/scaffolder/src/types.ts index 069c28d85c..8a67abeedb 100644 --- a/plugins/scaffolder/src/types.ts +++ b/plugins/scaffolder/src/types.ts @@ -17,6 +17,11 @@ import { TaskSpec } from '@backstage/plugin-scaffolder-common'; import { JsonObject, JsonValue, Observable } from '@backstage/types'; import { JSONSchema7 } from 'json-schema'; +/** + * The status of each task in a Scaffolder Job + * + * @public + */ export type ScaffolderTaskStatus = | 'open' | 'processing' @@ -24,8 +29,11 @@ export type ScaffolderTaskStatus = | 'completed' | 'skipped'; -export type JobStatus = 'PENDING' | 'STARTED' | 'COMPLETED' | 'FAILED'; - +/** + * The shape of each task returned from the `scaffolder-backend` + * + * @public + */ export type ScaffolderTask = { id: string; spec: TaskSpec; @@ -34,6 +42,11 @@ export type ScaffolderTask = { createdAt: string; }; +/** + * The response shape for the `listActions` call to the `scaffolder-backend` + * + * @public + */ export type ListActionsResponse = Array<{ id: string; description?: string; @@ -58,6 +71,12 @@ export type ScaffolderTaskOutput = { [key: string]: unknown; }; +/** + * The shape of each entry of parameters which gets rendered + * as a seperate step in the wizard input + * + * @public + */ export type TemplateParameterSchema = { title: string; steps: Array<{ @@ -66,6 +85,11 @@ export type TemplateParameterSchema = { }>; }; +/** + * The shape of a `LogEvent` message from the `scaffolder-backend` + * + * @public + */ export type LogEvent = { type: 'log' | 'completion'; body: { @@ -78,24 +102,49 @@ export type LogEvent = { taskId: string; }; +/** + * The input options to the `scaffold` method of the `ScaffolderClient`. + * + * @public + */ export interface ScaffolderScaffoldOptions { templateRef: string; values: Record; secrets?: Record; } +/** + * The response shape of the `scaffold` method of the `ScaffolderClient`. + * + * @public + */ export interface ScaffolderScaffoldResponse { taskId: string; } +/** + * The arguments for `getIntergationsList`. + * + * @public + */ export interface ScaffolderGetIntegrationsListOptions { allowedHosts: string[]; } +/** + * The response shape for `getIntegrationsList`. + * + * @public + */ export interface ScaffolderGetIntegrationsListResponse { integrations: { type: string; title: string; host: string }[]; } +/** + * The input options to the `streamLogs` method of the `ScaffolderClient`. + * + * @public + */ export interface ScaffolderStreamLogsOptions { taskId: string; after?: number; From c54fc496438004dc983d1bd0fd63160a5e482213 Mon Sep 17 00:00:00 2001 From: blam Date: Mon, 7 Mar 2022 21:56:20 +0000 Subject: [PATCH 09/40] chore: one more breaking change we missed Signed-off-by: blam --- .changeset/nine-frogs-yell.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.changeset/nine-frogs-yell.md b/.changeset/nine-frogs-yell.md index e34f00fec8..7513bea4a6 100644 --- a/.changeset/nine-frogs-yell.md +++ b/.changeset/nine-frogs-yell.md @@ -9,3 +9,5 @@ Removed the following previously deprecated exports: - **BREAKING**: Removed the deprecated `setSecret` method, please use `setSecrets` instead. - **BREAKING**: Removed the deprecated `TemplateCardComponent` and `TaskPageComponent` props from the `ScaffolderPage` component. These are now provided using the `components` prop with the shape `{{ TemplateCardComponent: () => JSX.Element, TaskPageComponent: () => JSX.Element }}` + +- **BREAKING**: Removed `JobStatus` as this type was actually a legacy type used in `v1alpha` templates and the workflow engine and should no longer be used or depended on. From e6b4b32958fbe006cc1ce12697dd9552ade33bdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Tue, 8 Mar 2022 09:19:56 +0100 Subject: [PATCH 10/40] fix typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/clever-garlics-rescue.md | 2 +- docs/integrations/gitlab/discovery.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.changeset/clever-garlics-rescue.md b/.changeset/clever-garlics-rescue.md index c0ec6550b3..a7dd387042 100644 --- a/.changeset/clever-garlics-rescue.md +++ b/.changeset/clever-garlics-rescue.md @@ -6,7 +6,7 @@ ```diff // In packages/backend/src/plugins/catalog.ts -+import { GitLabDiscoveryProcessor } from '@backstage/plugin-scaffolder-backend-module-gitlab'; ++import { GitLabDiscoveryProcessor } from '@backstage/plugin-catalog-backend-module-gitlab'; export default async function createPlugin( env: PluginEnvironment, diff --git a/docs/integrations/gitlab/discovery.md b/docs/integrations/gitlab/discovery.md index 65b31b9a3d..8d805fa362 100644 --- a/docs/integrations/gitlab/discovery.md +++ b/docs/integrations/gitlab/discovery.md @@ -40,7 +40,7 @@ of your backend. ```diff // In packages/backend/src/plugins/catalog.ts -+import { GitLabDiscoveryProcessor } from '@backstage/plugin-scaffolder-backend-module-gitlab'; ++import { GitLabDiscoveryProcessor } from '@backstage/plugin-catalog-backend-module-gitlab'; export default async function createPlugin( env: PluginEnvironment, From ab7cd7d70e574f22031237c02eee1d3adaa77df5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Tue, 8 Mar 2022 10:13:03 +0100 Subject: [PATCH 11/40] Do some groundwork for supporting the better-sqlite3 driver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/beige-lies-pay.md | 11 +++++++++++ packages/backend-common/src/database/connection.ts | 1 + .../backend-common/src/database/connectors/sqlite3.ts | 7 +++---- packages/backend-tasks/src/tasks/TaskWorker.ts | 7 +++---- packages/backend-tasks/src/tasks/util.ts | 2 +- .../src/lib/assets/StaticAssetsStore.test.ts | 7 +++---- .../app-backend/src/lib/assets/StaticAssetsStore.ts | 2 +- .../migrations/20210326100300_timestamptz.js | 4 ++-- .../migrations/20211117092217_optional_entity_ref.js | 4 ++-- .../migrations/20201123205611_relations_table_uniq.js | 4 ++-- .../src/database/DefaultProcessingDatabase.ts | 7 +++---- .../src/scaffolder/tasks/DatabaseTaskStore.ts | 2 +- 12 files changed, 33 insertions(+), 25 deletions(-) create mode 100644 .changeset/beige-lies-pay.md diff --git a/.changeset/beige-lies-pay.md b/.changeset/beige-lies-pay.md new file mode 100644 index 0000000000..f3d7d70c1f --- /dev/null +++ b/.changeset/beige-lies-pay.md @@ -0,0 +1,11 @@ +--- +'@backstage/backend-common': patch +'@backstage/backend-tasks': patch +'@backstage/plugin-app-backend': patch +'@backstage/plugin-auth-backend': patch +'@backstage/plugin-bazaar-backend': patch +'@backstage/plugin-catalog-backend': patch +'@backstage/plugin-scaffolder-backend': patch +--- + +Do some groundwork for supporting the `better-sqlite3` driver, to maybe eventually replace `@vscode/sqlite3` (#9912) diff --git a/packages/backend-common/src/database/connection.ts b/packages/backend-common/src/database/connection.ts index fd0769b8a2..b61aabbecb 100644 --- a/packages/backend-common/src/database/connection.ts +++ b/packages/backend-common/src/database/connection.ts @@ -33,6 +33,7 @@ type DatabaseClient = 'pg' | 'sqlite3' | 'mysql' | 'mysql2' | string; */ const ConnectorMapping: Record = { pg: pgConnector, + 'better-sqlite3': sqlite3Connector, sqlite3: sqlite3Connector, mysql: mysqlConnector, mysql2: mysqlConnector, diff --git a/packages/backend-common/src/database/connectors/sqlite3.ts b/packages/backend-common/src/database/connectors/sqlite3.ts index 8e47720f82..6426005f0e 100644 --- a/packages/backend-common/src/database/connectors/sqlite3.ts +++ b/packages/backend-common/src/database/connectors/sqlite3.ts @@ -13,12 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import path from 'path'; - -import { ensureDirSync } from 'fs-extra'; -import knexFactory, { Knex } from 'knex'; import { Config } from '@backstage/config'; +import { ensureDirSync } from 'fs-extra'; +import knexFactory, { Knex } from 'knex'; +import path from 'path'; import { mergeDatabaseConfig } from '../config'; import { DatabaseConnector } from '../types'; diff --git a/packages/backend-tasks/src/tasks/TaskWorker.ts b/packages/backend-tasks/src/tasks/TaskWorker.ts index 991d97b956..5d33bfa7a1 100644 --- a/packages/backend-tasks/src/tasks/TaskWorker.ts +++ b/packages/backend-tasks/src/tasks/TaskWorker.ts @@ -229,10 +229,9 @@ export class TaskWorker { // leaning on the database as a central clock source const dbNull = this.knex.raw('null'); const dt = Duration.fromISO(recurringAtMostEveryDuration).as('seconds'); - const nextRun = - this.knex.client.config.client === 'sqlite3' - ? this.knex.raw('datetime(next_run_start_at, ?)', [`+${dt} seconds`]) - : this.knex.raw(`next_run_start_at + interval '${dt} seconds'`); + const nextRun = this.knex.client.config.client.includes('sqlite3') + ? this.knex.raw('datetime(next_run_start_at, ?)', [`+${dt} seconds`]) + : this.knex.raw(`next_run_start_at + interval '${dt} seconds'`); const rows = await this.knex(DB_TASKS_TABLE) .where('id', '=', this.taskId) diff --git a/packages/backend-tasks/src/tasks/util.ts b/packages/backend-tasks/src/tasks/util.ts index 0509f29363..8b247cdeb8 100644 --- a/packages/backend-tasks/src/tasks/util.ts +++ b/packages/backend-tasks/src/tasks/util.ts @@ -40,7 +40,7 @@ export function nowPlus(duration: Duration | undefined, knex: Knex) { if (!seconds) { return knex.fn.now(); } - return knex.client.config.client === 'sqlite3' + return knex.client.config.client.includes('sqlite3') ? knex.raw(`datetime('now', ?)`, [`${seconds} seconds`]) : knex.raw(`now() + interval '${seconds} seconds'`); } diff --git a/plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts b/plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts index 0230486444..95de905e7a 100644 --- a/plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts +++ b/plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts @@ -140,10 +140,9 @@ describe('StaticAssetsStore', () => { const updated = await database('static_assets_cache') .where({ path: 'old' }) .update({ - last_modified_at: - database.client.config.client === 'sqlite3' - ? database.raw(`datetime('now', '-3600 seconds')`) - : database.raw(`now() + interval '-3600 seconds'`), + last_modified_at: database.client.config.client.includes('sqlite3') + ? database.raw(`datetime('now', '-3600 seconds')`) + : database.raw(`now() + interval '-3600 seconds'`), }); expect(updated).toBe(1); diff --git a/plugins/app-backend/src/lib/assets/StaticAssetsStore.ts b/plugins/app-backend/src/lib/assets/StaticAssetsStore.ts index 6b393ce036..70d664307e 100644 --- a/plugins/app-backend/src/lib/assets/StaticAssetsStore.ts +++ b/plugins/app-backend/src/lib/assets/StaticAssetsStore.ts @@ -133,7 +133,7 @@ export class StaticAssetsStore implements StaticAssetProvider { .where( 'last_modified_at', '<=', - this.#db.client.config.client === 'sqlite3' + this.#db.client.config.client.includes('sqlite3') ? this.#db.raw(`datetime('now', ?)`, [`-${maxAgeSeconds} seconds`]) : this.#db.raw(`now() + interval '${-maxAgeSeconds} seconds'`), ) diff --git a/plugins/auth-backend/migrations/20210326100300_timestamptz.js b/plugins/auth-backend/migrations/20210326100300_timestamptz.js index 144f450380..79b839dc6a 100644 --- a/plugins/auth-backend/migrations/20210326100300_timestamptz.js +++ b/plugins/auth-backend/migrations/20210326100300_timestamptz.js @@ -21,7 +21,7 @@ */ exports.up = async function up(knex) { // Sqlite does not support alter column. - if (knex.client.config.client !== 'sqlite3') { + if (knex.client.config.client.includes('sqlite3')) { await knex.schema.alterTable('signing_keys', table => { table .timestamp('created_at', { useTz: true, precision: 0 }) @@ -38,7 +38,7 @@ exports.up = async function up(knex) { */ exports.down = async function down(knex) { // Sqlite does not support alter column. - if (knex.client.config.client !== 'sqlite3') { + if (knex.client.config.client.includes('sqlite3')) { await knex.schema.alterTable('signing_keys', table => { table .timestamp('created_at', { useTz: false, precision: 0 }) diff --git a/plugins/bazaar-backend/migrations/20211117092217_optional_entity_ref.js b/plugins/bazaar-backend/migrations/20211117092217_optional_entity_ref.js index 03ed0a8d05..9dd8097ae2 100644 --- a/plugins/bazaar-backend/migrations/20211117092217_optional_entity_ref.js +++ b/plugins/bazaar-backend/migrations/20211117092217_optional_entity_ref.js @@ -15,7 +15,7 @@ */ exports.up = async function up(knex) { - if (knex.client.config.client === 'sqlite3') { + if (knex.client.config.client.includes('sqlite3')) { await knex.schema.dropTable('metadata'); await knex.schema.createTable('metadata', table => { table.increments('id').comment('Automatically generated unique ID'); @@ -93,7 +93,7 @@ exports.up = async function up(knex) { }; exports.down = async function down(knex) { - if (knex.client.config.client === 'sqlite3') { + if (knex.client.config.client.includes('sqlite3')) { await knex.schema.dropTable('metadata'); await knex.schema.createTable('metadata', table => { table diff --git a/plugins/catalog-backend/migrations/20201123205611_relations_table_uniq.js b/plugins/catalog-backend/migrations/20201123205611_relations_table_uniq.js index b3c3a042f5..996c69de8c 100644 --- a/plugins/catalog-backend/migrations/20201123205611_relations_table_uniq.js +++ b/plugins/catalog-backend/migrations/20201123205611_relations_table_uniq.js @@ -20,7 +20,7 @@ * @param {import('knex').Knex} knex */ exports.up = async function up(knex) { - if (knex.client.config.client === 'sqlite3') { + if (knex.client.config.client.includes('sqlite3')) { // sqlite doesn't support dropPrimary so we recreate it properly instead await knex.schema.dropTable('entities_relations'); await knex.schema.createTable('entities_relations', table => { @@ -58,7 +58,7 @@ exports.up = async function up(knex) { * @param {import('knex').Knex} knex */ exports.down = async function down(knex) { - if (knex.client.config.client === 'sqlite3') { + if (knex.client.config.client.includes('sqlite3')) { await knex.schema.dropTable('entities_relations'); await knex.schema.createTable('entities_relations', table => { table.comment('All relations between entities in the catalog'); diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 55f04c289c..7c278e7272 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -403,10 +403,9 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { items.map(i => i.entity_ref), ) .update({ - next_update_at: - tx.client.config.client === 'sqlite3' - ? tx.raw(`datetime('now', ?)`, [`${interval} seconds`]) - : tx.raw(`now() + interval '${interval} seconds'`), + next_update_at: tx.client.config.client.includes('sqlite3') + ? tx.raw(`datetime('now', ?)`, [`${interval} seconds`]) + : tx.raw(`now() + interval '${interval} seconds'`), }); return { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts index 65b890c265..9fef68529f 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/DatabaseTaskStore.ts @@ -182,7 +182,7 @@ export class DatabaseTaskStore implements TaskStore { .andWhere( 'last_heartbeat_at', '<=', - this.db.client.config.client === 'sqlite3' + this.db.client.config.client.includes('sqlite3') ? this.db.raw(`datetime('now', ?)`, [`-${timeoutS} seconds`]) : this.db.raw(`dateadd('second', ?, ?)`, [ `-${timeoutS}`, From 7290dda9d4e177f30b695c1a61c1df32cfc3daa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Tue, 8 Mar 2022 10:32:58 +0100 Subject: [PATCH 12/40] relax id requirement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/many-tools-buy.md | 5 +++++ packages/backend-tasks/src/tasks/util.test.ts | 4 ++-- packages/backend-tasks/src/tasks/util.ts | 8 +++----- .../src/processors/LdapOrgEntityProvider.ts | 8 +------- 4 files changed, 11 insertions(+), 14 deletions(-) create mode 100644 .changeset/many-tools-buy.md diff --git a/.changeset/many-tools-buy.md b/.changeset/many-tools-buy.md new file mode 100644 index 0000000000..c2a617edfd --- /dev/null +++ b/.changeset/many-tools-buy.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-tasks': patch +--- + +Relaxed the task ID requirement to now support any non-empty string diff --git a/packages/backend-tasks/src/tasks/util.test.ts b/packages/backend-tasks/src/tasks/util.test.ts index f74669a7f4..614de8c6c8 100644 --- a/packages/backend-tasks/src/tasks/util.test.ts +++ b/packages/backend-tasks/src/tasks/util.test.ts @@ -20,14 +20,14 @@ import { delegateAbortController, sleep, validateId } from './util'; describe('util', () => { describe('validateId', () => { - it.each(['a', 'a_b', 'ab123c_2'])( + it.each(['a', 'a_b', 'ab123c_2', 'a!', 'A', 'a-b', 'a.b', '_a', 'a_'])( 'accepts valid inputs, %p', async input => { expect(validateId(input)).toBeUndefined(); }, ); - it.each(['', 'a!', 'A', 'a-b', 'a.b', '_a', 'a_', null, Symbol('a')])( + it.each(['', null, Symbol('a')])( 'rejects invalid inputs, %p', async input => { expect(() => validateId(input as any)).toThrow(); diff --git a/packages/backend-tasks/src/tasks/util.ts b/packages/backend-tasks/src/tasks/util.ts index 0509f29363..829515d721 100644 --- a/packages/backend-tasks/src/tasks/util.ts +++ b/packages/backend-tasks/src/tasks/util.ts @@ -19,12 +19,10 @@ import { Knex } from 'knex'; import { DateTime, Duration } from 'luxon'; import { AbortController, AbortSignal } from 'node-abort-controller'; -// Keep the IDs compatible with e.g. Prometheus +// Keep the IDs compatible with e.g. Prometheus labels export function validateId(id: string) { - if (typeof id !== 'string' || !/^[a-z0-9]+(?:_[a-z0-9]+)*$/.test(id)) { - throw new InputError( - `${id} is not a valid ID, expected string of lowercase characters and digits separated by underscores`, - ); + if (typeof id !== 'string' || !id.trim()) { + throw new InputError(`${id} is not a valid ID, expected non-empty string`); } } diff --git a/plugins/catalog-backend-module-ldap/src/processors/LdapOrgEntityProvider.ts b/plugins/catalog-backend-module-ldap/src/processors/LdapOrgEntityProvider.ts index f7d07a15af..bad3941767 100644 --- a/plugins/catalog-backend-module-ldap/src/processors/LdapOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-ldap/src/processors/LdapOrgEntityProvider.ts @@ -217,7 +217,7 @@ export class LdapOrgEntityProvider implements EntityProvider { } this.scheduleFn = async () => { - const id = this.getScheduledTaskId(); + const id = `${this.getProviderName()}:refresh`; await schedule.run({ id, fn: async () => { @@ -236,12 +236,6 @@ export class LdapOrgEntityProvider implements EntityProvider { }); }; } - - // Gets a suitable scheduler task ID for this provider instance - private getScheduledTaskId(): string { - const rawId = `refresh_${this.getProviderName()}`; - return rawId.toLocaleLowerCase('en-US').replace(/[^a-z0-9]/g, '_'); - } } // Helps wrap the timing and logging behaviors From 793e720bc7b5996b8ed238e3daf24c16dcc89a7d Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 8 Mar 2022 13:09:56 +0100 Subject: [PATCH 13/40] docs: added initial package role migration docs Signed-off-by: Patrik Oldsberg --- docs/tutorials/package-role-migration.md | 143 +++++++++++++++++++++++ microsite/sidebars.json | 1 + mkdocs.yml | 1 + 3 files changed, 145 insertions(+) create mode 100644 docs/tutorials/package-role-migration.md diff --git a/docs/tutorials/package-role-migration.md b/docs/tutorials/package-role-migration.md new file mode 100644 index 0000000000..76847d47b1 --- /dev/null +++ b/docs/tutorials/package-role-migration.md @@ -0,0 +1,143 @@ +--- +id: package-role-migration +title: Package Role Migration +description: Guide for how to migrate packages to use the new role utility +--- + +The Backstage CLI has introduced the concept of package roles, whose purpose is to +enable more powerful tooling and leaner package configuration. More background and +information about the change can be found in the [original RFC](https://github.com/backstage/backstage/issues/8729). + +Package roles are implemented through a well-known `"backstage"."role"` field in the +`package.json` of each package. There are a handful of roles defined so far, and it +is not possible to use value outside the set of predefined roles. Some examples of +these roles are `frontend-plugin`, `node-library`, and `backend-plugin-module`. + +With roles in place in all packages, the Backstage CLI is able to automatically +determine how to handle each package. For example, the different build commands +have been replaced by a single one that instead knows how to build each role. +The test and lint configurations are also selected automatically based on the role, and +a new category of `repo` commands have been introduced in the CLI, which are able +to operate across all packages at once. + +Package roles have been used in the Backstage main repository for a while, and +we now recommend that all Backstage projects are migrated to use package roles. + +## Migration + +In order to make the migration as smooth as possible, `@backstage/cli` provides +a number of migration utilities. Using these in combination with some manual review +and optional steps should be all you need to migrate to package roles in most projects. + +Before you begin the migration, make sure you have updated to the most recent version of +the `@backstage/cli`. + +### TL;DR, Step 1-4: + +This is a sorter version of all of the steps below, in case you're in a hurry. + +Run the following commands: + +```sh +yarn backstage-cli migrate package-roles +yarn backstage-cli migrate package-scripts +yarn backstage-cli migrate package-lint-configs +``` + +Have a look at the new commands under `yarn backstage-cli repo`, and switch to them wherever you can. They tend to be a much faster compared to their `lerna` equivalents. + +### Step 1 - Add package roles + +The first step is to add the `"backstage"."role"` field to each package. This +is done by running the following command: + +```sh +yarn backstage-cli migrate package-roles +``` + +This will add the role field to each package in your project, detecting the role +based on existing information like what build scripts are in place and the package name. + +This automatic detection is not perfect, so it recommended to manually review the +roles that were assigned to each package. +You can use the [package role definitions](./not-found#TODO) as a reference. + +### Step 2 - Migrate package scripts + +The migration to package roles also introduces a new `package` command category to the CLI. +Each command under the `package` category is designed to be mapped directly to an entry in `"scripts"` in `package.json`. These commands replace the existing commands like `build`, `app:build`, `lint` and `test`. They look something like this: + +```json +{ + "scripts": { + "start": "backstage-cli package start", + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + ... + } +} +``` + +Every package role each has a fixed set of recommended scripts. It is strongly recommended that you use these scripts, as it allows for optimizations in other parts of the CLI. You can migrate to using all of these scripts by running the following command: + +```sh +yarn backstage-cli migrate package-scripts +``` + +The migration command also carries over any existing flags that were being passed in the old scripts. + +If you in the end do not want to use this exact script setup, it is still recommended to migrate to using the `package` commands, as the top-level commands will be deprecated and removed. If you don't want to use package roles either, you can pass an explicit role to some of the package commands, for example `yarn backstage-cli package build --role web-library`. + +### Step 3 - Migrate package ESLint configurations + +An area that has been simplified as part of the move to package roles is the ESLint configuration. Rather than having each package select which configuration they want (and getting it wrong), they now use a shared configuration factory that utilizes the package role. + +A minimal `.eslintrc.js` configuration now looks like this: + +```js +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); +``` + +You can provide custom overrides for each package using the optional second argument: + +```js +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname, { + ignorePatterns: ['templates/'], + rules: { + 'jest/expect-expect': 'off', + }, +}); +``` + +The configuration factory also provides utilities for extending the configuration in ways that are otherwise very cumbersome to do with plain ESLint, particularly for rules like `no-restricted-syntax`. You can read more about that in the [build system documentation](./not-found#TODO). + +To migrate the ESLint configuration of all packages in your project, run the following command: + +```sh +yarn backstage-cli migrate package-lint-configs +``` + +This will migrate all existing `.eslintrc.js` that extend the old configuration from `@backstage/cli`, as well as carry over any additional configuration. + +### Step 4 - Use `backstage-cli repo` + +The Backstage CLI recently introduced a new `repo` command category, which houses commands that operate on an entire monorepo at once. These commands work particularly well once packages have been migrated to use roles, as that allows for some very effective optimizations. It is typically much faster to use these commands compared to using tools like `lerna`, as they're able to avoid the overhead of calling package scripts through `yarn`. You can read more about the `repo` command in the [CLI command documentation](./not-found#TODO). + +The way to execute this step of the migration is not as well defined as the previous steps, as it depends on what your development and CI/CD setup looks like. Look for the following patterns to replace in your root `package.json` as well as CI/CD setup: + +- Commands that lint the entire repo should be replaced with `yarn backstage-cli repo lint` along with a `--since` flag if needed. For example this: + + ```sh + lerna run lint --since origin/master -- + ``` + + would be replaced by the following: + + ```sh + backstage-cli repo lint --since origin/master + ``` + +- In places where the entire repo is being built, use `yarn backstage-cli repo build`, which also supports the `--since` flag. The migration here is a bit more nuanced as it depends why you are building all packages. + - If you are building all packages to **verify** that you are able to build them, you most likely want `backstage-cli repo build --all`. The `--all` flag signals that bundled packages like `packages/app` and `packages/backend` should be build as well. Pair this up with a `--since` flag in CI to avoid needing to build all packages. + - If you are building all packages to **publish** them, then `backstage-cli repo build` is enough, as it builds all published packages. + - If you are building all packages to **deploy** them, you likely don't want to use the `repo` command at all, simply call `yarn build` in the packages you want to deploy instead. For example, if you are deploying the backend with a docker host build, it's enough to call `yarn build` inside `packages/backend`. diff --git a/microsite/sidebars.json b/microsite/sidebars.json index 5369182013..1d409ae051 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -273,6 +273,7 @@ "Tutorials": [ "tutorials/journey", "tutorials/quickstart-app-plugin", + "tutorials/package-role-migration", "tutorials/migrating-away-from-core", "tutorials/configuring-plugin-databases", "tutorials/switching-sqlite-postgres", diff --git a/mkdocs.yml b/mkdocs.yml index 4671a10b7a..65e1a85bb4 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -170,6 +170,7 @@ nav: - Deprecations: 'api/deprecations.md' - Tutorials: - Future developer journey: 'tutorials/journey.md' + - Package Role Migration: 'tutorials/package-role-migration.md' - Migrating away from @backstage/core: 'tutorials/migrating-away-from-core.md' - Adding Custom Plugin to Existing Monorepo App: 'tutorials/quickstart-app-plugin.md' - Switching Backstage from SQLite to PostgreSQL: 'tutorials/switching-sqlite-postgres.md' From dc104b3f4a391e0aadf58764b399e0730caa391c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Tue, 8 Mar 2022 12:02:02 +0100 Subject: [PATCH 14/40] more prep work for better-sqlite3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/beige-lies-pay.md | 1 + packages/backend-common/package.json | 1 + .../src/database/DatabaseManager.ts | 4 +- .../backend-common/src/database/connection.ts | 8 +- .../src/database/connectors/sqlite3.test.ts | 81 +++++++++++++++++ .../src/database/TestDatabases.ts | 5 +- .../backend-test-utils/src/database/types.ts | 1 + .../migrations/20200702153613_entities.js | 4 +- .../migrations/20200807120600_entitySearch.js | 4 +- .../20201005122705_add_entity_full_name.js | 2 +- .../20201006130744_entity_data_column.js | 2 +- .../migrations/20201210185851_fk_index.js | 4 +- .../20201230103504_update_log_varchar.js | 4 +- .../20210209121210_locations_fk_index.js | 4 +- .../src/database/DefaultProcessingDatabase.ts | 2 +- .../migrations/20210120143715_init.js | 2 +- yarn.lock | 86 ++++++++++++++++++- 17 files changed, 193 insertions(+), 22 deletions(-) diff --git a/.changeset/beige-lies-pay.md b/.changeset/beige-lies-pay.md index f3d7d70c1f..94206f9d9c 100644 --- a/.changeset/beige-lies-pay.md +++ b/.changeset/beige-lies-pay.md @@ -1,6 +1,7 @@ --- '@backstage/backend-common': patch '@backstage/backend-tasks': patch +'@backstage/backend-test-utils': patch '@backstage/plugin-app-backend': patch '@backstage/plugin-auth-backend': patch '@backstage/plugin-bazaar-backend': patch diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 907f8e53e8..48c2095028 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -106,6 +106,7 @@ "@types/unzipper": "^0.10.3", "@types/webpack-env": "^1.15.2", "aws-sdk-mock": "^5.2.1", + "better-sqlite3": "^7.5.0", "http-errors": "^2.0.0", "jest": "^26.0.1", "mock-fs": "^5.1.0", diff --git a/packages/backend-common/src/database/DatabaseManager.ts b/packages/backend-common/src/database/DatabaseManager.ts index 3aadd30148..503945e594 100644 --- a/packages/backend-common/src/database/DatabaseManager.ts +++ b/packages/backend-common/src/database/DatabaseManager.ts @@ -119,7 +119,7 @@ export class DatabaseManager { private getDatabaseName(pluginId: string): string | undefined { const connection = this.getConnectionConfig(pluginId); - if (this.getClientType(pluginId).client === 'sqlite3') { + if (this.getClientType(pluginId).client.includes('sqlite3')) { const sqliteFilename: string | undefined = ( connection as Knex.Sqlite3ConnectionConfig ).filename; @@ -224,7 +224,7 @@ export class DatabaseManager { ); if ( - client === 'sqlite3' && + client.includes('sqlite3') && 'filename' in baseConnection && baseConnection.filename !== ':memory:' ) { diff --git a/packages/backend-common/src/database/connection.ts b/packages/backend-common/src/database/connection.ts index b61aabbecb..5f487886b4 100644 --- a/packages/backend-common/src/database/connection.ts +++ b/packages/backend-common/src/database/connection.ts @@ -23,7 +23,13 @@ import { DatabaseConnector } from './types'; import { mysqlConnector, pgConnector, sqlite3Connector } from './connectors'; -type DatabaseClient = 'pg' | 'sqlite3' | 'mysql' | 'mysql2' | string; +type DatabaseClient = + | 'pg' + | 'better-sqlite3' + | 'sqlite3' + | 'mysql' + | 'mysql2' + | string; /** * Mapping of client type to supported database connectors diff --git a/packages/backend-common/src/database/connectors/sqlite3.test.ts b/packages/backend-common/src/database/connectors/sqlite3.test.ts index cfc9f61527..d751d895d0 100644 --- a/packages/backend-common/src/database/connectors/sqlite3.test.ts +++ b/packages/backend-common/src/database/connectors/sqlite3.test.ts @@ -101,3 +101,84 @@ describe('sqlite3', () => { }); }); }); + +describe('better-sqlite3', () => { + const createConfig = (connection: any) => + new ConfigReader({ client: 'better-sqlite3', connection }); + + describe('buildSqliteDatabaseConfig', () => { + it('builds an in-memory connection', () => { + expect(buildSqliteDatabaseConfig(createConfig(':memory:'))).toEqual({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true, + }); + }); + + it('builds an in-memory connection by override with filename', () => { + expect( + buildSqliteDatabaseConfig( + createConfig(path.join('path', 'to', 'foo')), + { connection: ':memory:' }, + ), + ).toEqual({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true, + }); + }); + + it('builds a persistent connection, normalize config with filename', () => { + expect( + buildSqliteDatabaseConfig(createConfig(path.join('path', 'to', 'foo'))), + ).toEqual({ + client: 'better-sqlite3', + connection: { filename: path.join('path', 'to', 'foo') }, + useNullAsDefault: true, + }); + }); + + it('builds a persistent connection', () => { + expect( + buildSqliteDatabaseConfig( + createConfig({ + filename: path.join('path', 'to', 'foo'), + }), + ), + ).toEqual({ + client: 'better-sqlite3', + connection: { + filename: path.join('path', 'to', 'foo'), + }, + useNullAsDefault: true, + }); + }); + + it('replaces the connection with an override', () => { + expect( + buildSqliteDatabaseConfig(createConfig(':memory:'), { + connection: { filename: path.join('path', 'to', 'foo') }, + }), + ).toEqual({ + client: 'better-sqlite3', + connection: { + filename: path.join('path', 'to', 'foo'), + }, + useNullAsDefault: true, + }); + }); + }); + + describe('createSqliteDatabaseClient', () => { + it('creates an in memory knex instance', () => { + expect( + createSqliteDatabaseClient( + createConfig({ + client: 'better-sqlite3', + connection: ':memory:', + }), + ), + ).toBeTruthy(); + }); + }); +}); diff --git a/packages/backend-test-utils/src/database/TestDatabases.ts b/packages/backend-test-utils/src/database/TestDatabases.ts index 243c6bbe52..a7f5ceb344 100644 --- a/packages/backend-test-utils/src/database/TestDatabases.ts +++ b/packages/backend-test-utils/src/database/TestDatabases.ts @@ -182,6 +182,7 @@ export class TestDatabases { return this.initPostgres(properties); case 'mysql2': return this.initMysql(properties); + case 'better-sqlite3': case 'sqlite3': return this.initSqlite(properties); default: @@ -240,13 +241,13 @@ export class TestDatabases { } private async initSqlite( - _properties: TestDatabaseProperties, + properties: TestDatabaseProperties, ): Promise { const databaseManager = DatabaseManager.fromConfig( new ConfigReader({ backend: { database: { - client: 'sqlite3', + client: properties.driver, connection: ':memory:', }, }, diff --git a/packages/backend-test-utils/src/database/types.ts b/packages/backend-test-utils/src/database/types.ts index d405f73738..5aba48952f 100644 --- a/packages/backend-test-utils/src/database/types.ts +++ b/packages/backend-test-utils/src/database/types.ts @@ -40,6 +40,7 @@ export type Instance = { databaseManager: DatabaseManager; connections: Array; }; + export const allDatabases: Record = Object.freeze({ POSTGRES_13: { diff --git a/plugins/catalog-backend/migrations/20200702153613_entities.js b/plugins/catalog-backend/migrations/20200702153613_entities.js index fef8a7c074..6292063dac 100644 --- a/plugins/catalog-backend/migrations/20200702153613_entities.js +++ b/plugins/catalog-backend/migrations/20200702153613_entities.js @@ -21,7 +21,7 @@ */ exports.up = async function up(knex) { // SQLite does not support FK and PK - if (knex.client.config.client !== 'sqlite3') { + if (!knex.client.config.client.includes('sqlite3')) { await knex.schema.alterTable('entities_search', table => { table.dropForeign(['entity_id']); }); @@ -130,7 +130,7 @@ exports.up = async function up(knex) { */ exports.down = async function down(knex) { // SQLite does not support FK and PK - if (knex.client.config.client !== 'sqlite3') { + if (!knex.client.config.client.includes('sqlite3')) { await knex.schema.alterTable('entities_search', table => { table.dropForeign(['entity_id']); }); diff --git a/plugins/catalog-backend/migrations/20200807120600_entitySearch.js b/plugins/catalog-backend/migrations/20200807120600_entitySearch.js index aa05e79be9..9c7d966087 100644 --- a/plugins/catalog-backend/migrations/20200807120600_entitySearch.js +++ b/plugins/catalog-backend/migrations/20200807120600_entitySearch.js @@ -21,7 +21,7 @@ */ exports.up = async function up(knex) { // Sqlite does not support alter column. - if (knex.client.config.client !== 'sqlite3') { + if (!knex.client.config.client.includes('sqlite3')) { await knex.schema.alterTable('entities_search', table => { table.text('value').nullable().alter({ alterType: true }); }); @@ -33,7 +33,7 @@ exports.up = async function up(knex) { */ exports.down = async function down(knex) { // Sqlite does not support alter column. - if (knex.client.config.client !== 'sqlite3') { + if (!knex.client.config.client.includes('sqlite3')) { await knex.schema.alterTable('entities_search', table => { table.string('value').nullable().alter({ alterType: true }); }); diff --git a/plugins/catalog-backend/migrations/20201005122705_add_entity_full_name.js b/plugins/catalog-backend/migrations/20201005122705_add_entity_full_name.js index 366a4b7044..cd13f610cd 100644 --- a/plugins/catalog-backend/migrations/20201005122705_add_entity_full_name.js +++ b/plugins/catalog-backend/migrations/20201005122705_add_entity_full_name.js @@ -31,7 +31,7 @@ exports.up = async function up(knex) { }); // SQLite does not support alter column - if (knex.client.config.client !== 'sqlite3') { + if (!knex.client.config.client.includes('sqlite3')) { await knex.schema.alterTable('entities', table => { table.text('full_name').notNullable().alter({ alterNullable: true }); }); diff --git a/plugins/catalog-backend/migrations/20201006130744_entity_data_column.js b/plugins/catalog-backend/migrations/20201006130744_entity_data_column.js index 35b0474f06..47097fc788 100644 --- a/plugins/catalog-backend/migrations/20201006130744_entity_data_column.js +++ b/plugins/catalog-backend/migrations/20201006130744_entity_data_column.js @@ -41,7 +41,7 @@ exports.up = async function up(knex) { }); // SQLite does not support ALTER COLUMN. - if (knex.client.config.client !== 'sqlite3') { + if (!knex.client.config.client.includes('sqlite3')) { await knex.schema.alterTable('entities', table => { table.text('data').notNullable().alter({ alterNullable: true }); }); diff --git a/plugins/catalog-backend/migrations/20201210185851_fk_index.js b/plugins/catalog-backend/migrations/20201210185851_fk_index.js index 11907b24f4..84ec03f4e3 100644 --- a/plugins/catalog-backend/migrations/20201210185851_fk_index.js +++ b/plugins/catalog-backend/migrations/20201210185851_fk_index.js @@ -20,7 +20,7 @@ * @param {import('knex').Knex} knex */ exports.up = async function up(knex) { - if (knex.client.config.client !== 'sqlite3') { + if (!knex.client.config.client.includes('sqlite3')) { await knex.schema.alterTable('entities_relations', table => { table.index('originating_entity_id', 'originating_entity_id_idx'); }); @@ -34,7 +34,7 @@ exports.up = async function up(knex) { * @param {import('knex').Knex} knex */ exports.down = async function down(knex) { - if (knex.client.config.client !== 'sqlite3') { + if (!knex.client.config.client.includes('sqlite3')) { await knex.schema.alterTable('entities_relations', table => { table.dropIndex([], 'originating_entity_id_idx'); }); diff --git a/plugins/catalog-backend/migrations/20201230103504_update_log_varchar.js b/plugins/catalog-backend/migrations/20201230103504_update_log_varchar.js index 9a5ccce9fd..a12130e95f 100644 --- a/plugins/catalog-backend/migrations/20201230103504_update_log_varchar.js +++ b/plugins/catalog-backend/migrations/20201230103504_update_log_varchar.js @@ -20,7 +20,7 @@ * @param {import('knex').Knex} knex */ exports.up = async function up(knex) { - if (knex.client.config.client !== 'sqlite3') { + if (!knex.client.config.client.includes('sqlite3')) { // We actually just want to widen columns, but can't do that while a // view is dependent on them - so we just reconstruct it exactly as it was await knex.schema @@ -49,7 +49,7 @@ exports.up = async function up(knex) { * @param {import('knex').Knex} knex */ exports.down = async function down(knex) { - if (knex.client.config.client !== 'sqlite3') { + if (!knex.client.config.client.includes('sqlite3')) { await knex.schema .raw('DROP VIEW location_update_log_latest;') .alterTable('location_update_log', table => { diff --git a/plugins/catalog-backend/migrations/20210209121210_locations_fk_index.js b/plugins/catalog-backend/migrations/20210209121210_locations_fk_index.js index 80925f47ce..34f465de9a 100644 --- a/plugins/catalog-backend/migrations/20210209121210_locations_fk_index.js +++ b/plugins/catalog-backend/migrations/20210209121210_locations_fk_index.js @@ -20,7 +20,7 @@ * @param {import('knex').Knex} knex */ exports.up = async function up(knex) { - if (knex.client.config.client !== 'sqlite3') { + if (!knex.client.config.client.includes('sqlite3')) { await knex.schema.alterTable('entities', table => { table.index('location_id', 'entity_location_id_idx'); }); @@ -34,7 +34,7 @@ exports.up = async function up(knex) { * @param {import('knex').Knex} knex */ exports.down = async function down(knex) { - if (knex.client.config.client !== 'sqlite3') { + if (!knex.client.config.client.includes('sqlite3')) { await knex.schema.alterTable('entities', table => { table.dropIndex([], 'entity_location_id_idx'); }); diff --git a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts index 7c278e7272..6ffd054435 100644 --- a/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/database/DefaultProcessingDatabase.ts @@ -588,7 +588,7 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { // We have to do this because the only way to detect if there was a conflict with // SQLite is to catch the error, while Postgres needs to ignore the conflict to not // break the ongoing transaction. - if (tx.client.config.client !== 'sqlite3') { + if (!tx.client.config.client.includes('sqlite3')) { query = query.onConflict('entity_ref').ignore() as any; // type here does not match runtime } diff --git a/plugins/scaffolder-backend/migrations/20210120143715_init.js b/plugins/scaffolder-backend/migrations/20210120143715_init.js index fb53ce5b5e..08385a3dab 100644 --- a/plugins/scaffolder-backend/migrations/20210120143715_init.js +++ b/plugins/scaffolder-backend/migrations/20210120143715_init.js @@ -75,7 +75,7 @@ exports.up = async function up(knex) { * @param {import('knex').Knex} knex */ exports.down = async function down(knex) { - if (knex.client.config.client !== 'sqlite3') { + if (!knex.client.config.client.includes('sqlite3')) { await knex.schema.alterTable('task_events', table => { table.dropIndex([], 'task_events_task_id_idx'); }); diff --git a/yarn.lock b/yarn.lock index 4327c31d5a..7480544eee 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8108,6 +8108,14 @@ better-path-resolve@1.0.0: dependencies: is-windows "^1.0.0" +better-sqlite3@^7.5.0: + version "7.5.0" + resolved "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-7.5.0.tgz#2a91cb616453f002096743b0e5b66a7021cd1c63" + integrity sha512-6FdG9DoytYGDhLW7VWW1vxjEz7xHkqK6LnaUQYA8d6GHNgZhu9PFX2xwKEEnSBRoT1J4PjTUPeg217ShxNmuPg== + dependencies: + bindings "^1.5.0" + prebuild-install "^7.0.0" + bfj@^7.0.2: version "7.0.2" resolved "https://registry.npmjs.org/bfj/-/bfj-7.0.2.tgz#1988ce76f3add9ac2913fd8ba47aad9e651bfbb2" @@ -8168,6 +8176,13 @@ binaryextensions@^4.15.0, binaryextensions@^4.16.0: resolved "https://registry.npmjs.org/binaryextensions/-/binaryextensions-4.18.0.tgz#22aeada2d14de062c60e8ca59a504a5636a76ceb" integrity sha512-PQu3Kyv9dM4FnwB7XGj1+HucW+ShvJzJqjuw1JkKVs1mWdwOKVcRjOi+pV9X52A0tNvrPCsPkbFFQb+wE1EAXw== +bindings@^1.5.0: + version "1.5.0" + resolved "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz#10353c9e945334bc0511a6d90b38fbc7c9c504df" + integrity sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ== + dependencies: + file-uri-to-path "1.0.0" + bintrees@1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/bintrees/-/bintrees-1.0.1.tgz#0e655c9b9c2435eaab68bf4027226d2b55a34524" @@ -10760,6 +10775,11 @@ detect-libc@^1.0.3: resolved "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz#fa137c4bd698edf55cd5cd02ac559f91a4c4ba9b" integrity sha1-+hN8S9aY7fVc1c0CrFWfkaTEups= +detect-libc@^2.0.0: + version "2.0.1" + resolved "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.1.tgz#e1897aa88fa6ad197862937fbc0441ef352ee0cd" + integrity sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w== + detect-newline@^3.0.0: version "3.1.0" resolved "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz#576f5dfc63ae1a192ff192d8ad3af6308991b651" @@ -12124,6 +12144,11 @@ expand-brackets@^2.1.4: snapdragon "^0.8.1" to-regex "^3.0.1" +expand-template@^2.0.3: + version "2.0.3" + resolved "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz#6e14b3fcee0f3a6340ecb57d2e8918692052a47c" + integrity sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg== + expect@^26.6.2: version "26.6.2" resolved "https://registry.npmjs.org/expect/-/expect-26.6.2.tgz#c6b996bf26bf3fe18b67b2d0f51fc981ba934417" @@ -12481,6 +12506,11 @@ file-type@^9.0.0: resolved "https://registry.npmjs.org/file-type/-/file-type-9.0.0.tgz#a68d5ad07f486414dfb2c8866f73161946714a18" integrity sha512-Qe/5NJrgIOlwijpq3B7BEpzPFcgzggOTagZmkXQY4LA6bsXKTUstK7Wp12lEJ/mLKTpvIZxmIuRcLYWT6ov9lw== +file-uri-to-path@1.0.0: + version "1.0.0" + resolved "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz#553a7b8446ff6f684359c445f1e37a05dacc33dd" + integrity sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw== + filelist@^1.0.1: version "1.0.2" resolved "https://registry.npmjs.org/filelist/-/filelist-1.0.2.tgz#80202f21462d4d1c2e214119b1807c1bc0380e5b" @@ -13143,6 +13173,11 @@ gitconfiglocal@^1.0.0: dependencies: ini "^1.3.2" +github-from-package@0.0.0: + version "0.0.0" + resolved "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz#97fb5d96bfde8973313f20e8288ef9a167fa64ce" + integrity sha1-l/tdlr/eiXMxPyDoKI75oWf6ZM4= + glob-parent@^5.1.0, glob-parent@^5.1.1, glob-parent@~5.1.2: version "5.1.2" resolved "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4" @@ -17986,6 +18021,11 @@ mkdirp-classic@^0.5.2: resolved "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.2.tgz#54c441ce4c96cd7790e10b41a87aa51068ecab2b" integrity sha512-ejdnDQcR75gwknmMw/tx02AuRs8jCtqFoFqDZMjiNxsu85sRIJVXDKHuLYvUUPRBUtV2FpSZa9bL1BUa3BdR2g== +mkdirp-classic@^0.5.3: + version "0.5.3" + resolved "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113" + integrity sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A== + mkdirp-infer-owner@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/mkdirp-infer-owner/-/mkdirp-infer-owner-2.0.0.tgz#55d3b368e7d89065c38f32fd38e638f0ab61d316" @@ -18227,6 +18267,11 @@ nanomatch@^1.2.9: snapdragon "^0.8.1" to-regex "^3.0.1" +napi-build-utils@^1.0.1: + version "1.0.2" + resolved "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz#b1fddc0b2c46e380a0b7a76f984dd47c41a13806" + integrity sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg== + natural-compare@^1.4.0: version "1.4.0" resolved "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" @@ -18281,6 +18326,13 @@ no-case@^3.0.4: lower-case "^2.0.2" tslib "^2.0.3" +node-abi@^3.3.0: + version "3.8.0" + resolved "https://registry.npmjs.org/node-abi/-/node-abi-3.8.0.tgz#679957dc8e7aa47b0a02589dbfde4f77b29ccb32" + integrity sha512-tzua9qWWi7iW4I42vUPKM+SfaF0vQSLAm4yO5J83mSwB7GeoWrDKC/K+8YCnYNwqP5duwazbw2X9l4m8SC2cUw== + dependencies: + semver "^7.3.5" + node-abort-controller@^3.0.1: version "3.0.1" resolved "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.0.1.tgz#f91fa50b1dee3f909afabb7e261b1e1d6b0cb74e" @@ -18659,7 +18711,7 @@ npm-run-path@^4.0.0, npm-run-path@^4.0.1: dependencies: path-key "^3.0.0" -npmlog@^4.1.2: +npmlog@^4.0.1, npmlog@^4.1.2: version "4.1.2" resolved "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz#08a7f2a8bf734604779a9efa4ad5cc717abb954b" integrity sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg== @@ -20233,6 +20285,25 @@ postgres-interval@^1.1.0: dependencies: xtend "^4.0.0" +prebuild-install@^7.0.0: + version "7.0.1" + resolved "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.0.1.tgz#c10075727c318efe72412f333e0ef625beaf3870" + integrity sha512-QBSab31WqkyxpnMWQxubYAHR5S9B2+r81ucocew34Fkl98FhvKIF50jIJnNOBmAZfyNV7vE5T6gd3hTVWgY6tg== + dependencies: + detect-libc "^2.0.0" + expand-template "^2.0.3" + github-from-package "0.0.0" + minimist "^1.2.3" + mkdirp-classic "^0.5.3" + napi-build-utils "^1.0.1" + node-abi "^3.3.0" + npmlog "^4.0.1" + pump "^3.0.0" + rc "^1.2.7" + simple-get "^4.0.0" + tar-fs "^2.0.0" + tunnel-agent "^0.6.0" + precond@0.2: version "0.2.3" resolved "https://registry.npmjs.org/precond/-/precond-0.2.3.tgz#aa9591bcaa24923f1e0f4849d240f47efc1075ac" @@ -20738,7 +20809,7 @@ rc-util@^5.16.1: react-is "^16.12.0" shallowequal "^1.1.0" -rc@^1.2.8: +rc@^1.2.7, rc@^1.2.8: version "1.2.8" resolved "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz#cd924bf5200a075b83c188cd6b9e211b7fc0d3ed" integrity sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw== @@ -22517,6 +22588,15 @@ simple-get@^3.0.2, simple-get@^3.0.3: once "^1.3.1" simple-concat "^1.0.0" +simple-get@^4.0.0: + version "4.0.1" + resolved "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz#4a39db549287c979d352112fa03fd99fd6bc3543" + integrity sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA== + dependencies: + decompress-response "^6.0.0" + once "^1.3.1" + simple-concat "^1.0.0" + simple-swizzle@^0.2.2: version "0.2.2" resolved "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz#a4da6b635ffcccca33f70d17cb92592de95e557a" @@ -23642,7 +23722,7 @@ tapable@^2.0.0, tapable@^2.1.1, tapable@^2.2.0, tapable@^2.2.1: resolved "https://registry.npmjs.org/tapable/-/tapable-2.2.1.tgz#1967a73ef4060a82f12ab96af86d52fdb76eeca0" integrity sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ== -tar-fs@2.1.1, tar-fs@^2.1.1: +tar-fs@2.1.1, tar-fs@^2.0.0, tar-fs@^2.1.1: version "2.1.1" resolved "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz#489a15ab85f1f0befabb370b7de4f9eb5cbe8784" integrity sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng== From d06d67bf8afe63e67ddcb5ee1aff067f8d08d994 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 8 Mar 2022 13:33:34 +0100 Subject: [PATCH 15/40] docs: add package role migration FAQ Signed-off-by: Patrik Oldsberg --- docs/tutorials/package-role-migration.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/tutorials/package-role-migration.md b/docs/tutorials/package-role-migration.md index 76847d47b1..2aee853673 100644 --- a/docs/tutorials/package-role-migration.md +++ b/docs/tutorials/package-role-migration.md @@ -141,3 +141,22 @@ The way to execute this step of the migration is not as well defined as the prev - If you are building all packages to **verify** that you are able to build them, you most likely want `backstage-cli repo build --all`. The `--all` flag signals that bundled packages like `packages/app` and `packages/backend` should be build as well. Pair this up with a `--since` flag in CI to avoid needing to build all packages. - If you are building all packages to **publish** them, then `backstage-cli repo build` is enough, as it builds all published packages. - If you are building all packages to **deploy** them, you likely don't want to use the `repo` command at all, simply call `yarn build` in the packages you want to deploy instead. For example, if you are deploying the backend with a docker host build, it's enough to call `yarn build` inside `packages/backend`. + +## FAQ + +### Why where packages roles introduced? + +To keep configuration lean, allow for more utilities and tooling, and to enable optimizations in the build system. You can read more about the reasoning in the [original RFC](https://github.com/backstage/backstage/issues/8729). + +### Do I have to migrate to using package roles? + +Short answer - yes. +Longer answer - mostly, you can get around having to declare package the role of your packages by instead explicitly declaring the role in the command invocation or configuration. For example, the `app:build` command will go away, but you can replace it with `package build --role frontend` if you don't want to declare the role in `package.json` . It is however strongly recommended to declare the package roles. + +### I have a package where none of the existing roles apply + +The `web-library`, `node-library` and `common-library` roles are general purpose roles that should cover most use cases. If you feel like none of those roles work for you either, then please open an issue in the [Backstage repo](https://github.com/backstage/backstage) and suggest the addition of a new role. + +### Should I include the role in published packages? + +Yes. While there is nothing that will consume the role at the moment, it is likely that future tooling will be able to provide a better experience for users when published packages include the role. From f9ed42aba091f1b94dad84786801fab9cf1832f4 Mon Sep 17 00:00:00 2001 From: Ben Lambert Date: Tue, 8 Mar 2022 12:49:26 +0000 Subject: [PATCH 16/40] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Ben Lambert Co-authored-by: Fredrik Adelöw --- .../src/components/fields/EntityPicker/EntityPicker.tsx | 2 +- .../src/components/fields/EntityTagsPicker/EntityTagsPicker.tsx | 2 +- .../components/fields/OwnedEntityPicker/OwnedEntityPicker.tsx | 2 +- .../src/components/fields/OwnerPicker/OwnerPicker.tsx | 2 +- .../src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx | 2 +- plugins/scaffolder/src/plugin.ts | 2 +- plugins/scaffolder/src/types.ts | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/plugins/scaffolder/src/components/fields/EntityPicker/EntityPicker.tsx b/plugins/scaffolder/src/components/fields/EntityPicker/EntityPicker.tsx index 17c501d3a1..c899b56683 100644 --- a/plugins/scaffolder/src/components/fields/EntityPicker/EntityPicker.tsx +++ b/plugins/scaffolder/src/components/fields/EntityPicker/EntityPicker.tsx @@ -38,7 +38,7 @@ export interface EntityPickerUiOptions { } /** - * The underling component that is rendered in the form for the `EntityPicker` + * The underlying component that is rendered in the form for the `EntityPicker` * field extension. * * @public diff --git a/plugins/scaffolder/src/components/fields/EntityTagsPicker/EntityTagsPicker.tsx b/plugins/scaffolder/src/components/fields/EntityTagsPicker/EntityTagsPicker.tsx index 582fe57d72..5cf98cb8aa 100644 --- a/plugins/scaffolder/src/components/fields/EntityTagsPicker/EntityTagsPicker.tsx +++ b/plugins/scaffolder/src/components/fields/EntityTagsPicker/EntityTagsPicker.tsx @@ -35,7 +35,7 @@ export interface EntityTagsPickerUiOptions { } /** - * The underling component that is rendered in the form for the `EntityTagsPicker` + * The underlying component that is rendered in the form for the `EntityTagsPicker` * field extension. * * @public diff --git a/plugins/scaffolder/src/components/fields/OwnedEntityPicker/OwnedEntityPicker.tsx b/plugins/scaffolder/src/components/fields/OwnedEntityPicker/OwnedEntityPicker.tsx index 5f314b6748..8e3590e8bf 100644 --- a/plugins/scaffolder/src/components/fields/OwnedEntityPicker/OwnedEntityPicker.tsx +++ b/plugins/scaffolder/src/components/fields/OwnedEntityPicker/OwnedEntityPicker.tsx @@ -40,7 +40,7 @@ export interface OwnedEntityPickerUiOptions { } /** - * The underling component that is rendered in the form for the `OwnedEntityPicker` + * The underlying component that is rendered in the form for the `OwnedEntityPicker` * field extension. * * @public diff --git a/plugins/scaffolder/src/components/fields/OwnerPicker/OwnerPicker.tsx b/plugins/scaffolder/src/components/fields/OwnerPicker/OwnerPicker.tsx index 5f9a05632c..a0c22b7fef 100644 --- a/plugins/scaffolder/src/components/fields/OwnerPicker/OwnerPicker.tsx +++ b/plugins/scaffolder/src/components/fields/OwnerPicker/OwnerPicker.tsx @@ -28,7 +28,7 @@ export interface OwnerPickerUiOptions { } /** - * The underling component that is rendered in the form for the `OwnerPicker` + * The underlying component that is rendered in the form for the `OwnerPicker` * field extension. * * @public diff --git a/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx b/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx index 5af0295d55..fbb7bcd821 100644 --- a/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx +++ b/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx @@ -51,7 +51,7 @@ export interface RepoUrlPickerUiOptions { } /** - * The underling component that is rendered in the form for the `RepoUrlPicker` + * The underlying component that is rendered in the form for the `RepoUrlPicker` * field extension. * * @public diff --git a/plugins/scaffolder/src/plugin.ts b/plugins/scaffolder/src/plugin.ts index eb8c0743a4..ffa0efaaa0 100644 --- a/plugins/scaffolder/src/plugin.ts +++ b/plugins/scaffolder/src/plugin.ts @@ -104,7 +104,7 @@ export const RepoUrlPickerFieldExtension = scaffolderPlugin.provide( ); /** - * A field extensions for picking users and groups out of the Catalog. + * A field extension for picking users and groups out of the Catalog. * * @public */ diff --git a/plugins/scaffolder/src/types.ts b/plugins/scaffolder/src/types.ts index 8a67abeedb..a542a8a473 100644 --- a/plugins/scaffolder/src/types.ts +++ b/plugins/scaffolder/src/types.ts @@ -123,7 +123,7 @@ export interface ScaffolderScaffoldResponse { } /** - * The arguments for `getIntergationsList`. + * The arguments for `getIntegrationsList`. * * @public */ From a610a4e0f9c974decea3d685b85355dd4306505f Mon Sep 17 00:00:00 2001 From: Ben Lambert Date: Tue, 8 Mar 2022 12:49:46 +0000 Subject: [PATCH 17/40] Update plugins/scaffolder/src/components/Router.tsx MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Signed-off-by: blam --- plugins/scaffolder/src/components/Router.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/scaffolder/src/components/Router.tsx b/plugins/scaffolder/src/components/Router.tsx index 3c906f21d4..261556d638 100644 --- a/plugins/scaffolder/src/components/Router.tsx +++ b/plugins/scaffolder/src/components/Router.tsx @@ -51,7 +51,7 @@ export type RouterProps = { }; /** - * The main entirypoint `Router` for the `ScaffolderPlugin`. + * The main entrypoint `Router` for the `ScaffolderPlugin`. * * @public */ From 4431319f4531f9a51a932152313a237c7b7685e8 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 8 Mar 2022 14:09:33 +0100 Subject: [PATCH 18/40] docs: add package roles sections to build system docs Signed-off-by: Patrik Oldsberg --- docs/local-dev/cli-build-system.md | 47 ++++++++++++++++++++++++ docs/tutorials/package-role-migration.md | 2 +- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/docs/local-dev/cli-build-system.md b/docs/local-dev/cli-build-system.md index aed3ba70bc..eef0a68af0 100644 --- a/docs/local-dev/cli-build-system.md +++ b/docs/local-dev/cli-build-system.md @@ -72,6 +72,53 @@ or IDE that has support for formatting, linting, and type checking. Let's dive into a detailed look at each of these steps and how they are implemented in a typical Backstage app. +## Package Roles + +> Package roles were introduced in March 2022. To migrate existing projects, see the [migration guide](../tutorials/package-role-migration.md). + +The Backstage build system uses the concept of package roles in order to help keep +configuration lean, provide utility and tooling, and enable optimizations. A package +role is a single string that identifies what the purpose of a package is, and it's +define in the `package.json` of each package like this: + +```json +{ + "name": "my-package", + "backstage": { + "role": "" + }, + ... +} +``` + +These are the available roles that are currently supported by the Backstage build system: + +| Role | Description | Example | +| ---------------------- | -------------------------------------------- | -------------------------------------------- | +| frontend | Bundled frontend application | `package/app` | +| backend | Bundled backend application | `packages/backend` | +| cli | Package used as a command-line interface | `@backstage/cli`, `@backstage/codemods` | +| web-library | Web library for use by other packages | `@backstage/plugin-catalog-react` | +| node-library | Node.js library for use by other packages | `@backstage/plugin-techdocs-node` | +| common-library | Isomorphic library for use by other packages | `@backstage/plugin-permission-common` | +| frontend-plugin | Backstage frontend plugin | `@backstage/plugin-scaffolder` | +| frontend-plugin-module | Backstage frontend plugin module | `@backstage/plugin-analytics-module-ga` | +| backend-plugin | Backstage backend plugin | `@backstage/plugin-auth-backend` | +| backend-plugin-module | Backstage backend plugin module | `@backstage/plugin-search-backend-module-pg` | + +Most of the steps that we cover below have an accompanying command that is intended to be used as a package script. The commands are all available under the `backstage-cli package` category, and many of the commands will behave differently depending on the role of the package. The commands are intended to be used like this: + +```json +{ + "scripts": { + "start": "backstage-cli package start", + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + ... + } +} +``` + ## Formatting The formatting setup lives completely within each Backstage application and is diff --git a/docs/tutorials/package-role-migration.md b/docs/tutorials/package-role-migration.md index 2aee853673..bdc50fc54d 100644 --- a/docs/tutorials/package-role-migration.md +++ b/docs/tutorials/package-role-migration.md @@ -60,7 +60,7 @@ based on existing information like what build scripts are in place and the packa This automatic detection is not perfect, so it recommended to manually review the roles that were assigned to each package. -You can use the [package role definitions](./not-found#TODO) as a reference. +You can use the [package role definitions](../local-dev/cli-build-system#package-roles) as a reference. ### Step 2 - Migrate package scripts From 8a5ff40dcd1f82453071d2f8830dfb9af43dcabe Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 8 Mar 2022 14:11:38 +0100 Subject: [PATCH 19/40] docs: update lint section in build system docs Signed-off-by: Patrik Oldsberg --- docs/local-dev/cli-build-system.md | 35 ++++++++++++++++++++++-- docs/tutorials/package-role-migration.md | 21 +------------- 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/docs/local-dev/cli-build-system.md b/docs/local-dev/cli-build-system.md index eef0a68af0..4b8d2410dc 100644 --- a/docs/local-dev/cli-build-system.md +++ b/docs/local-dev/cli-build-system.md @@ -143,9 +143,38 @@ configurations in turn build on top of the lint rules from In a standard Backstage setup, each individual package has its own lint configuration, along with a root configuration that applies to the entire -project. Each configuration is initially one that simply extends a base -configuration provided by the Backstage CLI, but they can be customized to fit -the needs of each package. +project. The configuration in each package starts out as a standard configuration +that is determined based on the package role, but it can be customized to fit the needs of each package. + +A minimal `.eslintrc.js` configuration now looks like this: + +```js +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); +``` + +But you can provide custom overrides for each package using the optional second argument: + +```js +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname, { + ignorePatterns: ['templates/'], + rules: { + 'jest/expect-expect': 'off', + }, +}); +``` + +The configuration factory also provides utilities for extending the configuration in ways that are otherwise very cumbersome to do with plain ESLint, particularly for rules like `no-restricted-syntax`. These are the extra keys that are available: + +| Key | Description | +| ----------------------- | ------------------------------------------------------------------ | +| `tsRules` | Additional rules to apply to TypeScript files | +| `testRules` | Additional rules to apply to tests files | +| `restrictedImports` | Additional paths to add to `no-restricted-imports` | +| `restrictedSrcImports` | Additional paths to add to `no-restricted-imports` in src files | +| `restrictedTestImports` | Additional paths to add to `no-restricted-imports` in test files | +| `restrictedSyntax` | Additional patterns to add to `no-restricted-syntax` | +| `restrictedSrcSyntax` | Additional patterns to add to `no-restricted-syntax` in src files | +| `restrictedTestSyntax` | Additional patterns to add to `no-restricted-syntax` in test files | ## Type Checking diff --git a/docs/tutorials/package-role-migration.md b/docs/tutorials/package-role-migration.md index bdc50fc54d..d9d9c8c91a 100644 --- a/docs/tutorials/package-role-migration.md +++ b/docs/tutorials/package-role-migration.md @@ -90,26 +90,7 @@ If you in the end do not want to use this exact script setup, it is still recomm ### Step 3 - Migrate package ESLint configurations -An area that has been simplified as part of the move to package roles is the ESLint configuration. Rather than having each package select which configuration they want (and getting it wrong), they now use a shared configuration factory that utilizes the package role. - -A minimal `.eslintrc.js` configuration now looks like this: - -```js -module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); -``` - -You can provide custom overrides for each package using the optional second argument: - -```js -module.exports = require('@backstage/cli/config/eslint-factory')(__dirname, { - ignorePatterns: ['templates/'], - rules: { - 'jest/expect-expect': 'off', - }, -}); -``` - -The configuration factory also provides utilities for extending the configuration in ways that are otherwise very cumbersome to do with plain ESLint, particularly for rules like `no-restricted-syntax`. You can read more about that in the [build system documentation](./not-found#TODO). +An area that has been simplified as part of the move to package roles is the ESLint configuration. Rather than having each package select which configuration they want (and getting it wrong), they now use a shared configuration factory that utilizes the package role. You can read more about the new configuration setup in the [build system documentation](../local-dev/cli-build-system#linting). To migrate the ESLint configuration of all packages in your project, run the following command: From 7efe6e84cb8bfcc2c65243d967c480c8424d3ab4 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 8 Mar 2022 14:11:57 +0100 Subject: [PATCH 20/40] docs: update build system docs to use roles Signed-off-by: Patrik Oldsberg --- docs/local-dev/cli-build-system.md | 35 ++++++++++++++---------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/docs/local-dev/cli-build-system.md b/docs/local-dev/cli-build-system.md index 4b8d2410dc..91767d3b60 100644 --- a/docs/local-dev/cli-build-system.md +++ b/docs/local-dev/cli-build-system.md @@ -243,11 +243,8 @@ nevertheless be useful to know how it works, since all of the published Backstage packages are built using this process. The build is currently using [Rollup](https://rollupjs.org/) and executes in -isolation for each individual package. There are currently three different -commands in the Backstage CLI that invokes the build process, `plugin:build`, -`backend:build`, and simply `build`. The two former are pre-configured commands -for frontend and backend plugins, while the `build` command provides more -control over the output. +isolation for each individual package. The build is invoked using the `package build` +command, and applies to all packages roles except the bundled ones, `frontend` and `backend`. There are three different possible outputs of the build process: JavaScript in CommonJS module format, JavaScript in ECMAScript module format, and type @@ -281,11 +278,11 @@ cover each combination of these cases separately. ### Frontend Development -There are two different commands that start the frontend development bundling: -`app:serve`, which serves an app and uses `src/index` as the entrypoint, and -`plugin:serve`, which serves a plugin and uses `dev/index` as the entrypoint. -These are typically invoked via the `yarn start` script, and are intended for -local development only. When running the bundle command, a development server +The frontend development setup is used for all packages with a frontend role, and +is invoked using the `package start` command. +The only difference between the different roles is that packages with the `'frontend'` +role use `src/index` as the entrypoint, while other roles instead use `dev/index`. +When running the start command, a development server will be set up that listens to the protocol, host and port set by `app.baseUrl` in the configuration. If needed it is also possible to override the listening options through the `app.listen` configuration. @@ -309,8 +306,8 @@ support for them instead. ### Frontend Production The frontend production bundling creates your typical web content bundle, all -contained within a single folder, ready for static serving. It is invoked using -the `app:build` command, and unlike the development bundling there is no way to +contained within a single folder, ready for static serving. It is used when building +packages with the `'frontend'` role, and unlike the development bundling there is no way to build a production bundle of an individual plugin. The output of the bundling process is written to the `dist` folder in the package. @@ -405,7 +402,7 @@ dependencies installed, and as soon as you copy over and extract the contents of the `bundle.tar.gz` archive on top of it, the backend will be ready to run. The following is an example of a `Dockerfile` that can be used to package the -output of `backstage-cli backend:bundle` into an image: +output of building a package with role `'backend'` into an image: ```Dockerfile FROM node:16-bullseye-slim @@ -617,12 +614,12 @@ The following is an excerpt of a typical setup of an isomorphic library package: "types": "dist/index.d.ts" }, "scripts": { - "build": "backstage-cli build", - "lint": "backstage-cli lint", - "test": "backstage-cli test", - "prepack": "backstage-cli prepack", - "postpack": "backstage-cli postpack", - "clean": "backstage-cli clean" + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "clean": "backstage-cli package clean", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack" }, "files": ["dist"], ``` From d2ecde959b9f4d550b6b10adb821ecc36bf0821b Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 8 Mar 2022 14:17:04 +0100 Subject: [PATCH 21/40] cli: mark package role command categories as stable Signed-off-by: Patrik Oldsberg --- .changeset/pretty-vans-unite.md | 5 +++++ packages/cli/src/commands/index.ts | 24 ++++++++---------------- 2 files changed, 13 insertions(+), 16 deletions(-) create mode 100644 .changeset/pretty-vans-unite.md diff --git a/.changeset/pretty-vans-unite.md b/.changeset/pretty-vans-unite.md new file mode 100644 index 0000000000..6efb6a57ec --- /dev/null +++ b/.changeset/pretty-vans-unite.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': patch +--- + +The new `package`, `repo`, and `migrate` command categories are now marked as stable. These are tied to the use of package roles, which we now encourage you to use. Please check out the [migration guide](https://backstage.io/docs/tutorials/package-role-migration). diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index a79f9fe81b..9b272fab95 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -27,10 +27,8 @@ const configOption = [ export function registerRepoCommand(program: CommanderStatic) { const command = program - .command('repo [command]', { hidden: true }) - .description( - 'Command that run across an entire Backstage project [EXPERIMENTAL]', - ); + .command('repo [command]') + .description('Command that run across an entire Backstage project'); command .command('build') @@ -65,17 +63,14 @@ export function registerRepoCommand(program: CommanderStatic) { export function registerScriptCommand(program: CommanderStatic) { const command = program - .command('package [command]', { hidden: true }) - .description('Lifecycle scripts for individual packages [EXPERIMENTAL]'); + .command('package [command]') + .description('Lifecycle scripts for individual packages'); command .command('start') .description('Start a package for local development') .option(...configOption) - .option( - '--role ', - 'Run the command with an explicit package role [EXPERIMENTAL]', - ) + .option('--role ', 'Run the command with an explicit package role') .option('--check', 'Enable type checking and linting if available') .option('--inspect', 'Enable debugger in Node.js environments') .option( @@ -87,10 +82,7 @@ export function registerScriptCommand(program: CommanderStatic) { command .command('build') .description('Build a package for production deployment or publishing') - .option( - '--role ', - 'Run the command with an explicit package role [EXPERIMENTAL]', - ) + .option('--role ', 'Run the command with an explicit package role') .option( '--minify', 'Minify the generated code. Does not apply to app or backend packages.', @@ -151,8 +143,8 @@ export function registerScriptCommand(program: CommanderStatic) { export function registerMigrateCommand(program: CommanderStatic) { const command = program - .command('migrate [command]', { hidden: true }) - .description('Migration utilities [EXPERIMENTAL]'); + .command('migrate [command]') + .description('Migration utilities'); command .command('package-roles') From 592ac178ec895020fc12a6fd473c3d442bde28ad Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 8 Mar 2022 14:26:03 +0100 Subject: [PATCH 22/40] chore: updating documentation and removing the last of v1beta2 Signed-off-by: blam --- .../writing-custom-field-extensions.md | 2 +- .../README.md | 31 ++++++------- .../scaffolder-backend-module-rails/README.md | 45 ++++++++++--------- .../README.md | 27 +++++------ .../bitbucket-demo/template.yaml | 18 ++++---- .../sample-templates/remote-templates.yaml | 1 - 6 files changed, 63 insertions(+), 61 deletions(-) diff --git a/docs/features/software-templates/writing-custom-field-extensions.md b/docs/features/software-templates/writing-custom-field-extensions.md index fab2e1e2cb..d5d4afbff2 100644 --- a/docs/features/software-templates/writing-custom-field-extensions.md +++ b/docs/features/software-templates/writing-custom-field-extensions.md @@ -141,7 +141,7 @@ Once it's been passed to the `ScaffolderPage` you should now be able to use the Something like this: ```yaml -apiVersion: backstage.io/v1beta2 +apiVersion: scaffolder.backstage.io/v1beta3 kind: Template metadata: name: Test template diff --git a/plugins/scaffolder-backend-module-cookiecutter/README.md b/plugins/scaffolder-backend-module-cookiecutter/README.md index 77ae93c5f3..5c337fda0a 100644 --- a/plugins/scaffolder-backend-module-cookiecutter/README.md +++ b/plugins/scaffolder-backend-module-cookiecutter/README.md @@ -44,7 +44,7 @@ return await createRouter({ After that you can use the action in your template: ```yaml -apiVersion: backstage.io/v1beta2 +apiVersion: scaffolder.backstage.io/v1beta3 kind: Template metadata: name: cookiecutter-demo @@ -109,30 +109,31 @@ spec: input: url: ./template values: - name: '{{ parameters.name }}' - owner: '{{ parameters.owner }}' - system: '{{ parameters.system }}' - destination: '{{ parseRepoUrl parameters.repoUrl }}' + name: ${{ parameters.name }} + owner: ${{ parameters.owner }} + system: ${{ parameters.system }} + destination: ${{ parameters.repoUrl | parseRepoUrl }} - id: publish - if: '{{ not parameters.dryRun }}' + if: ${{ parameters.dryRun !== true }} name: Publish action: publish:github input: - allowedHosts: ['github.com'] - description: 'This is {{ parameters.name }}' - repoUrl: '{{ parameters.repoUrl }}' + allowedHosts: + - github.com + description: This is {{ parameters.name }} + repoUrl: ${{ parameters.repoUrl }} - id: register - if: '{{ not parameters.dryRun }}' + if: ${{ parameters.dryRun !== true }} name: Register action: catalog:register input: - repoContentsUrl: '{{ steps.publish.output.repoContentsUrl }}' + repoContentsUrl: ${{ steps.publish.output.repoContentsUrl }} catalogInfoPath: '/catalog-info.yaml' - name: Results - if: '{{ parameters.dryRun }}' + if: ${{ parameters.dryRun }} action: debug:log input: listWorkspace: true @@ -140,10 +141,10 @@ spec: output: links: - title: Repository - url: '{{ steps.publish.output.remoteUrl }}' + url: ${{ steps.publish.output.remoteUrl }} - title: Open in catalog - icon: 'catalog' - entityRef: '{{ steps.register.output.entityRef }}' + icon: catalog + entityRef: ${{ steps.register.output.entityRef }} ``` You can also visit the `/create/actions` route in your Backstage application to find out more about the parameters this action accepts when it's installed to configure how you like. diff --git a/plugins/scaffolder-backend-module-rails/README.md b/plugins/scaffolder-backend-module-rails/README.md index 4ac52be941..1afaea2a1b 100644 --- a/plugins/scaffolder-backend-module-rails/README.md +++ b/plugins/scaffolder-backend-module-rails/README.md @@ -45,7 +45,7 @@ return await createRouter({ After that you can use the action in your template: ```yaml -apiVersion: backstage.io/v1beta2 +apiVersion: scaffolder.backstage.io/v1beta3 kind: Template metadata: name: rails-demo @@ -171,10 +171,10 @@ spec: input: url: ./template values: - name: '{{ parameters.name }}' - owner: '{{ parameters.owner }}' - system: '{{ parameters.system }}' - railsArguments: '{{ json parameters.railsArguments }}' + name: ${{ parameters.name }} + owner: ${{ parameters.owner }} + system: ${{ parameters.system }} + railsArguments: ${{ parameters.railsArguments }} - name: Write Catalog information action: catalog:write @@ -183,33 +183,34 @@ spec: apiVersion: 'backstage.io/v1alpha1' kind: Component metadata: - name: '{{ parameters.name }}' + name: ${{ parameters.name }} annotations: - github.com/project-slug: '{{ projectSlug parameters.repoUrl }}' + github.com/project-slug: ${{ parameters.repoUrl | projectSlug }} spec: type: service lifecycle: production - owner: '{{ parameters.owner }}' + owner: ${{ parameters.owner }} - id: publish - if: '{{ not parameters.dryRun }}' + if: ${{ parameters.dryRun !== true }} name: Publish action: publish:github input: - allowedHosts: ['github.com'] - description: 'This is {{ parameters.name }}' - repoUrl: '{{ parameters.repoUrl }}' + allowedHosts: + - github.com + description: This is {{ parameters.name }} + repoUrl: ${{ parameters.repoUrl }} - id: register - if: '{{ not parameters.dryRun }}' + if: ${{ parameters.dryRun !== true }} name: Register action: catalog:register input: - repoContentsUrl: '{{ steps.publish.output.repoContentsUrl }}' + repoContentsUrl: ${{ steps.publish.output.repoContentsUrl }} catalogInfoPath: '/catalog-info.yaml' - name: Results - if: '{{ parameters.dryRun }}' + if: ${{ parameters.dryRun }} action: debug:log input: listWorkspace: true @@ -217,10 +218,10 @@ spec: output: links: - title: Repository - url: '{{ steps.publish.output.remoteUrl }}' + url: ${{ steps.publish.output.remoteUrl }} - title: Open in catalog - icon: 'catalog' - entityRef: '{{ steps.register.output.entityRef }}' + icon: catalog + entityRef: ${{ steps.register.output.entityRef }} ``` ### What you need to run that action @@ -240,8 +241,8 @@ steps: url: ./template imageName: repository/rails:tag values: - name: '{{ parameters.name }}' - owner: '{{ parameters.owner }}' - system: '{{ parameters.system }}' - railsArguments: '{{ json parameters.railsArguments }}' + name: ${{ parameters.name }} + owner: ${{ parameters.owner }} + system: ${{ parameters.system }} + railsArguments: ${{ parameters.railsArguments }} ``` diff --git a/plugins/scaffolder-backend-module-yeoman/README.md b/plugins/scaffolder-backend-module-yeoman/README.md index bacbb87648..4cc57eac4a 100644 --- a/plugins/scaffolder-backend-module-yeoman/README.md +++ b/plugins/scaffolder-backend-module-yeoman/README.md @@ -44,7 +44,7 @@ return await createRouter({ After that you can use the action in your template: ```yaml -apiVersion: backstage.io/v1beta2 +apiVersion: scaffolder.backstage.io/v1beta3 kind: Template metadata: name: yeoman-demo @@ -107,29 +107,30 @@ spec: name: Yeoman action: run:yeoman input: - namespace: 'org:codeowners' + namespace: org:codeowners options: - codeowners: '@{{ parameters.owner }}' + codeowners: '@${{ parameters.owner }}' - id: publish - if: '{{ not parameters.dryRun }}' + if: ${{ parameters.dryRun !== true }} name: Publish action: publish:github input: - allowedHosts: ['github.com'] - description: 'This is {{ parameters.name }}' - repoUrl: '{{ parameters.repoUrl }}' + allowedHosts: + - github.com + description: This is ${{ parameters.name }} + repoUrl: ${{ parameters.repoUrl }} - id: register - if: '{{ not parameters.dryRun }}' + if: ${{ parameters.dryRun !== true }} name: Register action: catalog:register input: - repoContentsUrl: '{{ steps.publish.output.repoContentsUrl }}' + repoContentsUrl: ${{ steps.publish.output.repoContentsUrl }} catalogInfoPath: '/catalog-info.yaml' - name: Results - if: '{{ parameters.dryRun }}' + if: ${{ parameters.dryRun }} action: debug:log input: listWorkspace: true @@ -137,10 +138,10 @@ spec: output: links: - title: Repository - url: '{{ steps.publish.output.remoteUrl }}' + url: ${{ steps.publish.output.remoteUrl }} - title: Open in catalog - icon: 'catalog' - entityRef: '{{ steps.register.output.entityRef }}' + icon: catalog + entityRef: ${{ steps.register.output.entityRef }} ``` You can also visit the `/create/actions` route in your Backstage application to find out more about the parameters this action accepts when it's installed to configure how you like. diff --git a/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml b/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml index 87f4d449f5..b3b51d88eb 100644 --- a/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml +++ b/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml @@ -1,9 +1,9 @@ -apiVersion: backstage.io/v1beta2 +apiVersion: scaffolder.backstage.io/v1beta3 kind: Template metadata: name: bitbucket-demo title: Test Bitbucket RepoUrlPicker template - description: scaffolder v1beta2 template demo publishing to bitbucket + description: scaffolder v1beta3 template demo publishing to bitbucket spec: owner: backstage/techdocs-core type: service @@ -49,8 +49,8 @@ spec: input: url: ./template values: - name: '{{ parameters.name }}' - owner: '{{ parameters.owner }}' + name: ${{ parameters.name }} + owner: ${{ parameters.owner }} - id: fetch-docs name: Fetch Docs @@ -63,16 +63,16 @@ spec: name: Publish action: publish:bitbucket input: - description: 'This is {{ parameters.name }}' - repoUrl: '{{ parameters.repoUrl }}' + description: This is ${{ parameters.name }} + repoUrl: ${{ parameters.repoUrl }} - id: register name: Register action: catalog:register input: - repoContentsUrl: '{{ steps.publish.output.repoContentsUrl }}' + repoContentsUrl: ${{ steps.publish.output.repoContentsUrl }} catalogInfoPath: '/catalog-info.yaml' output: - remoteUrl: '{{ steps.publish.output.remoteUrl }}' - entityRef: '{{ steps.register.output.entityRef }}' + remoteUrl: ${{ steps.publish.output.remoteUrl }} + entityRef: ${{ steps.register.output.entityRef }} diff --git a/plugins/scaffolder-backend/sample-templates/remote-templates.yaml b/plugins/scaffolder-backend/sample-templates/remote-templates.yaml index c5b2f1efea..a812c9ce52 100644 --- a/plugins/scaffolder-backend/sample-templates/remote-templates.yaml +++ b/plugins/scaffolder-backend/sample-templates/remote-templates.yaml @@ -11,4 +11,3 @@ spec: - https://github.com/backstage/software-templates/blob/main/scaffolder-templates/pull-request/template.yaml - https://github.com/backstage/software-templates/blob/main/scaffolder-templates/react-ssr-template/template.yaml - https://github.com/backstage/software-templates/blob/main/scaffolder-templates/springboot-grpc-template/template.yaml - - https://github.com/backstage/software-templates/blob/main/scaffolder-templates/v1beta2-demo/template.yaml From 8122e2771775894507046e7c7264c383ee50a704 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 8 Mar 2022 14:27:08 +0100 Subject: [PATCH 23/40] chore: added changeset Signed-off-by: blam --- .changeset/big-meals-fly.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .changeset/big-meals-fly.md diff --git a/.changeset/big-meals-fly.md b/.changeset/big-meals-fly.md new file mode 100644 index 0000000000..be88400227 --- /dev/null +++ b/.changeset/big-meals-fly.md @@ -0,0 +1,8 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +'@backstage/plugin-scaffolder-backend-module-cookiecutter': patch +'@backstage/plugin-scaffolder-backend-module-rails': patch +'@backstage/plugin-scaffolder-backend-module-yeoman': patch +--- + +Updating documentation for supporting `apiVersion: scaffolder.backstage.io/v1beta3` From 567b14a27b2a52ebf59a357de95a9a5b1d4be5c0 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 8 Mar 2022 14:52:57 +0100 Subject: [PATCH 24/40] docs: update cli command docs Signed-off-by: Patrik Oldsberg --- docs/local-dev/cli-build-system.md | 16 +- docs/local-dev/cli-commands.md | 583 ++++++++--------------------- 2 files changed, 164 insertions(+), 435 deletions(-) diff --git a/docs/local-dev/cli-build-system.md b/docs/local-dev/cli-build-system.md index 91767d3b60..a850673090 100644 --- a/docs/local-dev/cli-build-system.md +++ b/docs/local-dev/cli-build-system.md @@ -326,13 +326,25 @@ correctly from linked in packages, the `ModuleScopePlugin` from [`react-dev-utils`](https://www.npmjs.com/package/react-dev-utils) which makes sure that imports don't reach outside the package, a few fallbacks for some Node.js modules like `'buffer'` and `'events'`, a plugin that writes the -frontend configuration to the bundle as `process.env.APP_CONFIG` and build -information as `process.env.BUILD_INFO`, and lastly minification handled by +frontend configuration to the bundle as `process.env.APP_CONFIG`, and lastly minification handled by [esbuild](https://esbuild.github.io/) using the [`esbuild-loader`](https://npm.im/esbuild-loader). There are of course also a set of loaders configured, which you can read more about in the [loaders](#loaders) and [transpilation](#transpilation) sections. +During the build, the following constants are also set: + +```java +process.env.NODE_ENV = 'production'; +process.env.BUILD_INFO = { + cliVersion: '0.4.0', // The version of the CLI package + gitVersion: 'v0.4.0-86-ge54815618', // output of `git describe --always` + packageVersion: '1.0.5', // The version of the app package itself + timestamp: 1678900000000, // Date.now() when the build started + commit: 'e548156182a973ed4b459e18533afc22c85ffff8', // output of `git rev-parse HEAD` +}; +``` + The output of the bundling process is split into two categories of files with separate caching strategies. The first is a set of generic assets with plain names in the root of the `dist/` folder. You will want to serve these with diff --git a/docs/local-dev/cli-commands.md b/docs/local-dev/cli-commands.md index eff6404e3b..78c311e5cf 100644 --- a/docs/local-dev/cli-commands.md +++ b/docs/local-dev/cli-commands.md @@ -7,251 +7,202 @@ description: Descriptions of all commands available in the CLI. This page lists all commands provided by the Backstage CLI, what they're for, and where to use them. -The documentation for each command begins with specifying its scope, this -indicates where the command should be used by selecting from the following list: - -- `app` - A frontend app package, such as `packages/app`. -- `backend` - A backend package, such as `packages/backend`. -- `frontend-plugin` - A frontend plugin package. -- `backend-plugin` - A backend plugin package. -- `root` - The monorepo root. -- `any` - Any kind of package, but not the repo root. - ## help This command displays a help summary or detailed help screens for each command. -Below is a cleaned up output of `yarn backstage-cli --help`. +Below is a cleaned up output of `yarn backstage-cli --help` ```text -app:build Build an app for a production release -app:serve Serve an app for local development +repo [command] Command that run across an entire Backstage project +package [command] Lifecycle scripts for individual packages +migrate [command] Migration utilities -backend:build Build a backend plugin -backend:bundle Bundle the backend into a deployment archive -backend:build-image Bundles the package into a docker image -backend:dev Start local development server with HMR for the backend +create Open up an interactive guide to creating new things in your app -plugin:build Build a plugin -plugin:diff Diff an existing plugin with the creation template -plugin:serve Serves the dev/ folder of a plugin +config:docs Browse the configuration reference documentation +config:print Print the app configuration for the current package +config:check Validate that the given configuration loads and matches schema +config:schema Dump the app configuration schema -build Build a package for publishing -build-workspace Builds a temporary dist workspace from the provided packages -lint Lint a package -test Run tests, forwarding args to Jest, defaulting to watch mode -clean Delete cache directories +versions:bump Bump Backstage packages to the latest versions +versions:check Check Backstage package versioning -create Open up an interactive guide to creating new things in your app -create-plugin Creates a new plugin in the current repository -remove-plugin Removes plugin in the current repository +build-workspace Builds a temporary dist workspace from the provided packages +create-github-app Create new GitHub App in your organization (experimental) -config:docs Browse the configuration reference documentation -config:print Print the app configuration for the current package -config:check Validate that the given configuration loads and matches schema -config:schema Dump the app configuration schema - -versions:bump Bump Backstage packages to the latest versions -versions:check Check Backstage package versioning - -prepack Prepares a package for packaging before publishing -postpack Restores the changes made by the prepack command - -create-github-app Create new GitHub App in your organization (experimental) - -info Show helpful information for debugging and reporting bugs -help [command] display help for command +info Show helpful information for debugging and reporting bugs +help [command] display help for command ``` -## app:build - -Scope: `app` - -Builds a bundle of static content from the app, which can then be served via any -static web server such as `nginx`, or via the -[`app-backend`](https://www.npmjs.com/package/@backstage/plugin-app-backend) -plugin directly from a Backstage backend instance. - -The command also reads and injects static configuration into the bundle. It is -important to note that when deploying using your own static content hosting -solution, this will be the final configuration used in the frontend unless you -for example hook in configuration loading from the backend. When using the -`nginx` based Dockerfile in this repo along with its included run script, -`APP_CONFIG_` environment variables will be injected into the frontend, and when -serving using the `app-backend` plugin, the configuration is completely injected -from the backend and the configuration at the time of calling this command will -not be used. - -Note that even when injecting configuration at runtime, it is not possible to -change the base path of the app. For example, if you at build time have -`app.baseUrl` set to `http://dev-app.com/my-app`, you can change that to -`https://prod-app.com/my-app`, but not to `https://prod-app.com`, as that would -change the path. - -During the build, the following variables are set: - -```java -process.env.NODE_ENV = 'production'; -process.env.BUILD_INFO = { - cliVersion: '0.4.0', // The version of the CLI package - gitVersion: 'v0.4.0-86-ge54815618', // output of `git describe --always` - packageVersion: '1.0.5', // The version of the app package itself - timestamp: 1678900000000, // Date.now() when the build started - commit: 'e548156182a973ed4b459e18533afc22c85ffff8', // output of `git rev-parse HEAD` -}; -``` - -Some CI environments do not properly report correct resource limits, potentially -leading to errors such as `ENOMEM` during compilation. If you run into this -issue you can limit the parallelization of the build process by setting the -environment variable `BACKSTAGE_CLI_BUILD_PARALLEL`, which is forwarded to the -[`terser-webpack-plugin`](https://github.com/webpack-contrib/terser-webpack-plugin#parallel). -You can set it to `false` or `1` to completely disable parallelization, but -usually a low value such as `2` is enough. +The `package` command category, `yarn backstage-cli package --help` ```text -Usage: backstage-cli app:build +start [options] Start a package for local development +build [options] Build a package for production deployment or publishing +lint [options] Lint a package +test Run tests, forwarding args to Jest, defaulting to watch mode +clean Delete cache directories +prepack Prepares a package for packaging before publishing +postpack Restores the changes made by the prepack command +``` + +The `repo` command category, `yarn backstage-cli repo --help` + +```text +build [options] Build packages in the project, excluding bundled app and backend packages. +lint [options] Lint all packages in the project +``` + +The `migrate` command category, `yarn backstage-cli migrate --help` + +```text +package-roles Add package role field to packages that don't have it +package-scripts Set package scripts according to each package role +package-lint-configs Migrates all packages to use @backstage/cli/config/eslint-factory +``` + +## repo build + +Builds all packages in the project, excluding bundled packages by default, i.e. ones +with the role `'frontend'` or `'backend'`. + +```text +Usage: backstage-cli repo build [options] + +Build packages in the project, excluding bundled app and backend packages. + +Options: + --all Build all packages, including bundled app and backend packages. + --since <ref> Only build packages and their dev dependents that changed since the specified ref +``` + +## repo lint + +Lint all packages in the project. + +```text +Usage: backstage-cli repo lint [options] + +Lint all packages in the project + +Options: + --format <format> Lint report output format (default: "eslint-formatter-friendly") + --since <ref> Only lint packages that changed since the specified ref + --fix Attempt to automatically fix violations +``` + +## package start + +Starts the package for local development. See the frontend and backend development parts in the build system [bundling](./cli-build-system#bundling) section for more details. + +```text +Usage: backstage-cli package start [options] + +Start a package for local development Options: - --stats Write bundle stats to output directory - --lax Do not require environment variables to be set --config <path> Config files to load instead of app-config.yaml (default: []) - -h, --help display help for command + --role <name> Run the command with an explicit package role + --check Enable type checking and linting if available + --inspect Enable debugger in Node.js environments + --inspect-brk Enable debugger in Node.js environments, breaking before code starts ``` -## app:serve +## package build -Scope: `app` - -Serve an app for local development. This starts up a local development server, -using a bundling configuration that is quite similar to that of the `app:build` -command, but with development features such as React Hot Module Replacement, -faster sourcemaps, no minification, etc. - -The static configuration is injected into the frontend, but it does not support -watching, meaning that changes in for example `app-config.yaml` are not -reflected until the serve process is restarted. - -During the build, the following variables are set: - -```java -process.env.NODE_ENV = 'development'; -process.env.BUILD_INFO = { /* See app:build */ }; -``` - -The server listening configuration is controlled through the static -configuration. The `app.baseUrl` determines the listening host and port, as well -as whether HTTPS is used or not. It is also possible to override the listening -host and port if needed by setting `app.listen.host` and `app.listen.port`. +Build an individual package based on its role. See the build system [building](./cli-build-system#building) and [bundling](./cli-build-system#bundling) sections for more details. ```text -Usage: backstage-cli app:serve [options] +Usage: backstage-cli package build [options] + +Build a package for production deployment or publishing Options: - --check Enable type checking and linting - --config <path> Config files to load instead of app-config.yaml (default: []) - -h, --help display help for command + --role <name> Run the command with an explicit package role + --minify Minify the generated code. Does not apply to app or backend packages. + --experimental-type-build Enable experimental type build. Does not apply to app or backend packages. + --skip-build-dependencies Skip the automatic building of local dependencies. Applies to backend packages only. + --stats If bundle stats are available, write them to the output directory. Applies to app packages only. + --config <path> Config files to load instead of app-config.yaml. Applies to app packages only. (default: []) ``` -## backend:build +## package lint -Scope: `backend-plugin` - -This builds a backend package for publishing and use in production. The build -output is written to `dist/`. Be sure to list any additional file that the -package depends on at runtime in the `"files"` field inside `package.json`, a -common example being the `migrations` directory. +Lint a package. In addition to the default `eslint` behavior, this command will +include TypeScript files, treat warnings as errors, and default to linting the +entire directory if no specific files are listed. For more information, see the +build system [linting](./cli-build-system.md#linting) section. ```text -Usage: backstage-cli backend:build [options] +Usage: backstage-cli package lint [options] + +Lint a package Options: - --minify Minify the generated code - -h, --help display help for command + --format <format> Lint report output format (default: "eslint-formatter-friendly") + --fix Attempt to automatically fix violations ``` -## backend:bundle +## package test -Scope: `backend` +Run tests, forwarding all unknown options to Jest, and defaulting to watch mode. +When executing the tests, `process.env.NODE_ENV` will be set to `"test"`. -Bundles the backend into a `dist/bundle.tar.gz` archive. See the -[backend bundling](./cli-build-system.md#backend-production-bundling) build -systems documentation for more details. +This command uses a default Jest configuration that is included in the CLI, +which is set up with similar goals for speed, scale, and working within a +monorepo. The configuration sets the `src` as the root directory, enforces the +`.test.` infix for tests, and uses `src/setupTests.ts` as the test setup +location. The included configuration also supports test execution at the root of +a yarn workspaces monorepo by automatically creating one grouped configuration +that includes all packages that have `backstage-cli test` in their package +`test` script. + +For more information about configuration overrides and editor support, see the [Jest Configuration section](./cli-build-system.md#jest-configuration) in the build system documentation. ```text -Usage: backstage-cli backend:bundle [options] +Usage: backstage-cli package test [options] -Bundle the backend into a deployment archive +Run tests, forwarding args to Jest, defaulting to watch mode Options: - --build-dependencies Build all local package dependencies before bundling the backend - -h, --help display help for command -``` - -## backend:build-image - -Scope: `backend` - -Builds a Docker image of the backend package, forwarding all unknown options to -`docker image build`. For example: - -```bash -yarn backstage-cli backend:build-image --build --tag my-backend-image -``` - -The image is built using the backend package along with all of its local package -dependencies. It expects to find a `Dockerfile` at the root of the backend -package, which will be used during the build. - -The Dockerfile is **NOT** executed within the package or repo itself. Because -the packages in the repo itself are configured for development instead of -production use, the final Docker build happens in a separate temporary -directory, to which the backend package and dependencies have been copied. Only -files listed within the `"files"` field within each package's `package.json` are -copied over, along with the root `package.json`, `yarn.lock`, and any -`app-config.*.yaml` files. - -During the build a `skeleton.tar` file is created and put at the repo root. This -file contains the `package.json` of each included package, which together with -the root `package.json` and `yarn.lock` can be used to run a cached -`yarn install` before the full production builds of all the packages are copied -over, providing a significant speedup if Docker build layer caching available. - -This command is experimental and we hope to be able to replace it with one that -is less integrated directly with Docker, and also supports multi-stage Docker -builds. It is possible to replicate most of what this command does by manually -building each package, and then use the `build-workspace` to create the -temporary workspace, and finally copy over any additional files to the workspace -and execute the Docker build within it. - -```text -Usage: backstage-cli backend:build-image [options] - -Options: - --build Build packages before packing them into the image --backstage-cli-help display help for command ``` -## backend:dev +## package clean -Scope: `backend`, `backend-plugin` - -Starts a backend package in development mode, with watch mode enabled for all -local dependencies. +Remove cache and output directories. ```text -Usage: backstage-cli backend:dev [options] +Usage: backstage-cli package clean [options] -Options: - --check Enable type checking and linting - --inspect Enable debugger - --config <path> Config files to load instead of app-config.yaml (default: []) - -h, --help display help for command +Delete cache directories +``` + +## package prepack + +This command should be added as `scripts.prepack` in all packages. It enables +packaging- and publish-time overrides for fields inside `packages.json`. +For more details, see the build system [publishing](./cli-build-system.md#publishing) section. + +```text +Usage: backstage-cli package prepack [options] + +Prepares a package for packaging before publishing +``` + +## package postpack + +This should be added as `scripts.postpack` in all packages. It restores +`package.json` to what it looked like before calling the `prepack` command. + +```text +Usage: backstage-cli package postpack [options] + +Restores the changes made by the prepack command ``` ## create -Scope: `root` - The `create` command opens up an interactive guide for you to create new things in your app. If you do not pass in any options it is completely interactive, but it is possible to pre-select what you want to create using the `--select` flag, @@ -278,181 +229,16 @@ this: Usage: backstage-cli create [options] Options: - --select Select the thing you want to be creating upfront - --option = Pre-fill options for the creation process (default: []) - --scope The scope to use for new packages - --npm-registry The package registry to use for new packages + --select <name> Select the thing you want to be creating upfront + --option <name>=<value> Pre-fill options for the creation process (default: []) + --scope <scope> The scope to use for new packages + --npm-registry <URL> The package registry to use for new packages --no-private Do not mark new packages as private -h, --help display help for command ``` -## create-plugin - -Scope: `root` - -Creates a new plugin within the repository. This command is typically wrapped up -in the root `package.json` to be executed with `yarn create-plugin`, using -options that are appropriate for the organization that owns the app repo. A -recommended scope for internal packages is `@internal`. - -```text -Usage: backstage-cli create-plugin [options] - -Options: - --backend Create plugin with the backend dependencies as default - --scope <scope> npm scope - --npm-registry <URL> npm registry URL - --no-private Public npm package - -h, --help display help for command -``` - -## remove-plugin - -Scope: `root` - -A utility to remove a plugin from a repo, essentially undoing everything that -was done by `create-plugin`. - -This is primarily intended as a utility for manual tests and end to end testing -scripts. - -```text -Usage: backstage-cli remove-plugin [options] - -Options: - -h, --help display help for command -``` - -## plugin:build - -Scope: `frontend-plugin` - -Build a frontend plugin for publishing to a package registry. There is no need -to run this command during development or even in CI unless the package is being -published. The `app:bundle` command does not use the output for this command -when bundling local package dependencies. - -The output is written to a `dist/` folder. It also outputs type declarations for -the plugin, and therefore requires `yarn tsc` to have been run first. The input -type declarations are expected to be found within `dist-types/` at the root of -the monorepo. - -```text -Usage: backstage-cli plugin:build [options] - -Options: - --minify Minify the generated code - -h, --help display help for command -``` - -## plugin:serve - -Scope: `frontend-plugin` - -Serves a frontend plugin by itself for isolated development. The serve task -itself is essentially identical to `app:serve`, but the entrypoint is instead -set to the `dev/` folder within the plugin. - -The `dev/` folder typically contains a small wrapper script that hooks up any -necessary mock APIs or other things that are needed for the plugin to function. -The `@backstage/dev-utils` package provides utilities to that end. - -```text -Usage: backstage-cli plugin:serve [options] - -Options: - --check Enable type checking and linting - --config <path> Config files to load instead of app-config.yaml (default: []) - -h, --help display help for command -``` - -## plugin:diff - -Scope: `frontend-plugin` - -Compares a frontend plugin to the `create-plugin` template, making sure that it -hasn't diverged from the template and recommending updates when it has. A good -practice is to run this command after updating the version of the CLI in a -project. - -```text -Usage: backstage-cli plugin:diff [options] - -Options: - --check Fail if changes are required - --yes Apply all changes - -h, --help display help for command -``` - -## build - -Scope: `any` - -Build a single package for publishing, just like the `plugin:build` and -`backend:build` commands. This command is intended for standalone packages that -aren't plugins, and for example support building of isomorphic packages for -usage in both the frontend and backend. - -For frontend packages you'll want to include `esm` output, and for backend -packages `cjs`. Whether to include `types` depends on if you need type -declarations for the package, and also requires `yarn tsc` to have been run -first. - -```text -Usage: backstage-cli build [options] - -Options: - --outputs <formats> List of formats to output [types,cjs,esm] - --minify Minify the generated code - -h, --help display help for command -``` - -## lint - -Scope: `any` - -Lint a package. In addition to the default `eslint` behavior, this command will -include TypeScript files, treat warnings as errors, and default to linting the -entire directory if no specific files are listed. - -```text -Usage: backstage-cli lint [options] - -Options: - --format <format> Lint report output format (default: "eslint-formatter-friendly") - --fix Attempt to automatically fix violations - -h, --help display help for command -``` - -## test - -Scope: `any` - -Run tests, forwarding all unknown options to Jest, and defaulting to watch mode. -When executing the tests, `process.env.NODE_ENV` will be set to `"test"`. - -This command uses a default Jest configuration that is included in the CLI, -which is set up with similar goals for speed, scale, and working within a -monorepo. The configuration sets the `src` as the root directory, enforces the -`.test.` infix for tests, and uses `src/setupTests.ts` as the test setup -location. The included configuration also supports test execution at the root of -a yarn workspaces monorepo by automatically creating one grouped configuration -that includes all packages that have `backstage-cli test` in their package -`test` script. - -For more information about configuration overrides and editor support, see the [Jest Configuration section](./cli-build-system.md#jest-configuration) in the build system documentation. - -```text -Usage: backstage-cli test [options] - -Options: - --backstage-cli-help display help for command -``` - ## config:docs -Scope: `root` - This commands opens up the reference documentation of your apps local configuration schema in the browser. This is useful to get an overview of what configuration values are available to use, a description of what they do and @@ -464,14 +250,12 @@ Usage: backstage-cli config:docs [options] Browse the configuration reference documentation Options: - --package Only include the schema that applies to the given package + --package <name> Only include the schema that applies to the given package -h, --help display help for command ``` ## config:print -Scope: `root` - Print the static configuration, defaulting to reading `app-config.yaml` in the repo root, using schema collected from all local packages in the repo. @@ -497,8 +281,6 @@ Options: ## config:check -Scope: `root` - Validate that static configuration loads and matches schema, defaulting to reading `app-config.yaml` in the repo root and using schema collected from all local packages in the repo. @@ -517,8 +299,6 @@ Options: ## config:schema -Scope: `root` - Dump the configuration schema that was collected from all local packages in the repo. @@ -538,8 +318,6 @@ Options: ## versions:bump -Scope: `root` - Bump all `@backstage` packages to the latest versions. This checks for updates in the package registry, and will update entries both in `yarn.lock` and `package.json` files when necessary. @@ -554,8 +332,6 @@ Options: ## versions:check -Scope: `root` - Validate `@backstage` dependencies within the repo, making sure that there are no duplicates of packages that might lead to breakages. @@ -571,63 +347,8 @@ Options: -h, --help display help for command ``` -## prepack - -Scope: `any` - -This command should be added as `scripts.prepack` in all packages. It enables -packaging- and publish-time overrides for fields inside `packages.json`. - -The checked in version of all packages in a Backstage monorepo are tailored for -local development, and as such `main` and similar fields inside `package.json` -point to development source, i.e. `src/index.ts`. Using this when publishing -would lead to a broken package, since `src/` is not included in the published -package and we instead need to point to files in the `dist/` directory. This -command allows for those fields to be rewritten when needed, and does so by -copying all fields within `publishConfig` to the top-level of each -`package.json`, skipping `access`, `registry`, and `tag`. - -The need for this command may be removed in the future, as this exact method of -overriding fields for publishing is already supported by some package managers. - -```text -Usage: backstage-cli prepack [options] - -Options: - -h, --help display help for command -``` - -## postpack - -Scope: `any` - -This should be added as `scripts.postpack` in all packages. It restores -`package.json` to what it looked like before calling the `prepack` command. - -```text -Usage: backstage-cli postpack [options] - -Options: - -h, --help display help for command -``` - -## clean - -Scope: `any` - -Remove cache and output directories. - -```text -Usage: backstage-cli clean [options] - -Options: - -h, --help display help for command -``` - ## build-workspace -Scope: `any`, `root` - Builds a mirror of the workspace using the packaged production version of each package. This essentially calls `yarn pack` in each included package and unpacks the resulting archive in the target `workspace-dir`. @@ -638,8 +359,6 @@ Usage: backstage-cli build-workspace [options] <workspace-dir> ## create-github-app -Scope: `root` - Creates a GitHub App in your GitHub organization. This is an alternative to token-based [GitHub integration](../integrations/github/locations.md). See [GitHub Apps for Backstage Authentication](../plugins/github-apps.md). @@ -653,8 +372,6 @@ Usage: backstage-cli create-github-app <github-org> ## info -Scope: `root` - Outputs debug information which is useful when opening an issue. Outputs system information, node.js and npm versions, CLI version and type (inside backstage repo or a created app), all `@backstage/*` package dependency versions. From 5f07ecbc8afbef5a83c15dec06dcf04cdeb3d8f9 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 8 Mar 2022 14:54:29 +0100 Subject: [PATCH 25/40] docs: add "package role" to CLI glossary Signed-off-by: Patrik Oldsberg --- docs/local-dev/cli-overview.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/local-dev/cli-overview.md b/docs/local-dev/cli-overview.md index 9e6884aab0..3312926068 100644 --- a/docs/local-dev/cli-overview.md +++ b/docs/local-dev/cli-overview.md @@ -50,3 +50,4 @@ improve the tooling, as well as to more easily keep the system up to date. - **Bundle** - A collection of the deployment artifacts. The output of the bundling process, which brings a collection of packages into a single collection of deployment artifacts. +- **Package Role** - The declared role of a package, see [package roles](./cli-build-system#package-roles). From 55999e82682089891b9d3897820c84bca9c68842 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 8 Mar 2022 15:03:50 +0100 Subject: [PATCH 26/40] chore: reworking entityLinks Signed-off-by: blam --- plugins/scaffolder-backend-module-cookiecutter/README.md | 2 +- plugins/scaffolder-backend-module-rails/README.md | 2 +- .../sample-templates/bitbucket-demo/template.yaml | 8 ++++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/plugins/scaffolder-backend-module-cookiecutter/README.md b/plugins/scaffolder-backend-module-cookiecutter/README.md index 5c337fda0a..f40200ee4d 100644 --- a/plugins/scaffolder-backend-module-cookiecutter/README.md +++ b/plugins/scaffolder-backend-module-cookiecutter/README.md @@ -121,7 +121,7 @@ spec: input: allowedHosts: - github.com - description: This is {{ parameters.name }} + description: This is ${{ parameters.name }} repoUrl: ${{ parameters.repoUrl }} - id: register diff --git a/plugins/scaffolder-backend-module-rails/README.md b/plugins/scaffolder-backend-module-rails/README.md index 1afaea2a1b..6236549c9d 100644 --- a/plugins/scaffolder-backend-module-rails/README.md +++ b/plugins/scaffolder-backend-module-rails/README.md @@ -198,7 +198,7 @@ spec: input: allowedHosts: - github.com - description: This is {{ parameters.name }} + description: This is ${{ parameters.name }} repoUrl: ${{ parameters.repoUrl }} - id: register diff --git a/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml b/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml index b3b51d88eb..8b6ff302ac 100644 --- a/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml +++ b/plugins/scaffolder-backend/sample-templates/bitbucket-demo/template.yaml @@ -74,5 +74,9 @@ spec: catalogInfoPath: '/catalog-info.yaml' output: - remoteUrl: ${{ steps.publish.output.remoteUrl }} - entityRef: ${{ steps.register.output.entityRef }} + links: + - title: Repository + url: ${{ steps.publish.output.remoteUrl }} + - title: Open in catalog + icon: catalog + entityRef: ${{ steps.register.output.entityRef }} From 5625443a265f8bda8a1f8a0e2269b65acd562d3c Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 8 Mar 2022 15:07:41 +0100 Subject: [PATCH 27/40] chore: added a note in the migration sheet Signed-off-by: blam --- .../migrating-from-v1beta2-to-v1beta3.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/features/software-templates/migrating-from-v1beta2-to-v1beta3.md b/docs/features/software-templates/migrating-from-v1beta2-to-v1beta3.md index 559cb036e0..1f771b4eaf 100644 --- a/docs/features/software-templates/migrating-from-v1beta2-to-v1beta3.md +++ b/docs/features/software-templates/migrating-from-v1beta2-to-v1beta3.md @@ -162,6 +162,24 @@ away in future versions and the `RepoUrlPicker` will return an object so `parameters.repoUrl` will already be a `{ host: string; owner: string; repo: string }` 🚀 +## Links should be used instead of named outputs + +Previously, it was possible to provide links to the frontend using the named output `entityRef` and `remoteUrl`. +These should be moved to `links` under the `output` object instead. + +```diff + output: +- remoteUrl: '{{ steps.publish.output.remoteUrl }}' +- entityRef: '{{ steps.register.output.entityRef }}' ++ links: ++ - title: Repository ++ url: ${{ steps.publish.output.remoteUrl }} ++ - title: Open in catalog ++ icon: catalog ++ entityRef: ${{ steps.register.output.entityRef }} + +``` + ### Summary Of course, we're always available on [discord](https://discord.gg/MUpMjP2) if From cd5172a1fb81a73930b5adbac8ed76d2c7a7ce01 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 8 Mar 2022 15:19:32 +0100 Subject: [PATCH 28/40] cli: update templates to use package roles Signed-off-by: Patrik Oldsberg --- .changeset/pretty-vans-unite.md | 6 +++++- .../default-backend-plugin/.eslintrc.js | 4 +--- .../default-backend-plugin/package.json.hbs | 17 ++++++++++------- .../default-common-plugin-package/.eslintrc.js | 4 +--- .../package.json.hbs | 15 +++++++++------ .../cli/templates/default-plugin/.eslintrc.js | 4 +--- .../templates/default-plugin/package.json.hbs | 18 ++++++++++-------- .../templates/scaffolder-module/.eslintrc.js | 4 +--- .../scaffolder-module/package.json.hbs | 16 ++++++++++------ 9 files changed, 48 insertions(+), 40 deletions(-) diff --git a/.changeset/pretty-vans-unite.md b/.changeset/pretty-vans-unite.md index 6efb6a57ec..faf535c053 100644 --- a/.changeset/pretty-vans-unite.md +++ b/.changeset/pretty-vans-unite.md @@ -2,4 +2,8 @@ '@backstage/cli': patch --- -The new `package`, `repo`, and `migrate` command categories are now marked as stable. These are tied to the use of package roles, which we now encourage you to use. Please check out the [migration guide](https://backstage.io/docs/tutorials/package-role-migration). +Package roles are now marked as stable and migration is encouraged. Please check out the [migration guide](https://backstage.io/docs/tutorials/package-role-migration). + +The new `package`, `repo`, and `migrate` command categories are now marked as stable. + +The package templates used by the `create` command have all been updated to use package roles. diff --git a/packages/cli/templates/default-backend-plugin/.eslintrc.js b/packages/cli/templates/default-backend-plugin/.eslintrc.js index 16a033dbc6..e2a53a6ad2 100644 --- a/packages/cli/templates/default-backend-plugin/.eslintrc.js +++ b/packages/cli/templates/default-backend-plugin/.eslintrc.js @@ -1,3 +1 @@ -module.exports = { - extends: [require.resolve('@backstage/cli/config/eslint.backend')], -}; +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/packages/cli/templates/default-backend-plugin/package.json.hbs b/packages/cli/templates/default-backend-plugin/package.json.hbs index 321ca9c546..dacc1d601a 100644 --- a/packages/cli/templates/default-backend-plugin/package.json.hbs +++ b/packages/cli/templates/default-backend-plugin/package.json.hbs @@ -15,14 +15,17 @@ "main": "dist/index.cjs.js", "types": "dist/index.d.ts" }, + "backstage": { + "role": "backend-plugin" + }, "scripts": { - "start": "backstage-cli backend:dev", - "build": "backstage-cli backend:build", - "lint": "backstage-cli lint", - "test": "backstage-cli test", - "prepack": "backstage-cli prepack", - "postpack": "backstage-cli postpack", - "clean": "backstage-cli clean" + "start": "backstage-cli package start", + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "clean": "backstage-cli package clean", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack" }, "dependencies": { "@backstage/backend-common": "{{versionQuery '@backstage/backend-common'}}", diff --git a/packages/cli/templates/default-common-plugin-package/.eslintrc.js b/packages/cli/templates/default-common-plugin-package/.eslintrc.js index 13573efa9c..e2a53a6ad2 100644 --- a/packages/cli/templates/default-common-plugin-package/.eslintrc.js +++ b/packages/cli/templates/default-common-plugin-package/.eslintrc.js @@ -1,3 +1 @@ -module.exports = { - extends: [require.resolve('@backstage/cli/config/eslint')], -}; +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/packages/cli/templates/default-common-plugin-package/package.json.hbs b/packages/cli/templates/default-common-plugin-package/package.json.hbs index efaee496e7..844a951d05 100644 --- a/packages/cli/templates/default-common-plugin-package/package.json.hbs +++ b/packages/cli/templates/default-common-plugin-package/package.json.hbs @@ -17,13 +17,16 @@ "module": "dist/index.esm.js", "types": "dist/index.d.ts" }, + "backstage": { + "role": "common-library" + }, "scripts": { - "build": "backstage-cli build", - "lint": "backstage-cli lint", - "test": "backstage-cli test", - "prepack": "backstage-cli prepack", - "postpack": "backstage-cli postpack", - "clean": "backstage-cli clean" + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "clean": "backstage-cli package clean", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack" }, "devDependencies": { "@backstage/cli": "{{versionQuery '@backstage/cli'}}" diff --git a/packages/cli/templates/default-plugin/.eslintrc.js b/packages/cli/templates/default-plugin/.eslintrc.js index 13573efa9c..e2a53a6ad2 100644 --- a/packages/cli/templates/default-plugin/.eslintrc.js +++ b/packages/cli/templates/default-plugin/.eslintrc.js @@ -1,3 +1 @@ -module.exports = { - extends: [require.resolve('@backstage/cli/config/eslint')], -}; +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/packages/cli/templates/default-plugin/package.json.hbs b/packages/cli/templates/default-plugin/package.json.hbs index d38a31fa7d..f7985b497e 100644 --- a/packages/cli/templates/default-plugin/package.json.hbs +++ b/packages/cli/templates/default-plugin/package.json.hbs @@ -15,15 +15,17 @@ "main": "dist/index.esm.js", "types": "dist/index.d.ts" }, + "backstage": { + "role": "frontend-plugin" + }, "scripts": { - "build": "backstage-cli plugin:build", - "start": "backstage-cli plugin:serve", - "lint": "backstage-cli lint", - "test": "backstage-cli test", - "diff": "backstage-cli plugin:diff", - "prepack": "backstage-cli prepack", - "postpack": "backstage-cli postpack", - "clean": "backstage-cli clean" + "start": "backstage-cli package start", + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "clean": "backstage-cli package clean", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack" }, "dependencies": { "@backstage/core-components": "{{versionQuery '@backstage/core-components'}}", diff --git a/packages/cli/templates/scaffolder-module/.eslintrc.js b/packages/cli/templates/scaffolder-module/.eslintrc.js index 16a033dbc6..e2a53a6ad2 100644 --- a/packages/cli/templates/scaffolder-module/.eslintrc.js +++ b/packages/cli/templates/scaffolder-module/.eslintrc.js @@ -1,3 +1 @@ -module.exports = { - extends: [require.resolve('@backstage/cli/config/eslint.backend')], -}; +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/packages/cli/templates/scaffolder-module/package.json.hbs b/packages/cli/templates/scaffolder-module/package.json.hbs index de6ffd9e3a..618e953b74 100644 --- a/packages/cli/templates/scaffolder-module/package.json.hbs +++ b/packages/cli/templates/scaffolder-module/package.json.hbs @@ -16,13 +16,17 @@ "main": "dist/index.cjs.js", "types": "dist/index.d.ts" }, + "backstage": { + "role": "backend-plugin-module" + }, "scripts": { - "build": "backstage-cli build --output cjs,types", - "lint": "backstage-cli lint", - "test": "backstage-cli test", - "prepack": "backstage-cli prepack", - "postpack": "backstage-cli postpack", - "clean": "backstage-cli clean" + "start": "backstage-cli package start", + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "clean": "backstage-cli package clean", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack" }, "dependencies": { "@backstage/plugin-scaffolder-backend": "{{versionQuery '@backstage/plugin-scaffolder-backend'}}" From bde30664c40b391c3918b37521e50acc01b3745a Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 8 Mar 2022 15:22:43 +0100 Subject: [PATCH 29/40] create-app: migrated to use package roles Signed-off-by: Patrik Oldsberg --- .changeset/twenty-birds-think.md | 19 +++++++++++++++++++ packages/app/package.json | 2 +- packages/backend/package.json | 4 ++-- .../templates/default-app/package.json.hbs | 6 +++--- .../default-app/packages/app/.eslintrc.js | 4 +--- .../default-app/packages/app/package.json.hbs | 13 ++++++++----- .../default-app/packages/backend/.eslintrc.js | 4 +--- .../packages/backend/package.json.hbs | 13 ++++++++----- 8 files changed, 43 insertions(+), 22 deletions(-) create mode 100644 .changeset/twenty-birds-think.md diff --git a/.changeset/twenty-birds-think.md b/.changeset/twenty-birds-think.md new file mode 100644 index 0000000000..5b1f9481c0 --- /dev/null +++ b/.changeset/twenty-birds-think.md @@ -0,0 +1,19 @@ +--- +'@backstage/create-app': patch +--- + +Updated template to use package roles. To apply this change to an existing app, check out the [migration guide](https://backstage.io/docs/tutorials/package-role-migration). + +Specifically the following scripts in the root `package.json` have also been updated: + +```diff +- "build": "lerna run build", ++ "build": "backstage-cli repo build --all", + +... + +- "lint": "lerna run lint --since origin/master --", +- "lint:all": "lerna run lint --", ++ "lint": "backstage-cli repo lint --since origin/master", ++ "lint:all": "backstage-cli repo lint", +``` diff --git a/packages/app/package.json b/packages/app/package.json index 45cde6df52..5ae2a1dd08 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -96,9 +96,9 @@ "build": "backstage-cli package build", "clean": "backstage-cli package clean", "test": "backstage-cli package test", + "lint": "backstage-cli package lint", "test:e2e": "start-server-and-test start http://localhost:3000 cy:dev", "test:e2e:ci": "start-server-and-test start http://localhost:3000 cy:run", - "lint": "backstage-cli package lint", "cy:dev": "cypress open", "cy:run": "cypress run" }, diff --git a/packages/backend/package.json b/packages/backend/package.json index b49882b837..dc07124910 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -18,12 +18,12 @@ "backstage" ], "scripts": { - "build": "backstage-cli package build", - "build-image": "docker build ../.. -f Dockerfile --tag example-backend", "start": "backstage-cli package start", + "build": "backstage-cli package build", "lint": "backstage-cli package lint", "test": "backstage-cli package test", "clean": "backstage-cli package clean", + "build-image": "docker build ../.. -f Dockerfile --tag example-backend", "migrate:create": "knex migrate:make -x ts" }, "dependencies": { diff --git a/packages/create-app/templates/default-app/package.json.hbs b/packages/create-app/templates/default-app/package.json.hbs index e7c39add25..884eaf23cf 100644 --- a/packages/create-app/templates/default-app/package.json.hbs +++ b/packages/create-app/templates/default-app/package.json.hbs @@ -9,7 +9,7 @@ "dev": "concurrently \"yarn start\" \"yarn start-backend\"", "start": "yarn workspace app start", "start-backend": "yarn workspace backend start", - "build": "lerna run build", + "build": "backstage-cli repo build --all", "build-image": "yarn workspace backend build-image", "tsc": "tsc", "tsc:full": "tsc --skipLibCheck false --incremental false", @@ -17,8 +17,8 @@ "diff": "lerna run diff --", "test": "backstage-cli test", "test:all": "lerna run test -- --coverage", - "lint": "lerna run lint --since origin/master --", - "lint:all": "lerna run lint --", + "lint": "backstage-cli repo lint --since origin/master", + "lint:all": "backstage-cli repo lint", "prettier:check": "prettier --check .", "create-plugin": "backstage-cli create-plugin --scope internal", "remove-plugin": "backstage-cli remove-plugin" diff --git a/packages/create-app/templates/default-app/packages/app/.eslintrc.js b/packages/create-app/templates/default-app/packages/app/.eslintrc.js index 13573efa9c..e2a53a6ad2 100644 --- a/packages/create-app/templates/default-app/packages/app/.eslintrc.js +++ b/packages/create-app/templates/default-app/packages/app/.eslintrc.js @@ -1,3 +1 @@ -module.exports = { - extends: [require.resolve('@backstage/cli/config/eslint')], -}; +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/packages/create-app/templates/default-app/packages/app/package.json.hbs b/packages/create-app/templates/default-app/packages/app/package.json.hbs index 53e0590db4..71d2ac9b3b 100644 --- a/packages/create-app/templates/default-app/packages/app/package.json.hbs +++ b/packages/create-app/templates/default-app/packages/app/package.json.hbs @@ -3,6 +3,9 @@ "version": "0.0.0", "private": true, "bundled": true, + "backstage": { + "role": "frontend" + }, "dependencies": { "@backstage/app-defaults": "^{{version '@backstage/app-defaults'}}", "@backstage/catalog-model": "^{{version '@backstage/catalog-model'}}", @@ -49,13 +52,13 @@ "start-server-and-test": "^1.10.11" }, "scripts": { - "start": "backstage-cli app:serve", - "build": "backstage-cli app:build", - "clean": "backstage-cli clean", - "test": "backstage-cli test", + "start": "backstage-cli package start", + "build": "backstage-cli package build", + "clean": "backstage-cli package clean", + "test": "backstage-cli package test", + "lint": "backstage-cli package lint", "test:e2e": "cross-env PORT=3001 start-server-and-test start http://localhost:3001 cy:dev", "test:e2e:ci": "cross-env PORT=3001 start-server-and-test start http://localhost:3001 cy:run", - "lint": "backstage-cli lint", "cy:dev": "cypress open", "cy:run": "cypress run" }, diff --git a/packages/create-app/templates/default-app/packages/backend/.eslintrc.js b/packages/create-app/templates/default-app/packages/backend/.eslintrc.js index 16a033dbc6..e2a53a6ad2 100644 --- a/packages/create-app/templates/default-app/packages/backend/.eslintrc.js +++ b/packages/create-app/templates/default-app/packages/backend/.eslintrc.js @@ -1,3 +1 @@ -module.exports = { - extends: [require.resolve('@backstage/cli/config/eslint.backend')], -}; +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/packages/create-app/templates/default-app/packages/backend/package.json.hbs b/packages/create-app/templates/default-app/packages/backend/package.json.hbs index feba5169ba..79d6772d7e 100644 --- a/packages/create-app/templates/default-app/packages/backend/package.json.hbs +++ b/packages/create-app/templates/default-app/packages/backend/package.json.hbs @@ -4,13 +4,16 @@ "main": "dist/index.cjs.js", "types": "src/index.ts", "private": true, + "backstage": { + "role": "backend" + }, "scripts": { - "build": "backstage-cli backend:bundle", + "start": "backstage-cli package start", + "build": "backstage-cli package build", + "lint": "backstage-cli package lint", + "test": "backstage-cli package test", + "clean": "backstage-cli package clean", "build-image": "docker build ../.. -f Dockerfile --tag backstage", - "start": "backstage-cli backend:dev", - "lint": "backstage-cli lint", - "test": "backstage-cli test", - "clean": "backstage-cli clean", "migrate:create": "knex migrate:make -x ts" }, "dependencies": { From 9d7fc090ac0263ea5440f9e702565f58637b74d0 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 8 Mar 2022 15:35:23 +0100 Subject: [PATCH 30/40] docs: package role migration guide tweaks Signed-off-by: Patrik Oldsberg --- docs/tutorials/package-role-migration.md | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/docs/tutorials/package-role-migration.md b/docs/tutorials/package-role-migration.md index d9d9c8c91a..a27dbb2e70 100644 --- a/docs/tutorials/package-role-migration.md +++ b/docs/tutorials/package-role-migration.md @@ -5,27 +5,26 @@ description: Guide for how to migrate packages to use the new role utility --- The Backstage CLI has introduced the concept of package roles, whose purpose is to -enable more powerful tooling and leaner package configuration. More background and -information about the change can be found in the [original RFC](https://github.com/backstage/backstage/issues/8729). +enable more powerful tooling, optimizations, and leaner package configuration. More background and +information about the change can be found in the [original RFC](https://github.com/backstage/backstage/issues/8729) and the [FAQ](#faq) on this page. Package roles are implemented through a well-known `"backstage"."role"` field in the `package.json` of each package. There are a handful of roles defined so far, and it -is not possible to use value outside the set of predefined roles. Some examples of -these roles are `frontend-plugin`, `node-library`, and `backend-plugin-module`. +is not possible to use values outside the [set of predefined roles](../local-dev/cli-build-system#package-roles). With roles in place in all packages, the Backstage CLI is able to automatically determine how to handle each package. For example, the different build commands have been replaced by a single one that instead knows how to build each role. The test and lint configurations are also selected automatically based on the role, and a new category of `repo` commands have been introduced in the CLI, which are able -to operate across all packages at once. +to operate across all packages simultaneously. Package roles have been used in the Backstage main repository for a while, and we now recommend that all Backstage projects are migrated to use package roles. ## Migration -In order to make the migration as smooth as possible, `@backstage/cli` provides +In order to make the migration as smooth as possible `@backstage/cli` provides a number of migration utilities. Using these in combination with some manual review and optional steps should be all you need to migrate to package roles in most projects. @@ -48,24 +47,20 @@ Have a look at the new commands under `yarn backstage-cli repo`, and switch to t ### Step 1 - Add package roles -The first step is to add the `"backstage"."role"` field to each package. This -is done by running the following command: +The first step is to add the `"backstage"."role"` field to each package. This can of course be done manually, but the following command will attempt to automatically detect the role of each package in your project: ```sh yarn backstage-cli migrate package-roles ``` -This will add the role field to each package in your project, detecting the role -based on existing information like what build scripts are in place and the package name. - -This automatic detection is not perfect, so it recommended to manually review the +The automatic detection is not perfect, so it recommended to manually review the roles that were assigned to each package. You can use the [package role definitions](../local-dev/cli-build-system#package-roles) as a reference. ### Step 2 - Migrate package scripts The migration to package roles also introduces a new `package` command category to the CLI. -Each command under the `package` category is designed to be mapped directly to an entry in `"scripts"` in `package.json`. These commands replace the existing commands like `build`, `app:build`, `lint` and `test`. They look something like this: +Each command under the `package` category is designed to be mapped directly to an entry in `"scripts"` in `package.json`. These commands replace the existing commands like `build`, `app:build`, `lint`, and `test`. They look something like this: ```json { @@ -102,7 +97,7 @@ This will migrate all existing `.eslintrc.js` that extend the old configuration ### Step 4 - Use `backstage-cli repo` -The Backstage CLI recently introduced a new `repo` command category, which houses commands that operate on an entire monorepo at once. These commands work particularly well once packages have been migrated to use roles, as that allows for some very effective optimizations. It is typically much faster to use these commands compared to using tools like `lerna`, as they're able to avoid the overhead of calling package scripts through `yarn`. You can read more about the `repo` command in the [CLI command documentation](./not-found#TODO). +The Backstage CLI recently introduced a new `repo` command category, which houses commands that operate on an entire monorepo at once. These commands work particularly well once packages have been migrated to use roles, as that allows for some very effective optimizations. It is typically much faster to use these commands compared to using tools like `lerna`, as they're able to avoid the overhead of calling package scripts through `yarn` and can operate on multiple packages at once. You can read more about the `repo` command in the [CLI command documentation](../local-dev/cli-commands#repo-build). The way to execute this step of the migration is not as well defined as the previous steps, as it depends on what your development and CI/CD setup looks like. Look for the following patterns to replace in your root `package.json` as well as CI/CD setup: @@ -132,6 +127,7 @@ To keep configuration lean, allow for more utilities and tooling, and to enable ### Do I have to migrate to using package roles? Short answer - yes. + Longer answer - mostly, you can get around having to declare package the role of your packages by instead explicitly declaring the role in the command invocation or configuration. For example, the `app:build` command will go away, but you can replace it with `package build --role frontend` if you don't want to declare the role in `package.json` . It is however strongly recommended to declare the package roles. ### I have a package where none of the existing roles apply From d082db302b06288359969474b8b81d8c6a857877 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 8 Mar 2022 15:37:49 +0100 Subject: [PATCH 31/40] cli: mark deprecated commands as deprecated Signed-off-by: Patrik Oldsberg --- .changeset/pretty-vans-unite.md | 2 ++ packages/cli/src/commands/index.ts | 34 ++++++++++++++++++------------ 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/.changeset/pretty-vans-unite.md b/.changeset/pretty-vans-unite.md index faf535c053..318e86d069 100644 --- a/.changeset/pretty-vans-unite.md +++ b/.changeset/pretty-vans-unite.md @@ -6,4 +6,6 @@ Package roles are now marked as stable and migration is encouraged. Please check The new `package`, `repo`, and `migrate` command categories are now marked as stable. +Marked all commands that are being replaced by the new `package` and `repo` commands as deprecated. + The package templates used by the `create` command have all been updated to use package roles. diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index 9b272fab95..bc427bb19b 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -172,7 +172,7 @@ export function registerCommands(program: CommanderStatic) { // TODO(Rugvip): Deprecate in favor of package variant program .command('app:build') - .description('Build an app for a production release') + .description('Build an app for a production release [DEPRECATED]') .option('--stats', 'Write bundle stats to output directory') .option(...configOption) .action(lazy(() => import('./app/build').then(m => m.default))); @@ -180,7 +180,7 @@ export function registerCommands(program: CommanderStatic) { // TODO(Rugvip): Deprecate in favor of package variant program .command('app:serve') - .description('Serve an app for local development') + .description('Serve an app for local development [DEPRECATED]') .option('--check', 'Enable type checking and linting') .option(...configOption) .action(lazy(() => import('./app/serve').then(m => m.default))); @@ -188,7 +188,7 @@ export function registerCommands(program: CommanderStatic) { // TODO(Rugvip): Deprecate in favor of package variant program .command('backend:build') - .description('Build a backend plugin') + .description('Build a backend plugin [DEPRECATED]') .option('--minify', 'Minify the generated code') .option('--experimental-type-build', 'Enable experimental type build') .action(lazy(() => import('./backend/build').then(m => m.default))); @@ -196,7 +196,7 @@ export function registerCommands(program: CommanderStatic) { // TODO(Rugvip): Deprecate in favor of package variant program .command('backend:bundle') - .description('Bundle the backend into a deployment archive') + .description('Bundle the backend into a deployment archive [DEPRECATED]') .option( '--build-dependencies', 'Build all local package dependencies before bundling the backend', @@ -206,7 +206,9 @@ export function registerCommands(program: CommanderStatic) { // TODO(Rugvip): Deprecate in favor of package variant program .command('backend:dev') - .description('Start local development server with HMR for the backend') + .description( + 'Start local development server with HMR for the backend [DEPRECATED]', + ) .option('--check', 'Enable type checking and linting') .option('--inspect', 'Enable debugger') .option('--inspect-brk', 'Enable debugger with await to attach debugger') @@ -255,7 +257,7 @@ export function registerCommands(program: CommanderStatic) { // TODO(Rugvip): Deprecate in favor of package variant program .command('plugin:build') - .description('Build a plugin') + .description('Build a plugin [DEPRECATED]') .option('--minify', 'Minify the generated code') .option('--experimental-type-build', 'Enable experimental type build') .action(lazy(() => import('./plugin/build').then(m => m.default))); @@ -263,7 +265,7 @@ export function registerCommands(program: CommanderStatic) { // TODO(Rugvip): Deprecate in favor of package variant program .command('plugin:serve') - .description('Serves the dev/ folder of a plugin') + .description('Serves the dev/ folder of a plugin [DEPRECATED]') .option('--check', 'Enable type checking and linting') .option(...configOption) .action(lazy(() => import('./plugin/serve').then(m => m.default))); @@ -278,7 +280,7 @@ export function registerCommands(program: CommanderStatic) { // TODO(Rugvip): Deprecate in favor of package variant program .command('build') - .description('Build a package for publishing') + .description('Build a package for publishing [DEPRECATED]') .option('--outputs ', 'List of formats to output [types,cjs,esm]') .option('--minify', 'Minify the generated code') .option('--experimental-type-build', 'Enable experimental type build') @@ -293,7 +295,7 @@ export function registerCommands(program: CommanderStatic) { 'eslint-formatter-friendly', ) .option('--fix', 'Attempt to automatically fix violations') - .description('Lint a package') + .description('Lint a package [DEPRECATED]') .action(lazy(() => import('./lint').then(m => m.default))); // TODO(Rugvip): Deprecate in favor of package variant @@ -301,7 +303,9 @@ export function registerCommands(program: CommanderStatic) { .command('test') .allowUnknownOption(true) // Allows the command to run, but we still need to parse raw args .helpOption(', --backstage-cli-help') // Let Jest handle help - .description('Run tests, forwarding args to Jest, defaulting to watch mode') + .description( + 'Run tests, forwarding args to Jest, defaulting to watch mode [DEPRECATED]', + ) .action(lazy(() => import('./testCommand').then(m => m.default))); program @@ -385,19 +389,23 @@ export function registerCommands(program: CommanderStatic) { // TODO(Rugvip): Deprecate in favor of package variant program .command('prepack') - .description('Prepares a package for packaging before publishing') + .description( + 'Prepares a package for packaging before publishing [DEPRECATED]', + ) .action(lazy(() => import('./pack').then(m => m.pre))); // TODO(Rugvip): Deprecate in favor of package variant program .command('postpack') - .description('Restores the changes made by the prepack command') + .description( + 'Restores the changes made by the prepack command [DEPRECATED]', + ) .action(lazy(() => import('./pack').then(m => m.post))); // TODO(Rugvip): Deprecate in favor of package variant program .command('clean') - .description('Delete cache directories') + .description('Delete cache directories [DEPRECATED]') .action(lazy(() => import('./clean/clean').then(m => m.default))); program From 227714a0ef2d2028746835d9f1032ee7265016be Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 8 Mar 2022 15:45:34 +0100 Subject: [PATCH 32/40] docs: update multi-stage docker build to use package role build Signed-off-by: Patrik Oldsberg --- docs/deployment/docker.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/deployment/docker.md b/docs/deployment/docker.md index 36967f352a..0b99e81d64 100644 --- a/docs/deployment/docker.md +++ b/docs/deployment/docker.md @@ -167,7 +167,9 @@ RUN yarn install --frozen-lockfile --network-timeout 600000 && rm -rf "$(yarn ca COPY . . RUN yarn tsc -RUN yarn --cwd packages/backend backstage-cli backend:bundle --build-dependencies +RUN yarn --cwd packages/backend build +# If you have not yet migrated to package roles, use the following command instead: +# RUN yarn --cwd packages/backend backstage-cli backend:bundle --build-dependencies # Stage 3 - Build the actual backend image and install production dependencies FROM node:16-bullseye-slim From a0d60e72647f62abda288770293bbe496b381424 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 8 Mar 2022 15:48:28 +0100 Subject: [PATCH 33/40] chore: remove the golang cookiecutter template as we're no longer providing cookiecutter out of the box Signed-off-by: blam --- packages/create-app/templates/default-app/app-config.yaml.hbs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/create-app/templates/default-app/app-config.yaml.hbs b/packages/create-app/templates/default-app/app-config.yaml.hbs index 618c06a0c3..3afed0ed82 100644 --- a/packages/create-app/templates/default-app/app-config.yaml.hbs +++ b/packages/create-app/templates/default-app/app-config.yaml.hbs @@ -116,10 +116,6 @@ catalog: target: https://github.com/backstage/software-templates/blob/main/scaffolder-templates/springboot-grpc-template/template.yaml rules: - allow: [Template] - - type: url - target: https://github.com/spotify/cookiecutter-golang/blob/master/template.yaml - rules: - - allow: [Template] - type: url target: https://github.com/backstage/software-templates/blob/main/scaffolder-templates/docs-template/template.yaml rules: From 8a57b6595bda5c25346b3320ffdd17fd5bb3c917 Mon Sep 17 00:00:00 2001 From: blam Date: Tue, 8 Mar 2022 15:50:30 +0100 Subject: [PATCH 34/40] chore: added changeset Signed-off-by: blam --- .changeset/curvy-forks-cross.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .changeset/curvy-forks-cross.md diff --git a/.changeset/curvy-forks-cross.md b/.changeset/curvy-forks-cross.md new file mode 100644 index 0000000000..feced6ed0e --- /dev/null +++ b/.changeset/curvy-forks-cross.md @@ -0,0 +1,14 @@ +--- +'@backstage/create-app': patch +--- + +Removed the `cookiecutter-golang` template from the default `create-app` install as we no longer provide `cookiecutter` action out of the box. + +You can remove the template by removing the following lines from your `app-config.yaml` under `catalog.locations`: + +```diff +- - type: url +- target: https://github.com/spotify/cookiecutter-golang/blob/master/template.yaml +- rules: +- - allow: [Template] +``` From f751e845725af4cc1bafeb4aaad526b8ec639c5f Mon Sep 17 00:00:00 2001 From: Dominik Schwank Date: Tue, 8 Mar 2022 16:39:41 +0100 Subject: [PATCH 35/40] fix(catalog-backend-module-ldap): ignore search reference Having search referrals in the response causes the processor to stop right now. Instead of throwing an error they can simply be ignored and logged out. Signed-off-by: Dominik Schwank --- .changeset/quiet-seals-fix.md | 5 +++++ plugins/catalog-backend-module-ldap/src/ldap/client.ts | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 .changeset/quiet-seals-fix.md diff --git a/.changeset/quiet-seals-fix.md b/.changeset/quiet-seals-fix.md new file mode 100644 index 0000000000..d7c98958be --- /dev/null +++ b/.changeset/quiet-seals-fix.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-ldap': patch +--- + +Ignore search referrals instead of throwing an error. diff --git a/plugins/catalog-backend-module-ldap/src/ldap/client.ts b/plugins/catalog-backend-module-ldap/src/ldap/client.ts index 414d043d3d..e778b0cfe4 100644 --- a/plugins/catalog-backend-module-ldap/src/ldap/client.ts +++ b/plugins/catalog-backend-module-ldap/src/ldap/client.ts @@ -95,7 +95,7 @@ export class LdapClient { } res.on('searchReference', () => { - reject(new Error('Unable to handle referral')); + this.logger.warn('Received unsupported search referral'); }); res.on('searchEntry', entry => { @@ -154,7 +154,7 @@ export class LdapClient { } res.on('searchReference', () => { - reject(new Error('Unable to handle referral')); + this.logger.warn('Received unsupported search referral'); }); res.on('searchEntry', entry => { From 29525665875e89efc6bef671e8b2f40c3b44c731 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 8 Mar 2022 17:28:28 +0100 Subject: [PATCH 36/40] catalog-model: enable usage of : and / in entity names Signed-off-by: Patrik Oldsberg --- .changeset/fuzzy-roses-swim.md | 7 ++++++ packages/catalog-model/src/entity/ref.test.ts | 23 +++++++++++++++---- packages/catalog-model/src/entity/ref.ts | 21 +++++++++++------ 3 files changed, 39 insertions(+), 12 deletions(-) create mode 100644 .changeset/fuzzy-roses-swim.md diff --git a/.changeset/fuzzy-roses-swim.md b/.changeset/fuzzy-roses-swim.md new file mode 100644 index 0000000000..234d79c96a --- /dev/null +++ b/.changeset/fuzzy-roses-swim.md @@ -0,0 +1,7 @@ +--- +'@backstage/catalog-model': patch +--- + +Updated `parseEntityRef` to allow `:` and `/` in the entity name. For example, parsing `'component:default/foo:bar'` will result in the name `'foo:bar'`. + +Note that only parsing `'foo:bar'` itself will result in the name `'bar'` and the entity kind `'foo'`, meaning this is a particularly nasty trap for user defined entity references. For this reason it is strongly discouraged to use names that contain these characters, and the catalog model does not allow it by default. However, this change now makes is possible to use these names if the default catalog validation is replaced, and in particular a high level of automation of the catalog population can limit issues that it might otherwise cause. diff --git a/packages/catalog-model/src/entity/ref.test.ts b/packages/catalog-model/src/entity/ref.test.ts index 6cb7612439..0945eddf6e 100644 --- a/packages/catalog-model/src/entity/ref.test.ts +++ b/packages/catalog-model/src/entity/ref.test.ts @@ -34,11 +34,24 @@ describe('ref', () => { it('rejects bad inputs', () => { expect(() => parseEntityRef(null as any)).toThrow(); expect(() => parseEntityRef(7 as any)).toThrow(); - expect(() => parseEntityRef('a:b:c')).toThrow(); - expect(() => parseEntityRef('a/b/c')).toThrow(); - expect(() => parseEntityRef('a/b:c')).toThrow(); - expect(() => parseEntityRef('a:b/c/d')).toThrow(); - expect(() => parseEntityRef('a:b/c:d')).toThrow(); + }); + + it('allows names with : and /', () => { + expect( + parseEntityRef('a:b:c', { defaultKind: 'k', defaultNamespace: 'ns' }), + ).toEqual({ kind: 'a', namespace: 'ns', name: 'b:c' }); + expect( + parseEntityRef('a/b/c', { defaultKind: 'k', defaultNamespace: 'ns' }), + ).toEqual({ kind: 'k', namespace: 'a', name: 'b/c' }); + expect( + parseEntityRef('a/b:c', { defaultKind: 'k', defaultNamespace: 'ns' }), + ).toEqual({ kind: 'k', namespace: 'a', name: 'b:c' }); + expect( + parseEntityRef('a:b/c/d', { defaultKind: 'k', defaultNamespace: 'ns' }), + ).toEqual({ kind: 'a', namespace: 'b', name: 'c/d' }); + expect( + parseEntityRef('a:b/c:d', { defaultKind: 'k', defaultNamespace: 'ns' }), + ).toEqual({ kind: 'a', namespace: 'b', name: 'c:d' }); }); it('rejects empty parts in strings', () => { diff --git a/packages/catalog-model/src/entity/ref.ts b/packages/catalog-model/src/entity/ref.ts index acede5a56b..cfe979423f 100644 --- a/packages/catalog-model/src/entity/ref.ts +++ b/packages/catalog-model/src/entity/ref.ts @@ -23,18 +23,25 @@ function parseRefString(ref: string): { namespace?: string; name: string; } { - const match = /^([^:/]+:)?([^:/]+\/)?([^:/]+)$/.exec(ref.trim()); - if (!match) { + let colonI = ref.indexOf(':'); + const slashI = ref.indexOf('/'); + + // If the / is ahead of the :, treat the rest as the name + if (slashI !== -1 && slashI < colonI) { + colonI = -1; + } + + const kind = colonI === -1 ? undefined : ref.slice(0, colonI); + const namespace = slashI === -1 ? undefined : ref.slice(colonI + 1, slashI); + const name = ref.slice(Math.max(colonI + 1, slashI + 1)); + + if (kind === '' || namespace === '' || name === '') { throw new TypeError( `Entity reference "${ref}" was not on the form [:][/]`, ); } - return { - kind: match[1]?.slice(0, -1), - namespace: match[2]?.slice(0, -1), - name: match[3], - }; + return { kind, namespace, name }; } /** From 3cdb6c79ba128782efd7d3721931e522c7cca197 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 8 Mar 2022 19:35:14 +0100 Subject: [PATCH 37/40] docs: fix build system doc links Signed-off-by: Patrik Oldsberg --- docs/local-dev/cli-commands.md | 4 ++-- docs/local-dev/cli-overview.md | 2 +- docs/tutorials/package-role-migration.md | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/local-dev/cli-commands.md b/docs/local-dev/cli-commands.md index 78c311e5cf..7157fad2bf 100644 --- a/docs/local-dev/cli-commands.md +++ b/docs/local-dev/cli-commands.md @@ -93,7 +93,7 @@ Options: ## package start -Starts the package for local development. See the frontend and backend development parts in the build system [bundling](./cli-build-system#bundling) section for more details. +Starts the package for local development. See the frontend and backend development parts in the build system [bundling](./cli-build-system.md#bundling) section for more details. ```text Usage: backstage-cli package start [options] @@ -110,7 +110,7 @@ Options: ## package build -Build an individual package based on its role. See the build system [building](./cli-build-system#building) and [bundling](./cli-build-system#bundling) sections for more details. +Build an individual package based on its role. See the build system [building](./cli-build-system.md#building) and [bundling](./cli-build-system.md#bundling) sections for more details. ```text Usage: backstage-cli package build [options] diff --git a/docs/local-dev/cli-overview.md b/docs/local-dev/cli-overview.md index 3312926068..74c09d2865 100644 --- a/docs/local-dev/cli-overview.md +++ b/docs/local-dev/cli-overview.md @@ -50,4 +50,4 @@ improve the tooling, as well as to more easily keep the system up to date. - **Bundle** - A collection of the deployment artifacts. The output of the bundling process, which brings a collection of packages into a single collection of deployment artifacts. -- **Package Role** - The declared role of a package, see [package roles](./cli-build-system#package-roles). +- **Package Role** - The declared role of a package, see [package roles](./cli-build-system.md#package-roles). diff --git a/docs/tutorials/package-role-migration.md b/docs/tutorials/package-role-migration.md index a27dbb2e70..6d262e45c3 100644 --- a/docs/tutorials/package-role-migration.md +++ b/docs/tutorials/package-role-migration.md @@ -10,7 +10,7 @@ information about the change can be found in the [original RFC](https://github.c Package roles are implemented through a well-known `"backstage"."role"` field in the `package.json` of each package. There are a handful of roles defined so far, and it -is not possible to use values outside the [set of predefined roles](../local-dev/cli-build-system#package-roles). +is not possible to use values outside the [set of predefined roles](../local-dev/cli-build-system.md#package-roles). With roles in place in all packages, the Backstage CLI is able to automatically determine how to handle each package. For example, the different build commands @@ -55,7 +55,7 @@ yarn backstage-cli migrate package-roles The automatic detection is not perfect, so it recommended to manually review the roles that were assigned to each package. -You can use the [package role definitions](../local-dev/cli-build-system#package-roles) as a reference. +You can use the [package role definitions](../local-dev/cli-build-system.md#package-roles) as a reference. ### Step 2 - Migrate package scripts @@ -85,7 +85,7 @@ If you in the end do not want to use this exact script setup, it is still recomm ### Step 3 - Migrate package ESLint configurations -An area that has been simplified as part of the move to package roles is the ESLint configuration. Rather than having each package select which configuration they want (and getting it wrong), they now use a shared configuration factory that utilizes the package role. You can read more about the new configuration setup in the [build system documentation](../local-dev/cli-build-system#linting). +An area that has been simplified as part of the move to package roles is the ESLint configuration. Rather than having each package select which configuration they want (and getting it wrong), they now use a shared configuration factory that utilizes the package role. You can read more about the new configuration setup in the [build system documentation](../local-dev/cli-build-system.md#linting). To migrate the ESLint configuration of all packages in your project, run the following command: @@ -97,7 +97,7 @@ This will migrate all existing `.eslintrc.js` that extend the old configuration ### Step 4 - Use `backstage-cli repo` -The Backstage CLI recently introduced a new `repo` command category, which houses commands that operate on an entire monorepo at once. These commands work particularly well once packages have been migrated to use roles, as that allows for some very effective optimizations. It is typically much faster to use these commands compared to using tools like `lerna`, as they're able to avoid the overhead of calling package scripts through `yarn` and can operate on multiple packages at once. You can read more about the `repo` command in the [CLI command documentation](../local-dev/cli-commands#repo-build). +The Backstage CLI recently introduced a new `repo` command category, which houses commands that operate on an entire monorepo at once. These commands work particularly well once packages have been migrated to use roles, as that allows for some very effective optimizations. It is typically much faster to use these commands compared to using tools like `lerna`, as they're able to avoid the overhead of calling package scripts through `yarn` and can operate on multiple packages at once. You can read more about the `repo` command in the [CLI command documentation](../local-dev/cli-commands.md#repo-build). The way to execute this step of the migration is not as well defined as the previous steps, as it depends on what your development and CI/CD setup looks like. Look for the following patterns to replace in your root `package.json` as well as CI/CD setup: From ea22c04a4ff1bd24e259a2358bb813b0d50fb5ac Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 8 Mar 2022 19:37:18 +0100 Subject: [PATCH 38/40] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Patrik Oldsberg Co-authored-by: Fredrik Adelöw --- docs/local-dev/cli-build-system.md | 2 +- docs/tutorials/package-role-migration.md | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/local-dev/cli-build-system.md b/docs/local-dev/cli-build-system.md index a850673090..be07a67f22 100644 --- a/docs/local-dev/cli-build-system.md +++ b/docs/local-dev/cli-build-system.md @@ -79,7 +79,7 @@ implemented in a typical Backstage app. The Backstage build system uses the concept of package roles in order to help keep configuration lean, provide utility and tooling, and enable optimizations. A package role is a single string that identifies what the purpose of a package is, and it's -define in the `package.json` of each package like this: +defined in the `package.json` of each package like this: ```json { diff --git a/docs/tutorials/package-role-migration.md b/docs/tutorials/package-role-migration.md index 6d262e45c3..10d6eca586 100644 --- a/docs/tutorials/package-role-migration.md +++ b/docs/tutorials/package-role-migration.md @@ -43,7 +43,7 @@ yarn backstage-cli migrate package-scripts yarn backstage-cli migrate package-lint-configs ``` -Have a look at the new commands under `yarn backstage-cli repo`, and switch to them wherever you can. They tend to be a much faster compared to their `lerna` equivalents. +Have a look at the new commands under `yarn backstage-cli repo`, and switch to them wherever you can. They tend to be much faster compared to their `lerna` equivalents. ### Step 1 - Add package roles @@ -53,7 +53,7 @@ The first step is to add the `"backstage"."role"` field to each package. This ca yarn backstage-cli migrate package-roles ``` -The automatic detection is not perfect, so it recommended to manually review the +The automatic detection is not perfect, so it is recommended to manually review the roles that were assigned to each package. You can use the [package role definitions](../local-dev/cli-build-system.md#package-roles) as a reference. @@ -73,7 +73,7 @@ Each command under the `package` category is designed to be mapped directly to a } ``` -Every package role each has a fixed set of recommended scripts. It is strongly recommended that you use these scripts, as it allows for optimizations in other parts of the CLI. You can migrate to using all of these scripts by running the following command: +Every package role has a fixed set of recommended scripts. It is strongly recommended that you use these scripts, as it allows for optimizations in other parts of the CLI. You can migrate to using all of these scripts by running the following command: ```sh yarn backstage-cli migrate package-scripts @@ -113,14 +113,14 @@ The way to execute this step of the migration is not as well defined as the prev backstage-cli repo lint --since origin/master ``` -- In places where the entire repo is being built, use `yarn backstage-cli repo build`, which also supports the `--since` flag. The migration here is a bit more nuanced as it depends why you are building all packages. - - If you are building all packages to **verify** that you are able to build them, you most likely want `backstage-cli repo build --all`. The `--all` flag signals that bundled packages like `packages/app` and `packages/backend` should be build as well. Pair this up with a `--since` flag in CI to avoid needing to build all packages. +- In places where the entire repo is being built, use `yarn backstage-cli repo build`, which also supports the `--since` flag. The migration here is a bit more nuanced as it depends on why you are building all packages. + - If you are building all packages to **verify** that you are able to build them, you most likely want `backstage-cli repo build --all`. The `--all` flag signals that bundled packages like `packages/app` and `packages/backend` should be built as well. Pair this up with a `--since` flag in CI to avoid needing to build all packages. - If you are building all packages to **publish** them, then `backstage-cli repo build` is enough, as it builds all published packages. - If you are building all packages to **deploy** them, you likely don't want to use the `repo` command at all, simply call `yarn build` in the packages you want to deploy instead. For example, if you are deploying the backend with a docker host build, it's enough to call `yarn build` inside `packages/backend`. ## FAQ -### Why where packages roles introduced? +### Why were package roles introduced? To keep configuration lean, allow for more utilities and tooling, and to enable optimizations in the build system. You can read more about the reasoning in the [original RFC](https://github.com/backstage/backstage/issues/8729). @@ -128,11 +128,11 @@ To keep configuration lean, allow for more utilities and tooling, and to enable Short answer - yes. -Longer answer - mostly, you can get around having to declare package the role of your packages by instead explicitly declaring the role in the command invocation or configuration. For example, the `app:build` command will go away, but you can replace it with `package build --role frontend` if you don't want to declare the role in `package.json` . It is however strongly recommended to declare the package roles. +Longer answer - mostly, you can get around having to declare the role of your packages by instead explicitly declaring the role in the command invocation or configuration. For example, the `app:build` command will go away, but you can replace it with `package build --role frontend` if you don't want to declare the role in `package.json` . It is however strongly recommended to declare the package roles. ### I have a package where none of the existing roles apply -The `web-library`, `node-library` and `common-library` roles are general purpose roles that should cover most use cases. If you feel like none of those roles work for you either, then please open an issue in the [Backstage repo](https://github.com/backstage/backstage) and suggest the addition of a new role. +The `web-library`, `node-library` and `common-library` roles are general purpose roles that should cover most use cases. If you feel like none of those roles work for you, then please open an issue in the [Backstage repo](https://github.com/backstage/backstage) and suggest the addition of a new role. ### Should I include the role in published packages? From aabe31d5d13734e2716a9faed0b25f7478e532fc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Mar 2022 04:11:58 +0000 Subject: [PATCH 39/40] build(deps): bump keyv from 4.1.0 to 4.1.1 Bumps [keyv](https://github.com/jaredwray/keyv) from 4.1.0 to 4.1.1. - [Release notes](https://github.com/jaredwray/keyv/releases) - [Commits](https://github.com/jaredwray/keyv/commits) --- updated-dependencies: - dependency-name: keyv dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 7480544eee..d5efd0cf37 100644 --- a/yarn.lock +++ b/yarn.lock @@ -16219,9 +16219,9 @@ keyv@^3.0.0: json-buffer "3.0.0" keyv@^4.0.0, keyv@^4.0.3: - version "4.1.0" - resolved "https://registry.npmjs.org/keyv/-/keyv-4.1.0.tgz#8ab5ca4ae6a34e05c629531d9a7f871575af0d5b" - integrity sha512-YsY3wr6HabE11/sscee+3nZ03XjvkrPWGouAmJFBdZoK92wiOlJCzI5/sDEIKdJhdhHO144ei45U9gXfbu14Uw== + version "4.1.1" + resolved "https://registry.npmjs.org/keyv/-/keyv-4.1.1.tgz#02c538bfdbd2a9308cc932d4096f05ae42bfa06a" + integrity sha512-tGv1yP6snQVDSM4X6yxrv2zzq/EvpW+oYiUz6aueW1u9CtS8RzUQYxxmFwgZlO2jSgCxQbchhxaqXXp2hnKGpQ== dependencies: json-buffer "3.0.1" From 6049d33313efbde04d23a9c4d0a4e4fca4884523 Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 9 Mar 2022 10:08:50 +0100 Subject: [PATCH 40/40] chore: updating this in documentation though too Signed-off-by: blam --- docs/features/software-templates/adding-templates.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/features/software-templates/adding-templates.md b/docs/features/software-templates/adding-templates.md index ed7a103bc4..db765fc7ee 100644 --- a/docs/features/software-templates/adding-templates.md +++ b/docs/features/software-templates/adding-templates.md @@ -94,7 +94,7 @@ for example: catalog: locations: - type: url - target: https://github.com/spotify/cookiecutter-golang/blob/master/template.yaml + target: https://github.com/backstage/software-templates/blob/main/scaffolder-templates/react-ssr-template/template.yaml rules: - allow: [Template] ```