From 361ca74ac20262fab9e98e87c9ddefdad8ed65ea Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Fri, 9 Dec 2022 16:34:15 -0800 Subject: [PATCH 001/182] Added a mechanism to perform out-of-sequence deltas Signed-off-by: Damon Kaswell --- .../package.json | 1 + .../src/engine/IncrementalIngestionEngine.ts | 47 +- .../src/module/WrapperProviders.ts | 10 +- ...talIngestionEntityProviderCatalogModule.ts | 2 +- .../src/router/routes.ts | 441 ++++++++++-------- .../src/service/IncrementalCatalogBuilder.ts | 10 +- .../src/types.ts | 14 +- 7 files changed, 314 insertions(+), 211 deletions(-) diff --git a/plugins/catalog-backend-module-incremental-ingestion/package.json b/plugins/catalog-backend-module-incremental-ingestion/package.json index 15826cd65a..25252e74f4 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/package.json +++ b/plugins/catalog-backend-module-incremental-ingestion/package.json @@ -40,6 +40,7 @@ "@backstage/catalog-model": "workspace:^", "@backstage/config": "workspace:^", "@backstage/errors": "workspace:^", + "@backstage/plugin-events-node": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", "@backstage/plugin-catalog-node": "workspace:^", "@backstage/plugin-permission-common": "workspace:^", diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts b/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts index e2000c609e..ece2c5f574 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts @@ -21,14 +21,18 @@ import { performance } from 'perf_hooks'; import { Duration, DurationObjectUnits } from 'luxon'; import { v4 } from 'uuid'; import { stringifyError } from '@backstage/errors'; +import { EventParams, EventSubscriber } from '@backstage/plugin-events-node'; -export class IncrementalIngestionEngine implements IterationEngine { +export class IncrementalIngestionEngine + implements IterationEngine, EventSubscriber +{ private readonly restLength: Duration; private readonly backoff: DurationObjectUnits[]; + private readonly providerEventTopic: string; private manager: IncrementalIngestionDatabaseManager; - constructor(private options: IterationEngineOptions) { + constructor(private options: IterationEngineOptions) { this.manager = options.manager; this.restLength = Duration.fromObject(options.restLength); this.backoff = options.backoff ?? [ @@ -37,6 +41,7 @@ export class IncrementalIngestionEngine implements IterationEngine { { minutes: 30 }, { hours: 3 }, ]; + this.providerEventTopic = `${options.provider.getProviderName()}-delta`; } async taskFn(signal: AbortSignal) { @@ -326,4 +331,42 @@ export class IncrementalIngestionEngine implements IterationEngine { removed, }); } + + async onEvent(params: EventParams): Promise { + const { topic, eventPayload } = params; + if (topic !== this.providerEventTopic) { + return; + } + + const { logger, provider, connection } = this.options; + logger.info( + `incremental-engine: Received ${this.providerEventTopic} event`, + ); + + const payload = eventPayload as TInput; + + if (!provider.deltaMapper) { + return; + } + + const update = provider.deltaMapper(payload); + + if (update.delta) { + await connection.applyMutation({ + type: 'delta', + ...update.delta, + }); + logger.info( + `incremental-engine: Processed ${this.providerEventTopic} event`, + ); + } else { + logger.info( + `incremental-engine: Rejected ${this.providerEventTopic} event - empty or invalid`, + ); + } + } + + supportsEventTopics(): string[] { + return [this.providerEventTopic]; + } } diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts index 44da5067fd..ef8ac07b04 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts @@ -28,7 +28,7 @@ import { Duration } from 'luxon'; import { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager'; import { applyDatabaseMigrations } from '../database/migrations'; import { IncrementalIngestionEngine } from '../engine/IncrementalIngestionEngine'; -import { createIncrementalProviderRouter } from '../router/routes'; +import { IncrementalProviderRouter } from '../router/routes'; import { IncrementalEntityProvider, IncrementalEntityProviderOptions, @@ -72,14 +72,14 @@ export class WrapperProviders { } async adminRouter(): Promise { - return createIncrementalProviderRouter( + return await new IncrementalProviderRouter( new IncrementalIngestionDatabaseManager({ client: this.options.client }), loggerToWinstonLogger(this.options.logger), - ); + ).createRouter(); } - private async startProvider( - provider: IncrementalEntityProvider, + private async startProvider( + provider: IncrementalEntityProvider, providerOptions: IncrementalEntityProviderOptions, connection: EntityProviderConnection, ) { diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts index 0ecb45292c..2ed7dda728 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts @@ -38,7 +38,7 @@ export const incrementalIngestionEntityProviderCatalogModule = env, options: { providers: Array<{ - provider: IncrementalEntityProvider; + provider: IncrementalEntityProvider; options: IncrementalEntityProviderOptions; }>; }, diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts b/plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts index 9f480e879e..d03619d5a4 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts @@ -15,218 +15,269 @@ */ import { errorHandler } from '@backstage/backend-common'; +import { stringifyError } from '@backstage/errors'; +import { EventBroker, EventPublisher } from '@backstage/plugin-events-node'; import express from 'express'; import Router from 'express-promise-router'; import { Logger } from 'winston'; import { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager'; import { PROVIDER_BASE_PATH, PROVIDER_CLEANUP, PROVIDER_HEALTH } from './paths'; -export const createIncrementalProviderRouter = async ( - manager: IncrementalIngestionDatabaseManager, - logger: Logger, -) => { - const router = Router(); - router.use(express.json()); +export class IncrementalProviderRouter implements EventPublisher { + private manager: IncrementalIngestionDatabaseManager; + private logger: Logger; + private eventBroker: EventBroker | undefined; - // Get the overall health of all incremental providers - router.get(PROVIDER_HEALTH, async (_, res) => { - const records = await manager.healthcheck(); - const providers = records.map(record => record.provider_name); - const duplicates = [ - ...new Set(providers.filter((e, i, a) => a.indexOf(e) !== i)), - ]; + constructor(manager: IncrementalIngestionDatabaseManager, logger: Logger) { + this.manager = manager; + this.logger = logger; + } - if (duplicates.length > 0) { - res.json({ healthy: false, duplicateIngestions: duplicates }); - } else { - res.json({ healthy: true }); - } - }); + async setEventBroker(eventBroker: EventBroker): Promise { + this.eventBroker = eventBroker; + } - // Clean up and pause all providers - router.post(PROVIDER_CLEANUP, async (_, res) => { - const result = await manager.cleanupProviders(); - res.json(result); - }); + async createRouter() { + const router = Router(); + router.use(express.json()); - // Get basic status of the provider - router.get(PROVIDER_BASE_PATH, async (req, res) => { - const { provider } = req.params; - const record = await manager.getCurrentIngestionRecord(provider); - if (record) { - res.json({ - success: true, - status: { - current_action: record.status, - next_action_at: new Date(record.next_action_at), - }, - last_error: record.last_error, - }); - } else { - const providers: string[] = await manager.listProviders(); - if (providers.includes(provider)) { + // Get the overall health of all incremental providers + router.get(PROVIDER_HEALTH, async (_, res) => { + const records = await this.manager.healthcheck(); + const providers = records.map(record => record.provider_name); + const duplicates = [ + ...new Set(providers.filter((e, i, a) => a.indexOf(e) !== i)), + ]; + + if (duplicates.length > 0) { + res.json({ healthy: false, duplicateIngestions: duplicates }); + } else { + res.json({ healthy: true }); + } + }); + + // Clean up and pause all providers + router.post(PROVIDER_CLEANUP, async (_, res) => { + const result = await this.manager.cleanupProviders(); + res.json(result); + }); + + // Get basic status of the provider + router.get(PROVIDER_BASE_PATH, async (req, res) => { + const { provider } = req.params; + const record = await this.manager.getCurrentIngestionRecord(provider); + if (record) { res.json({ success: true, status: { - current_action: 'rest complete, waiting to start', + current_action: record.status, + next_action_at: new Date(record.next_action_at), }, + last_error: record.last_error, }); } else { - logger.error( - `${provider} - No ingestion record found in the database!`, - ); - res.status(404).json({ - success: false, - status: {}, - last_error: `Provider '${provider}' not found`, - }); + const providers: string[] = await this.manager.listProviders(); + if (providers.includes(provider)) { + res.json({ + success: true, + status: { + current_action: 'rest complete, waiting to start', + }, + }); + } else { + this.logger.error( + `${provider} - No ingestion record found in the database!`, + ); + res.status(404).json({ + success: false, + status: {}, + last_error: `Provider '${provider}' not found`, + }); + } } - } - }); - - // Trigger the provider's next action - router.post(`${PROVIDER_BASE_PATH}/trigger`, async (req, res) => { - const { provider } = req.params; - const record = await manager.getCurrentIngestionRecord(provider); - if (record) { - await manager.triggerNextProviderAction(provider); - res.json({ - success: true, - message: `${provider}: Next action triggered.`, - }); - } else { - const providers: string[] = await manager.listProviders(); - if (providers.includes(provider)) { - logger.debug(`${provider} - Ingestion record found`); - res.json({ - success: true, - message: 'Unable to trigger next action (provider is restarting)', - }); - } else { - res.status(404).json({ - success: false, - message: `Provider '${provider}' not found`, - }); - } - } - }); - - // Start a brand-new ingestion cycle for the provider. - // (Cancel's the current run if active, or marks it complete if resting) - router.post(`${PROVIDER_BASE_PATH}/start`, async (req, res) => { - const { provider } = req.params; - - const record = await manager.getCurrentIngestionRecord(provider); - if (record) { - const ingestionId = record.id; - if (record.status === 'resting') { - await manager.setProviderComplete(ingestionId); - } else { - await manager.setProviderCanceling(ingestionId); - } - res.json({ - success: true, - message: `${provider}: Next cycle triggered.`, - }); - } else { - const providers: string[] = await manager.listProviders(); - if (providers.includes(provider)) { - logger.debug(`${provider} - Ingestion record found`); - res.json({ - success: true, - message: 'Provider is already restarting', - }); - } else { - res.status(404).json({ - success: false, - message: `Provider '${provider}' not found`, - }); - } - } - }); - - // Stop the provider and pause it for 24 hours - router.post(`${PROVIDER_BASE_PATH}/cancel`, async (req, res) => { - const { provider } = req.params; - const record = await manager.getCurrentIngestionRecord(provider); - if (record) { - const next_action_at = new Date(); - next_action_at.setTime(next_action_at.getTime() + 24 * 60 * 60 * 1000); - await manager.updateByName(provider, { - next_action: 'nothing (done)', - ingestion_completed_at: new Date(), - next_action_at, - status: 'resting', - }); - res.json({ - success: true, - message: `${provider}: Current ingestion canceled.`, - }); - } else { - const providers: string[] = await manager.listProviders(); - if (providers.includes(provider)) { - logger.debug(`${provider} - Ingestion record found`); - res.json({ - success: true, - message: 'Provider is currently restarting, please wait.', - }); - } else { - res.status(404).json({ - success: false, - message: `Provider '${provider}' not found`, - }); - } - } - }); - - // Wipe out all ingestion records for the provider and pause for 24 hours - router.delete(PROVIDER_BASE_PATH, async (req, res) => { - const { provider } = req.params; - const result = await manager.purgeAndResetProvider(provider); - res.json(result); - }); - - // Get the ingestion marks for the current cycle - router.get(`${PROVIDER_BASE_PATH}/marks`, async (req, res) => { - const { provider } = req.params; - const record = await manager.getCurrentIngestionRecord(provider); - if (record) { - const id = record.id; - const records = await manager.getAllMarks(id); - res.json({ success: true, records }); - } else { - const providers: string[] = await manager.listProviders(); - if (providers.includes(provider)) { - logger.debug(`${provider} - Ingestion record found`); - res.json({ - success: true, - message: 'No records yet (provider is restarting)', - }); - } else { - logger.error( - `${provider} - No ingestion record found in the database!`, - ); - res.status(404).json({ - success: false, - status: {}, - last_error: `Provider '${provider}' not found`, - }); - } - } - }); - - router.delete(`${PROVIDER_BASE_PATH}/marks`, async (req, res) => { - const { provider } = req.params; - const deletions = await manager.clearFinishedIngestions(provider); - - res.json({ - success: true, - message: `Expired marks for provider '${provider}' removed.`, - deletions, }); - }); - router.use(errorHandler()); + // Trigger the provider's next action + router.post(`${PROVIDER_BASE_PATH}/trigger`, async (req, res) => { + const { provider } = req.params; + const record = await this.manager.getCurrentIngestionRecord(provider); + if (record) { + await this.manager.triggerNextProviderAction(provider); + res.json({ + success: true, + message: `${provider}: Next action triggered.`, + }); + } else { + const providers: string[] = await this.manager.listProviders(); + if (providers.includes(provider)) { + this.logger.debug(`${provider} - Ingestion record found`); + res.json({ + success: true, + message: 'Unable to trigger next action (provider is restarting)', + }); + } else { + res.status(404).json({ + success: false, + message: `Provider '${provider}' not found`, + }); + } + } + }); - return router; -}; + // Start a brand-new ingestion cycle for the provider. + // (Cancel's the current run if active, or marks it complete if resting) + router.post(`${PROVIDER_BASE_PATH}/start`, async (req, res) => { + const { provider } = req.params; + + const record = await this.manager.getCurrentIngestionRecord(provider); + if (record) { + const ingestionId = record.id; + if (record.status === 'resting') { + await this.manager.setProviderComplete(ingestionId); + } else { + await this.manager.setProviderCanceling(ingestionId); + } + res.json({ + success: true, + message: `${provider}: Next cycle triggered.`, + }); + } else { + const providers: string[] = await this.manager.listProviders(); + if (providers.includes(provider)) { + this.logger.debug(`${provider} - Ingestion record found`); + res.json({ + success: true, + message: 'Provider is already restarting', + }); + } else { + res.status(404).json({ + success: false, + message: `Provider '${provider}' not found`, + }); + } + } + }); + + // Stop the provider and pause it for 24 hours + router.post(`${PROVIDER_BASE_PATH}/cancel`, async (req, res) => { + const { provider } = req.params; + const record = await this.manager.getCurrentIngestionRecord(provider); + if (record) { + const next_action_at = new Date(); + next_action_at.setTime(next_action_at.getTime() + 24 * 60 * 60 * 1000); + await this.manager.updateByName(provider, { + next_action: 'nothing (done)', + ingestion_completed_at: new Date(), + next_action_at, + status: 'resting', + }); + res.json({ + success: true, + message: `${provider}: Current ingestion canceled.`, + }); + } else { + const providers: string[] = await this.manager.listProviders(); + if (providers.includes(provider)) { + this.logger.debug(`${provider} - Ingestion record found`); + res.json({ + success: true, + message: 'Provider is currently restarting, please wait.', + }); + } else { + res.status(404).json({ + success: false, + message: `Provider '${provider}' not found`, + }); + } + } + }); + + // Wipe out all ingestion records for the provider and pause for 24 hours + router.delete(PROVIDER_BASE_PATH, async (req, res) => { + const { provider } = req.params; + const result = await this.manager.purgeAndResetProvider(provider); + res.json(result); + }); + + // Get the ingestion marks for the current cycle + router.get(`${PROVIDER_BASE_PATH}/marks`, async (req, res) => { + const { provider } = req.params; + const record = await this.manager.getCurrentIngestionRecord(provider); + if (record) { + const id = record.id; + const records = await this.manager.getAllMarks(id); + res.json({ success: true, records }); + } else { + const providers: string[] = await this.manager.listProviders(); + if (providers.includes(provider)) { + this.logger.debug(`${provider} - Ingestion record found`); + res.json({ + success: true, + message: 'No records yet (provider is restarting)', + }); + } else { + this.logger.error( + `${provider} - No ingestion record found in the database!`, + ); + res.status(404).json({ + success: false, + status: {}, + last_error: `Provider '${provider}' not found`, + }); + } + } + }); + + router.delete(`${PROVIDER_BASE_PATH}/marks`, async (req, res) => { + const { provider } = req.params; + const deletions = await this.manager.clearFinishedIngestions(provider); + + res.json({ + success: true, + message: `Expired marks for provider '${provider}' removed.`, + deletions, + }); + }); + + router.post(`${PROVIDER_BASE_PATH}/delta`, async (req, res) => { + const { provider } = req.params; + + const topic = `${provider}-push`; + + const eventPayload = req.body; + + if (!this.eventBroker) { + res.status(500).json({ + success: false, + provider, + message: `The payload could not be processed!`, + }); + throw new Error('Event broker not initialized!'); + } + + try { + await this.eventBroker.publish({ + topic, + eventPayload, + }); + res.json({ + success: true, + provider, + message: 'Payload submitted.', + }); + } catch (e) { + res.status(500).json({ + success: false, + provider, + message: `There was an error submitting the payload: ${stringifyError( + e, + )}`, + }); + } + }); + + router.use(errorHandler()); + + return router; + } +} diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts b/plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts index d6bc5549aa..e65698d4a4 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts @@ -24,7 +24,7 @@ import { Knex } from 'knex'; import { IncrementalIngestionEngine } from '../engine/IncrementalIngestionEngine'; import { applyDatabaseMigrations } from '../database/migrations'; import { IncrementalIngestionDatabaseManager } from '../database/IncrementalIngestionDatabaseManager'; -import { createIncrementalProviderRouter } from '../router/routes'; +import { IncrementalProviderRouter } from '../router/routes'; import { Deferred } from '../util'; /** @public */ @@ -60,16 +60,16 @@ export class IncrementalCatalogBuilder { router: 'IncrementalProviderAdmin', }); - const incrementalAdminRouter = await createIncrementalProviderRouter( + const incrementalAdminRouter = await new IncrementalProviderRouter( this.manager, routerLogger, - ); + ).createRouter(); return { incrementalAdminRouter }; } - addIncrementalEntityProvider( - provider: IncrementalEntityProvider, + addIncrementalEntityProvider( + provider: IncrementalEntityProvider, options: IncrementalEntityProviderOptions, ) { const { burstInterval, burstLength, restLength } = options; diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/types.ts b/plugins/catalog-backend-module-incremental-ingestion/src/types.ts index 51a8738500..79563fd2e8 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/types.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/types.ts @@ -46,7 +46,7 @@ import { IncrementalIngestionDatabaseManager } from './database/IncrementalInges * * @public */ -export interface IncrementalEntityProvider { +export interface IncrementalEntityProvider { /** * This name must be unique between all of the entity providers * operating in the catalog. @@ -75,6 +75,14 @@ export interface IncrementalEntityProvider { * @param burst - a function which performs a series of iterations */ around(burst: (context: TContext) => Promise): Promise; + + /** + * If present, this method maps incoming payloads to apply updates + * outside of the incremental ingestion schedule. + */ + deltaMapper?: (payload: TInput) => { + delta: { added: DeferredEntity[]; removed: DeferredEntity[] } | undefined; + }; } /** @@ -154,11 +162,11 @@ export interface IterationEngine { taskFn: TaskFunction; } -export interface IterationEngineOptions { +export interface IterationEngineOptions { logger: Logger; connection: EntityProviderConnection; manager: IncrementalIngestionDatabaseManager; - provider: IncrementalEntityProvider; + provider: IncrementalEntityProvider; restLength: DurationObjectUnits; ready: Promise; backoff?: IncrementalEntityProviderOptions['backoff']; From c10f27b44cece704e1488808a685b043df1043ad Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Fri, 9 Dec 2022 16:56:15 -0800 Subject: [PATCH 002/182] Fix test and module Signed-off-by: Damon Kaswell --- .../api-report.md | 16 ++++++++++++---- ...lIngestionEntityProviderCatalogModule.test.ts | 2 +- .../src/run.ts | 2 +- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/plugins/catalog-backend-module-incremental-ingestion/api-report.md b/plugins/catalog-backend-module-incremental-ingestion/api-report.md index 387e505c33..11ffdd565c 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/api-report.md +++ b/plugins/catalog-backend-module-incremental-ingestion/api-report.md @@ -33,8 +33,8 @@ export type EntityIteratorResult = // @public (undocumented) export class IncrementalCatalogBuilder { // (undocumented) - addIncrementalEntityProvider( - provider: IncrementalEntityProvider, + addIncrementalEntityProvider( + provider: IncrementalEntityProvider, options: IncrementalEntityProviderOptions, ): void; // (undocumented) @@ -48,8 +48,16 @@ export class IncrementalCatalogBuilder { } // @public -export interface IncrementalEntityProvider { +export interface IncrementalEntityProvider { around(burst: (context: TContext) => Promise): Promise; + deltaMapper?: (payload: TInput) => { + delta: + | { + added: DeferredEntity[]; + removed: DeferredEntity[]; + } + | undefined; + }; getProviderName(): string; next( context: TContext, @@ -70,7 +78,7 @@ export interface IncrementalEntityProviderOptions { // @alpha export const incrementalIngestionEntityProviderCatalogModule: (options: { providers: { - provider: IncrementalEntityProvider; + provider: IncrementalEntityProvider; options: IncrementalEntityProviderOptions; }[]; }) => BackendFeature; diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.test.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.test.ts index 06cbcb010e..d196693db8 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.test.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.test.ts @@ -24,7 +24,7 @@ import { incrementalIngestionEntityProviderCatalogModule } from './incrementalIn describe('bitbucketServerEntityProviderCatalogModule', () => { it('should register provider at the catalog extension point', async () => { - const provider1: IncrementalEntityProvider = { + const provider1: IncrementalEntityProvider = { getProviderName: () => 'provider1', around: burst => burst(0), next: async (cursor, _context) => { diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/run.ts b/plugins/catalog-backend-module-incremental-ingestion/src/run.ts index 08d95f4417..2dee5716c0 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/run.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/run.ts @@ -36,7 +36,7 @@ import { incrementalIngestionEntityProviderCatalogModule, } from '.'; -const provider: IncrementalEntityProvider = { +const provider: IncrementalEntityProvider = { getProviderName: () => 'test-provider', around: burst => burst(0), next: async (_context, cursor) => { From 1ba120faa3483413627f4cac4767580f0d3fa4b6 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Fri, 9 Dec 2022 16:58:48 -0800 Subject: [PATCH 003/182] Update changelog Signed-off-by: Damon Kaswell --- .changeset/bright-eagles-love.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/bright-eagles-love.md diff --git a/.changeset/bright-eagles-love.md b/.changeset/bright-eagles-love.md new file mode 100644 index 0000000000..eade94f6f8 --- /dev/null +++ b/.changeset/bright-eagles-love.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-incremental-ingestion': minor +--- + +Added new mechanism to handle deltas in incremental providers From 034830f54ff4a1bbdfb26f9792a0d95e04962c71 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Sat, 10 Dec 2022 05:58:41 -0800 Subject: [PATCH 004/182] Update yarn.lock Signed-off-by: Damon Kaswell --- .../catalog-backend-module-incremental-ingestion/package.json | 2 +- yarn.lock | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-incremental-ingestion/package.json b/plugins/catalog-backend-module-incremental-ingestion/package.json index 25252e74f4..ace6cfebdb 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/package.json +++ b/plugins/catalog-backend-module-incremental-ingestion/package.json @@ -40,9 +40,9 @@ "@backstage/catalog-model": "workspace:^", "@backstage/config": "workspace:^", "@backstage/errors": "workspace:^", - "@backstage/plugin-events-node": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", "@backstage/plugin-catalog-node": "workspace:^", + "@backstage/plugin-events-node": "workspace:^", "@backstage/plugin-permission-common": "workspace:^", "@types/express": "^4.17.6", "@types/luxon": "^3.0.0", diff --git a/yarn.lock b/yarn.lock index 33e5718702..4bf126ba7f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5109,6 +5109,7 @@ __metadata: "@backstage/errors": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" "@backstage/plugin-catalog-node": "workspace:^" + "@backstage/plugin-events-node": "workspace:^" "@backstage/plugin-permission-common": "workspace:^" "@types/express": ^4.17.6 "@types/luxon": ^3.0.0 From e88d16ee91632aa21b7f9043732b8037c85805f1 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 15 Dec 2022 09:06:14 -0800 Subject: [PATCH 005/182] Synchronize ingestion_mark_entities table with deltas Signed-off-by: Damon Kaswell --- .../IncrementalIngestionDatabaseManager.ts | 26 +++++++++++++++++ .../src/engine/IncrementalIngestionEngine.ts | 29 +++++++++++++++++++ .../src/types.ts | 7 ++++- 3 files changed, 61 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/database/IncrementalIngestionDatabaseManager.ts b/plugins/catalog-backend-module-incremental-ingestion/src/database/IncrementalIngestionDatabaseManager.ts index 9eb52c4558..bb319b311e 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/database/IncrementalIngestionDatabaseManager.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/database/IncrementalIngestionDatabaseManager.ts @@ -269,6 +269,17 @@ export class IncrementalIngestionDatabaseManager { }); } + /** + * This method is used to remove entity records from the ingestion_mark_entities + * table by their entity reference. + */ + async deleteEntityRecordsByRef(entities: { entityRef: string }[]) { + const refs = entities.map(e => e.entityRef); + await this.client.transaction(async tx => { + await tx('ingestion_mark_entities').delete().whereIn('ref', refs); + }); + } + /** * Creates a new ingestion record. * @param provider - string @@ -535,6 +546,21 @@ export class IncrementalIngestionDatabaseManager { }); } + /** + * Returns the first record from `ingestion_marks` for the supplied ingestionId. + * @param ingestionId - string + * @returns MarkRecord | undefined + */ + async getFirstMark(ingestionId: string) { + return await this.client.transaction(async tx => { + const mark = await tx('ingestion_marks') + .where('ingestion_id', ingestionId) + .orderBy('sequence', 'asc') + .first(); + return mark; + }); + } + async getAllMarks(ingestionId: string) { return await this.client.transaction(async tx => { const marks = await tx('ingestion_marks') diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts b/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts index ece2c5f574..f0507dbbc6 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts @@ -339,6 +339,7 @@ export class IncrementalIngestionEngine } const { logger, provider, connection } = this.options; + const providerName = provider.getProviderName(); logger.info( `incremental-engine: Received ${this.providerEventTopic} event`, ); @@ -352,6 +353,34 @@ export class IncrementalIngestionEngine const update = provider.deltaMapper(payload); if (update.delta) { + if (update.delta.added.length > 0) { + const ingestionRecord = await this.manager.getCurrentIngestionRecord( + providerName, + ); + + if (!ingestionRecord) { + logger.debug( + `incremental-engine: Skipping delta addition because incremental ingestion is restarting.`, + ); + } else { + const mark = + ingestionRecord.status === 'resting' + ? await this.manager.getLastMark(ingestionRecord.id) + : await this.manager.getFirstMark(ingestionRecord.id); + + if (!mark) { + throw new Error( + `Cannot apply delta, page records are missing! Please re-run incremental ingestion for ${providerName}.`, + ); + } + await this.manager.createMarkEntities(mark.id, update.delta.added); + } + } + + if (update.delta.removed.length > 0) { + await this.manager.deleteEntityRecordsByRef(update.delta.removed); + } + await connection.applyMutation({ type: 'delta', ...update.delta, diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/types.ts b/plugins/catalog-backend-module-incremental-ingestion/src/types.ts index 79563fd2e8..f1325f5cc9 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/types.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/types.ts @@ -81,7 +81,12 @@ export interface IncrementalEntityProvider { * outside of the incremental ingestion schedule. */ deltaMapper?: (payload: TInput) => { - delta: { added: DeferredEntity[]; removed: DeferredEntity[] } | undefined; + delta: + | { + added: DeferredEntity[]; + removed: { entityRef: string }[]; + } + | undefined; }; } From 7fd3b629df1753de8c7e4de187fa74223da0811a Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 15 Dec 2022 09:12:47 -0800 Subject: [PATCH 006/182] Formatting and api report Signed-off-by: Damon Kaswell --- .../api-report.md | 4 +++- .../src/engine/IncrementalIngestionEngine.ts | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend-module-incremental-ingestion/api-report.md b/plugins/catalog-backend-module-incremental-ingestion/api-report.md index 11ffdd565c..c56ae0e7bf 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/api-report.md +++ b/plugins/catalog-backend-module-incremental-ingestion/api-report.md @@ -54,7 +54,9 @@ export interface IncrementalEntityProvider { delta: | { added: DeferredEntity[]; - removed: DeferredEntity[]; + removed: { + entityRef: string; + }[]; } | undefined; }; diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts b/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts index f0507dbbc6..5f5df5ffe1 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts @@ -389,7 +389,7 @@ export class IncrementalIngestionEngine `incremental-engine: Processed ${this.providerEventTopic} event`, ); } else { - logger.info( + logger.warn( `incremental-engine: Rejected ${this.providerEventTopic} event - empty or invalid`, ); } From 1e828e1f3e16b688ddf30b018cd8e36eb9dae88f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 2 Jan 2023 18:22:24 +0000 Subject: [PATCH 007/182] chore(deps): update dependency @testing-library/cypress to v9 Signed-off-by: Renovate Bot --- packages/app/package.json | 2 +- yarn.lock | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/app/package.json b/packages/app/package.json index bdf019f107..ac42281a78 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -83,7 +83,7 @@ }, "devDependencies": { "@backstage/test-utils": "workspace:^", - "@testing-library/cypress": "^8.0.2", + "@testing-library/cypress": "^9.0.0", "@testing-library/jest-dom": "^5.10.1", "@testing-library/react": "^12.1.3", "@testing-library/user-event": "^14.0.0", diff --git a/yarn.lock b/yarn.lock index f0435a6f9e..67f2dac999 100644 --- a/yarn.lock +++ b/yarn.lock @@ -13624,15 +13624,15 @@ __metadata: languageName: unknown linkType: soft -"@testing-library/cypress@npm:^8.0.2": - version: 8.0.7 - resolution: "@testing-library/cypress@npm:8.0.7" +"@testing-library/cypress@npm:^9.0.0": + version: 9.0.0 + resolution: "@testing-library/cypress@npm:9.0.0" dependencies: "@babel/runtime": ^7.14.6 "@testing-library/dom": ^8.1.0 peerDependencies: - cypress: ^2.1.0 || ^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0 || ^9.0.0 || ^10.0.0 || ^11.0.0 - checksum: e005bc1a7ec808706c57e95ed312069fb5be39ea7362900dc2a32c09f124d478ade69ebcd7df88c076e3867ab328ae6e6ce13791bdf042621ff66b56552bf74b + cypress: ^12.0.0 + checksum: fbd24e8f0b8a60279b336de5f6bc0e7ad6fb31316eacab5128dacc7fccde1eb40935b90f2c3bddc7d814115fe3965c6dbf011785448cd15b5a5b0bc40ef5bb4c languageName: node linkType: hard @@ -22304,7 +22304,7 @@ __metadata: "@roadiehq/backstage-plugin-github-insights": ^2.0.5 "@roadiehq/backstage-plugin-github-pull-requests": ^2.2.7 "@roadiehq/backstage-plugin-travis-ci": ^2.0.5 - "@testing-library/cypress": ^8.0.2 + "@testing-library/cypress": ^9.0.0 "@testing-library/jest-dom": ^5.10.1 "@testing-library/react": ^12.1.3 "@testing-library/user-event": ^14.0.0 From c8e09cc383da3b2126b69aa8b8f64449b1cf78fb Mon Sep 17 00:00:00 2001 From: Gabriel Testault Date: Thu, 5 Jan 2023 16:16:48 +0100 Subject: [PATCH 008/182] fix(techdocs): Fixed bug in Techdocs reader where a techdocs page with a hash in the URL did not always jump to the document anchor. the scrollIntoAnchor transformer now jumps to the anchor only after css waas loaded instead of jumping after 200ms. Signed-off-by: Gabriel Testault --- .changeset/gold-masks-sleep.md | 5 + .../transformers/scrollIntoAnchor.test.ts | 92 +++++++++++++++---- .../reader/transformers/scrollIntoAnchor.ts | 23 +++-- 3 files changed, 95 insertions(+), 25 deletions(-) create mode 100644 .changeset/gold-masks-sleep.md diff --git a/.changeset/gold-masks-sleep.md b/.changeset/gold-masks-sleep.md new file mode 100644 index 0000000000..425a1abdc0 --- /dev/null +++ b/.changeset/gold-masks-sleep.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-techdocs': patch +--- + +Fixed bug in Techdocs reader where a techdocs page with a hash in the URL did not always jump to the document anchor. diff --git a/plugins/techdocs/src/reader/transformers/scrollIntoAnchor.test.ts b/plugins/techdocs/src/reader/transformers/scrollIntoAnchor.test.ts index 896f909be4..e222c24a4c 100644 --- a/plugins/techdocs/src/reader/transformers/scrollIntoAnchor.test.ts +++ b/plugins/techdocs/src/reader/transformers/scrollIntoAnchor.test.ts @@ -15,46 +15,104 @@ */ import { scrollIntoAnchor } from '../transformers'; - -jest.useFakeTimers(); +import { createTestShadowDom } from '../../test-utils'; +import { Transformer } from './transformer'; +import mkdocsIndex from '../../test-utils/fixtures/mkdocs-index'; +import { SHADOW_DOM_STYLE_LOAD_EVENT } from '@backstage/plugin-techdocs-react'; describe('scrollIntoAnchor', () => { - const transformer = scrollIntoAnchor(); - const dom = { querySelector: jest.fn() }; + const scrollIntoView = jest.fn(); + let querySelectorSpy: jest.SpyInstance; + let addEventListenerSpy: jest.SpyInstance; + let removeEventListenerSpy: jest.SpyInstance; + const applySpies: Transformer = dom => { + querySelectorSpy = jest.spyOn(dom, 'querySelector'); + querySelectorSpy.mockReturnValue({ scrollIntoView }); + addEventListenerSpy = jest.spyOn(dom, 'addEventListener'); + removeEventListenerSpy = jest.spyOn(dom, 'removeEventListener'); + return dom; + }; afterEach(() => { jest.clearAllMocks(); }); it('does nothing if there is no anchor element', async () => { - transformer(dom as unknown as Element); - jest.advanceTimersByTime(200); - expect(dom.querySelector).not.toHaveBeenCalled(); + await createTestShadowDom(mkdocsIndex, { + preTransformers: [], + postTransformers: [applySpies, scrollIntoAnchor()], + }); + expect(querySelectorSpy).not.toHaveBeenCalled(); + expect(addEventListenerSpy).toHaveBeenCalled(); + expect(removeEventListenerSpy).toHaveBeenCalled(); + expect(addEventListenerSpy).toHaveBeenCalledWith( + SHADOW_DOM_STYLE_LOAD_EVENT, + expect.any(Function), + ); + expect(removeEventListenerSpy).toHaveBeenCalledTimes(1); + expect(removeEventListenerSpy).toHaveBeenCalledWith( + SHADOW_DOM_STYLE_LOAD_EVENT, + expect.any(Function), + ); + // check that the same function is passed to both addEventListener and removeEventListener + expect(addEventListenerSpy.mock.calls[0][1]).toBe( + removeEventListenerSpy.mock.calls[0][1], + ); }); it('scroll to the hash anchor element', async () => { - const scrollIntoView = jest.fn(); - dom.querySelector.mockReturnValue({ scrollIntoView }); window.location.hash = '#hash'; - transformer(dom as unknown as Element); - jest.advanceTimersByTime(200); - expect(dom.querySelector).toHaveBeenCalledWith( + await createTestShadowDom(mkdocsIndex, { + preTransformers: [], + postTransformers: [applySpies, scrollIntoAnchor()], + }); + expect(querySelectorSpy).toHaveBeenCalledWith( expect.stringMatching('[id="hash"]'), ); expect(scrollIntoView).toHaveBeenCalledWith(); + expect(addEventListenerSpy).toHaveBeenCalledTimes(1); + expect(removeEventListenerSpy).toHaveBeenCalledTimes(1); + expect(addEventListenerSpy).toHaveBeenCalledWith( + SHADOW_DOM_STYLE_LOAD_EVENT, + expect.any(Function), + ); + expect(removeEventListenerSpy).toHaveBeenCalledTimes(1); + expect(removeEventListenerSpy).toHaveBeenCalledWith( + SHADOW_DOM_STYLE_LOAD_EVENT, + expect.any(Function), + ); + // check that the same function is passed to both addEventListener and removeEventListener + expect(addEventListenerSpy.mock.calls[0][1]).toBe( + removeEventListenerSpy.mock.calls[0][1], + ); window.location.hash = ''; }); it('works for anchor starting with number', async () => { - const scrollIntoView = jest.fn(); - dom.querySelector.mockReturnValue({ scrollIntoView }); + querySelectorSpy.mockReturnValue({ scrollIntoView }); window.location.hash = '#1-hash'; - transformer(dom as unknown as Element); - jest.advanceTimersByTime(200); - expect(dom.querySelector).toHaveBeenCalledWith( + await createTestShadowDom(mkdocsIndex, { + preTransformers: [], + postTransformers: [applySpies, scrollIntoAnchor()], + }); + expect(querySelectorSpy).toHaveBeenCalledWith( expect.stringMatching('[id="1-hash"]'), ); expect(scrollIntoView).toHaveBeenCalledWith(); + expect(addEventListenerSpy).toHaveBeenCalledTimes(1); + expect(addEventListenerSpy).toHaveBeenCalledWith( + SHADOW_DOM_STYLE_LOAD_EVENT, + expect.any(Function), + ); + expect(removeEventListenerSpy).toHaveBeenCalledTimes(1); + expect(removeEventListenerSpy).toHaveBeenCalledWith( + SHADOW_DOM_STYLE_LOAD_EVENT, + expect.any(Function), + ); + // check that the same function is passed to both addEventListener and removeEventListener + expect(addEventListenerSpy.mock.calls[0][1]).toBe( + removeEventListenerSpy.mock.calls[0][1], + ); window.location.hash = ''; }); }); diff --git a/plugins/techdocs/src/reader/transformers/scrollIntoAnchor.ts b/plugins/techdocs/src/reader/transformers/scrollIntoAnchor.ts index 7ba02bcdfc..0be6fefc0b 100644 --- a/plugins/techdocs/src/reader/transformers/scrollIntoAnchor.ts +++ b/plugins/techdocs/src/reader/transformers/scrollIntoAnchor.ts @@ -15,17 +15,24 @@ */ import type { Transformer } from './transformer'; +import { SHADOW_DOM_STYLE_LOAD_EVENT } from '@backstage/plugin-techdocs-react'; export const scrollIntoAnchor = (): Transformer => { return dom => { - setTimeout(() => { - // Scroll to the desired anchor on initial navigation - if (window.location.hash) { - const hash = window.location.hash.slice(1); - // fix invalid selector error for anchor starting with number - dom?.querySelector(`[id="${hash}"]`)?.scrollIntoView(); - } - }, 200); + dom.addEventListener( + SHADOW_DOM_STYLE_LOAD_EVENT, + function handleShadowDomStyleLoad() { + if (window.location.hash) { + const hash = window.location.hash.slice(1); + // fix invalid selector error for anchor starting with number + dom?.querySelector(`[id="${hash}"]`)?.scrollIntoView(); + } + dom.removeEventListener( + SHADOW_DOM_STYLE_LOAD_EVENT, + handleShadowDomStyleLoad, + ); + }, + ); return dom; }; }; From 87ab76e55c6f1ff4936e51b23202af9fdd945749 Mon Sep 17 00:00:00 2001 From: Dang Huynh Date: Fri, 6 Jan 2023 15:23:38 -0600 Subject: [PATCH 009/182] Fixed AWS SDK V3 bug with s3ForcePathStyle Signed-off-by: Dang Huynh --- .changeset/cyan-deers-walk.md | 5 +++++ plugins/techdocs-node/src/stages/publish/awsS3.ts | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 .changeset/cyan-deers-walk.md diff --git a/.changeset/cyan-deers-walk.md b/.changeset/cyan-deers-walk.md new file mode 100644 index 0000000000..2a9265090b --- /dev/null +++ b/.changeset/cyan-deers-walk.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-techdocs-node': minor +--- + +Fixed bug caused by recent migration to AWS SDK V3 for techdocs-node. Instead of s3ForcePathStyle, forcePathStyle should be passed. diff --git a/plugins/techdocs-node/src/stages/publish/awsS3.ts b/plugins/techdocs-node/src/stages/publish/awsS3.ts index a13baef926..6b528cfd30 100644 --- a/plugins/techdocs-node/src/stages/publish/awsS3.ts +++ b/plugins/techdocs-node/src/stages/publish/awsS3.ts @@ -152,7 +152,7 @@ export class AwsS3Publish implements PublisherBase { // AWS forcePathStyle is an optional config. If missing, it defaults to false. Needs to be enabled for cases // where endpoint url points to locally hosted S3 compatible storage like Localstack - const s3ForcePathStyle = config.getOptionalBoolean( + const forcePathStyle = config.getOptionalBoolean( 'techdocs.publisher.awsS3.s3ForcePathStyle', ); @@ -161,7 +161,7 @@ export class AwsS3Publish implements PublisherBase { credentialDefaultProvider: () => sdkCredentialProvider, ...(region && { region }), ...(endpoint && { endpoint }), - ...(s3ForcePathStyle && { s3ForcePathStyle }), + ...(forcePathStyle && { forcePathStyle }), }); const legacyPathCasing = From cb5a4d48498c3f4fbdb3f01ff7b406c50b8889dd Mon Sep 17 00:00:00 2001 From: Gabriel Testault Date: Tue, 10 Jan 2023 12:59:52 +0100 Subject: [PATCH 010/182] fix(techdocs): remove comment about implementation details Signed-off-by: Gabriel Testault --- plugins/techdocs/src/reader/transformers/scrollIntoAnchor.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/techdocs/src/reader/transformers/scrollIntoAnchor.ts b/plugins/techdocs/src/reader/transformers/scrollIntoAnchor.ts index 0be6fefc0b..97a18b7241 100644 --- a/plugins/techdocs/src/reader/transformers/scrollIntoAnchor.ts +++ b/plugins/techdocs/src/reader/transformers/scrollIntoAnchor.ts @@ -24,7 +24,6 @@ export const scrollIntoAnchor = (): Transformer => { function handleShadowDomStyleLoad() { if (window.location.hash) { const hash = window.location.hash.slice(1); - // fix invalid selector error for anchor starting with number dom?.querySelector(`[id="${hash}"]`)?.scrollIntoView(); } dom.removeEventListener( From c2df6f01927fabcb51fe6a8bed4dd3a9e0db301e Mon Sep 17 00:00:00 2001 From: Dang Huynh Date: Tue, 10 Jan 2023 08:53:28 -0600 Subject: [PATCH 011/182] Minor change to patch Signed-off-by: Dang Huynh --- .changeset/cyan-deers-walk.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/cyan-deers-walk.md b/.changeset/cyan-deers-walk.md index 2a9265090b..236eb7935b 100644 --- a/.changeset/cyan-deers-walk.md +++ b/.changeset/cyan-deers-walk.md @@ -1,5 +1,5 @@ --- -'@backstage/plugin-techdocs-node': minor +'@backstage/plugin-techdocs-node': patch --- Fixed bug caused by recent migration to AWS SDK V3 for techdocs-node. Instead of s3ForcePathStyle, forcePathStyle should be passed. From 8e025f13470a7d11a59a8173ab9b8af68814d30a Mon Sep 17 00:00:00 2001 From: Diego Bardari Date: Wed, 11 Jan 2023 16:40:16 +0100 Subject: [PATCH 012/182] Added support for externalId when assuming role in AwsS3EntityProvider Signed-off-by: Diego Bardari --- .changeset/lemon-tables-train.md | 5 +++++ .../src/credentials/AwsCredentials.ts | 2 ++ 2 files changed, 7 insertions(+) create mode 100644 .changeset/lemon-tables-train.md diff --git a/.changeset/lemon-tables-train.md b/.changeset/lemon-tables-train.md new file mode 100644 index 0000000000..39208a5edf --- /dev/null +++ b/.changeset/lemon-tables-train.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-aws': patch +--- + +Added support for externalId when assuming role in AwsS3EntityProvider diff --git a/plugins/catalog-backend-module-aws/src/credentials/AwsCredentials.ts b/plugins/catalog-backend-module-aws/src/credentials/AwsCredentials.ts index ffb84d1c34..fded78aed4 100644 --- a/plugins/catalog-backend-module-aws/src/credentials/AwsCredentials.ts +++ b/plugins/catalog-backend-module-aws/src/credentials/AwsCredentials.ts @@ -27,6 +27,7 @@ export class AwsCredentials { accessKeyId?: string; secretAccessKey?: string; roleArn?: string; + externalId?: string; }, roleSessionName: string, ): Credentials | CredentialsOptions | undefined { @@ -52,6 +53,7 @@ export class AwsCredentials { params: { RoleArn: roleArn, RoleSessionName: roleSessionName, + ExternalId: config.externalId, }, }); } From bcb0ff4f90a3bdb9a70789ec97bbdec4ee72b311 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 12 Jan 2023 08:02:29 -0800 Subject: [PATCH 013/182] Simplify type params where they aren't needed Signed-off-by: Damon Kaswell --- .../src/engine/IncrementalIngestionEngine.ts | 8 +++---- .../src/module/WrapperProviders.test.ts | 4 ++-- .../src/module/WrapperProviders.ts | 6 +++--- ...gestionEntityProviderCatalogModule.test.ts | 2 +- ...talIngestionEntityProviderCatalogModule.ts | 2 +- .../src/run.ts | 4 ++-- .../src/service/IncrementalCatalogBuilder.ts | 4 ++-- .../src/types.ts | 21 ++++++++----------- 8 files changed, 23 insertions(+), 28 deletions(-) diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts b/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts index 6213dcf786..e4d11b5f0d 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts @@ -23,7 +23,7 @@ import { v4 } from 'uuid'; import { stringifyError } from '@backstage/errors'; import { EventParams, EventSubscriber } from '@backstage/plugin-events-node'; -export class IncrementalIngestionEngine +export class IncrementalIngestionEngine implements IterationEngine, EventSubscriber { private readonly restLength: Duration; @@ -32,7 +32,7 @@ export class IncrementalIngestionEngine private manager: IncrementalIngestionDatabaseManager; - constructor(private options: IterationEngineOptions) { + constructor(private options: IterationEngineOptions) { this.manager = options.manager; this.restLength = Duration.fromObject(options.restLength); this.backoff = options.backoff ?? [ @@ -347,13 +347,11 @@ export class IncrementalIngestionEngine `incremental-engine: Received ${this.providerEventTopic} event`, ); - const payload = eventPayload as TInput; - if (!provider.deltaMapper) { return; } - const update = provider.deltaMapper(payload); + const update = provider.deltaMapper(eventPayload); if (update.delta) { if (update.delta.added.length > 0) { diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts index 0d910f664d..500d30d992 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts @@ -41,7 +41,7 @@ describe('WrapperProviders', () => { async databaseId => { const client = await databases.init(databaseId); - const provider1: IncrementalEntityProvider = { + const provider1: IncrementalEntityProvider = { getProviderName: () => 'provider1', around: burst => burst(0), next: async (_context, cursor) => { @@ -51,7 +51,7 @@ describe('WrapperProviders', () => { }, }; - const provider2: IncrementalEntityProvider = { + const provider2: IncrementalEntityProvider = { getProviderName: () => 'provider2', around: burst => burst(0), next: async (_context, cursor) => { diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts index 2308db681b..6c82928bbf 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts @@ -58,7 +58,7 @@ export class WrapperProviders { ) {} wrap( - provider: IncrementalEntityProvider, + provider: IncrementalEntityProvider, options: IncrementalEntityProviderOptions, ): EntityProvider { this.numberOfProvidersToConnect += 1; @@ -81,8 +81,8 @@ export class WrapperProviders { ).createRouter(); } - private async startProvider( - provider: IncrementalEntityProvider, + private async startProvider( + provider: IncrementalEntityProvider, providerOptions: IncrementalEntityProviderOptions, connection: EntityProviderConnection, ) { diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.test.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.test.ts index d196693db8..06dfd1467c 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.test.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.test.ts @@ -24,7 +24,7 @@ import { incrementalIngestionEntityProviderCatalogModule } from './incrementalIn describe('bitbucketServerEntityProviderCatalogModule', () => { it('should register provider at the catalog extension point', async () => { - const provider1: IncrementalEntityProvider = { + const provider1: IncrementalEntityProvider = { getProviderName: () => 'provider1', around: burst => burst(0), next: async (cursor, _context) => { diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts index 2ed7dda728..68f8bf1c90 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts @@ -38,7 +38,7 @@ export const incrementalIngestionEntityProviderCatalogModule = env, options: { providers: Array<{ - provider: IncrementalEntityProvider; + provider: IncrementalEntityProvider; options: IncrementalEntityProviderOptions; }>; }, diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/run.ts b/plugins/catalog-backend-module-incremental-ingestion/src/run.ts index 2dee5716c0..0f28dd7004 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/run.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/run.ts @@ -36,10 +36,10 @@ import { incrementalIngestionEntityProviderCatalogModule, } from '.'; -const provider: IncrementalEntityProvider = { +const provider: IncrementalEntityProvider = { getProviderName: () => 'test-provider', around: burst => burst(0), - next: async (_context, cursor) => { + next: async (_context, cursor: number | undefined) => { await new Promise(resolve => setTimeout(resolve, 500)); if (cursor === undefined || cursor < 3) { console.log(`### Returning batch #${cursor}`); diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts b/plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts index e65698d4a4..875e77a7e9 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts @@ -68,8 +68,8 @@ export class IncrementalCatalogBuilder { return { incrementalAdminRouter }; } - addIncrementalEntityProvider( - provider: IncrementalEntityProvider, + addIncrementalEntityProvider( + provider: IncrementalEntityProvider, options: IncrementalEntityProviderOptions, ) { const { burstInterval, burstLength, restLength } = options; diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/types.ts b/plugins/catalog-backend-module-incremental-ingestion/src/types.ts index f1325f5cc9..acd5371573 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/types.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/types.ts @@ -46,7 +46,7 @@ import { IncrementalIngestionDatabaseManager } from './database/IncrementalInges * * @public */ -export interface IncrementalEntityProvider { +export interface IncrementalEntityProvider { /** * This name must be unique between all of the entity providers * operating in the catalog. @@ -62,10 +62,7 @@ export interface IncrementalEntityProvider { * @returns The entities to be ingested, as well as the cursor of * the next page after this one. */ - next( - context: TContext, - cursor?: TCursor, - ): Promise>; + next(context: unknown, cursor?: unknown): Promise; /** * Do any setup and teardown necessary in order to provide the @@ -74,13 +71,13 @@ export interface IncrementalEntityProvider { * * @param burst - a function which performs a series of iterations */ - around(burst: (context: TContext) => Promise): Promise; + around(burst: (context: unknown) => Promise): Promise; /** * If present, this method maps incoming payloads to apply updates * outside of the incremental ingestion schedule. */ - deltaMapper?: (payload: TInput) => { + deltaMapper?: (payload: unknown) => { delta: | { added: DeferredEntity[]; @@ -96,16 +93,16 @@ export interface IncrementalEntityProvider { * * @public */ -export type EntityIteratorResult = +export type EntityIteratorResult = | { done: false; entities: DeferredEntity[]; - cursor: T; + cursor: unknown; } | { done: true; entities?: DeferredEntity[]; - cursor?: T; + cursor?: unknown; }; /** @public */ @@ -167,11 +164,11 @@ export interface IterationEngine { taskFn: TaskFunction; } -export interface IterationEngineOptions { +export interface IterationEngineOptions { logger: Logger; connection: EntityProviderConnection; manager: IncrementalIngestionDatabaseManager; - provider: IncrementalEntityProvider; + provider: IncrementalEntityProvider; restLength: DurationObjectUnits; ready: Promise; backoff?: IncrementalEntityProviderOptions['backoff']; From d5415fedea08d3746ed40329e6a3d8b8ceecc839 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 12 Jan 2023 08:06:32 -0800 Subject: [PATCH 014/182] Update API report Signed-off-by: Damon Kaswell --- .../api-report.md | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/plugins/catalog-backend-module-incremental-ingestion/api-report.md b/plugins/catalog-backend-module-incremental-ingestion/api-report.md index c56ae0e7bf..aa526a8fe8 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/api-report.md +++ b/plugins/catalog-backend-module-incremental-ingestion/api-report.md @@ -18,23 +18,23 @@ import { Router } from 'express'; import type { UrlReader } from '@backstage/backend-common'; // @public -export type EntityIteratorResult = +export type EntityIteratorResult = | { done: false; entities: DeferredEntity[]; - cursor: T; + cursor: unknown; } | { done: true; entities?: DeferredEntity[]; - cursor?: T; + cursor?: unknown; }; // @public (undocumented) export class IncrementalCatalogBuilder { // (undocumented) - addIncrementalEntityProvider( - provider: IncrementalEntityProvider, + addIncrementalEntityProvider( + provider: IncrementalEntityProvider, options: IncrementalEntityProviderOptions, ): void; // (undocumented) @@ -48,9 +48,9 @@ export class IncrementalCatalogBuilder { } // @public -export interface IncrementalEntityProvider { - around(burst: (context: TContext) => Promise): Promise; - deltaMapper?: (payload: TInput) => { +export interface IncrementalEntityProvider { + around(burst: (context: unknown) => Promise): Promise; + deltaMapper?: (payload: unknown) => { delta: | { added: DeferredEntity[]; @@ -61,10 +61,7 @@ export interface IncrementalEntityProvider { | undefined; }; getProviderName(): string; - next( - context: TContext, - cursor?: TCursor, - ): Promise>; + next(context: unknown, cursor?: unknown): Promise; } // @public (undocumented) @@ -80,7 +77,7 @@ export interface IncrementalEntityProviderOptions { // @alpha export const incrementalIngestionEntityProviderCatalogModule: (options: { providers: { - provider: IncrementalEntityProvider; + provider: IncrementalEntityProvider; options: IncrementalEntityProviderOptions; }[]; }) => BackendFeature; From 66d70c7d3c8450c3e2efd675c2fdba87ec1a4b3b Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 12 Jan 2023 09:38:14 -0800 Subject: [PATCH 015/182] Revert "Simplify type params where they aren't needed" This reverts commit bcb0ff4f90a3bdb9a70789ec97bbdec4ee72b311. Signed-off-by: Damon Kaswell --- .../src/engine/IncrementalIngestionEngine.ts | 8 ++++--- .../src/module/WrapperProviders.test.ts | 4 ++-- .../src/module/WrapperProviders.ts | 6 +++--- ...gestionEntityProviderCatalogModule.test.ts | 2 +- ...talIngestionEntityProviderCatalogModule.ts | 2 +- .../src/run.ts | 4 ++-- .../src/service/IncrementalCatalogBuilder.ts | 4 ++-- .../src/types.ts | 21 +++++++++++-------- 8 files changed, 28 insertions(+), 23 deletions(-) diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts b/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts index e4d11b5f0d..6213dcf786 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts @@ -23,7 +23,7 @@ import { v4 } from 'uuid'; import { stringifyError } from '@backstage/errors'; import { EventParams, EventSubscriber } from '@backstage/plugin-events-node'; -export class IncrementalIngestionEngine +export class IncrementalIngestionEngine implements IterationEngine, EventSubscriber { private readonly restLength: Duration; @@ -32,7 +32,7 @@ export class IncrementalIngestionEngine private manager: IncrementalIngestionDatabaseManager; - constructor(private options: IterationEngineOptions) { + constructor(private options: IterationEngineOptions) { this.manager = options.manager; this.restLength = Duration.fromObject(options.restLength); this.backoff = options.backoff ?? [ @@ -347,11 +347,13 @@ export class IncrementalIngestionEngine `incremental-engine: Received ${this.providerEventTopic} event`, ); + const payload = eventPayload as TInput; + if (!provider.deltaMapper) { return; } - const update = provider.deltaMapper(eventPayload); + const update = provider.deltaMapper(payload); if (update.delta) { if (update.delta.added.length > 0) { diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts index 500d30d992..0d910f664d 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.test.ts @@ -41,7 +41,7 @@ describe('WrapperProviders', () => { async databaseId => { const client = await databases.init(databaseId); - const provider1: IncrementalEntityProvider = { + const provider1: IncrementalEntityProvider = { getProviderName: () => 'provider1', around: burst => burst(0), next: async (_context, cursor) => { @@ -51,7 +51,7 @@ describe('WrapperProviders', () => { }, }; - const provider2: IncrementalEntityProvider = { + const provider2: IncrementalEntityProvider = { getProviderName: () => 'provider2', around: burst => burst(0), next: async (_context, cursor) => { diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts index 6c82928bbf..2308db681b 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts @@ -58,7 +58,7 @@ export class WrapperProviders { ) {} wrap( - provider: IncrementalEntityProvider, + provider: IncrementalEntityProvider, options: IncrementalEntityProviderOptions, ): EntityProvider { this.numberOfProvidersToConnect += 1; @@ -81,8 +81,8 @@ export class WrapperProviders { ).createRouter(); } - private async startProvider( - provider: IncrementalEntityProvider, + private async startProvider( + provider: IncrementalEntityProvider, providerOptions: IncrementalEntityProviderOptions, connection: EntityProviderConnection, ) { diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.test.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.test.ts index 06dfd1467c..d196693db8 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.test.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.test.ts @@ -24,7 +24,7 @@ import { incrementalIngestionEntityProviderCatalogModule } from './incrementalIn describe('bitbucketServerEntityProviderCatalogModule', () => { it('should register provider at the catalog extension point', async () => { - const provider1: IncrementalEntityProvider = { + const provider1: IncrementalEntityProvider = { getProviderName: () => 'provider1', around: burst => burst(0), next: async (cursor, _context) => { diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts index 68f8bf1c90..2ed7dda728 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts @@ -38,7 +38,7 @@ export const incrementalIngestionEntityProviderCatalogModule = env, options: { providers: Array<{ - provider: IncrementalEntityProvider; + provider: IncrementalEntityProvider; options: IncrementalEntityProviderOptions; }>; }, diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/run.ts b/plugins/catalog-backend-module-incremental-ingestion/src/run.ts index 0f28dd7004..2dee5716c0 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/run.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/run.ts @@ -36,10 +36,10 @@ import { incrementalIngestionEntityProviderCatalogModule, } from '.'; -const provider: IncrementalEntityProvider = { +const provider: IncrementalEntityProvider = { getProviderName: () => 'test-provider', around: burst => burst(0), - next: async (_context, cursor: number | undefined) => { + next: async (_context, cursor) => { await new Promise(resolve => setTimeout(resolve, 500)); if (cursor === undefined || cursor < 3) { console.log(`### Returning batch #${cursor}`); diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts b/plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts index 875e77a7e9..e65698d4a4 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts @@ -68,8 +68,8 @@ export class IncrementalCatalogBuilder { return { incrementalAdminRouter }; } - addIncrementalEntityProvider( - provider: IncrementalEntityProvider, + addIncrementalEntityProvider( + provider: IncrementalEntityProvider, options: IncrementalEntityProviderOptions, ) { const { burstInterval, burstLength, restLength } = options; diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/types.ts b/plugins/catalog-backend-module-incremental-ingestion/src/types.ts index acd5371573..f1325f5cc9 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/types.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/types.ts @@ -46,7 +46,7 @@ import { IncrementalIngestionDatabaseManager } from './database/IncrementalInges * * @public */ -export interface IncrementalEntityProvider { +export interface IncrementalEntityProvider { /** * This name must be unique between all of the entity providers * operating in the catalog. @@ -62,7 +62,10 @@ export interface IncrementalEntityProvider { * @returns The entities to be ingested, as well as the cursor of * the next page after this one. */ - next(context: unknown, cursor?: unknown): Promise; + next( + context: TContext, + cursor?: TCursor, + ): Promise>; /** * Do any setup and teardown necessary in order to provide the @@ -71,13 +74,13 @@ export interface IncrementalEntityProvider { * * @param burst - a function which performs a series of iterations */ - around(burst: (context: unknown) => Promise): Promise; + around(burst: (context: TContext) => Promise): Promise; /** * If present, this method maps incoming payloads to apply updates * outside of the incremental ingestion schedule. */ - deltaMapper?: (payload: unknown) => { + deltaMapper?: (payload: TInput) => { delta: | { added: DeferredEntity[]; @@ -93,16 +96,16 @@ export interface IncrementalEntityProvider { * * @public */ -export type EntityIteratorResult = +export type EntityIteratorResult = | { done: false; entities: DeferredEntity[]; - cursor: unknown; + cursor: T; } | { done: true; entities?: DeferredEntity[]; - cursor?: unknown; + cursor?: T; }; /** @public */ @@ -164,11 +167,11 @@ export interface IterationEngine { taskFn: TaskFunction; } -export interface IterationEngineOptions { +export interface IterationEngineOptions { logger: Logger; connection: EntityProviderConnection; manager: IncrementalIngestionDatabaseManager; - provider: IncrementalEntityProvider; + provider: IncrementalEntityProvider; restLength: DurationObjectUnits; ready: Promise; backoff?: IncrementalEntityProviderOptions['backoff']; From a319a5977408f8cee44f81548c2313df4e7569ce Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 12 Jan 2023 09:54:40 -0800 Subject: [PATCH 016/182] One more try on types Signed-off-by: Damon Kaswell --- .../api-report.md | 21 +++++++++++-------- .../src/engine/IncrementalIngestionEngine.ts | 8 +++---- .../src/module/WrapperProviders.ts | 4 ++-- ...gestionEntityProviderCatalogModule.test.ts | 2 +- ...talIngestionEntityProviderCatalogModule.ts | 2 +- .../src/run.ts | 2 +- .../src/service/IncrementalCatalogBuilder.ts | 4 ++-- .../src/types.ts | 8 +++---- 8 files changed, 26 insertions(+), 25 deletions(-) diff --git a/plugins/catalog-backend-module-incremental-ingestion/api-report.md b/plugins/catalog-backend-module-incremental-ingestion/api-report.md index aa526a8fe8..c79d07ad26 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/api-report.md +++ b/plugins/catalog-backend-module-incremental-ingestion/api-report.md @@ -18,23 +18,23 @@ import { Router } from 'express'; import type { UrlReader } from '@backstage/backend-common'; // @public -export type EntityIteratorResult = +export type EntityIteratorResult = | { done: false; entities: DeferredEntity[]; - cursor: unknown; + cursor: T; } | { done: true; entities?: DeferredEntity[]; - cursor?: unknown; + cursor?: T; }; // @public (undocumented) export class IncrementalCatalogBuilder { // (undocumented) - addIncrementalEntityProvider( - provider: IncrementalEntityProvider, + addIncrementalEntityProvider( + provider: IncrementalEntityProvider, options: IncrementalEntityProviderOptions, ): void; // (undocumented) @@ -48,8 +48,8 @@ export class IncrementalCatalogBuilder { } // @public -export interface IncrementalEntityProvider { - around(burst: (context: unknown) => Promise): Promise; +export interface IncrementalEntityProvider { + around(burst: (context: TContext) => Promise): Promise; deltaMapper?: (payload: unknown) => { delta: | { @@ -61,7 +61,10 @@ export interface IncrementalEntityProvider { | undefined; }; getProviderName(): string; - next(context: unknown, cursor?: unknown): Promise; + next( + context: TContext, + cursor?: TCursor, + ): Promise>; } // @public (undocumented) @@ -77,7 +80,7 @@ export interface IncrementalEntityProviderOptions { // @alpha export const incrementalIngestionEntityProviderCatalogModule: (options: { providers: { - provider: IncrementalEntityProvider; + provider: IncrementalEntityProvider; options: IncrementalEntityProviderOptions; }[]; }) => BackendFeature; diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts b/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts index 6213dcf786..e4d11b5f0d 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts @@ -23,7 +23,7 @@ import { v4 } from 'uuid'; import { stringifyError } from '@backstage/errors'; import { EventParams, EventSubscriber } from '@backstage/plugin-events-node'; -export class IncrementalIngestionEngine +export class IncrementalIngestionEngine implements IterationEngine, EventSubscriber { private readonly restLength: Duration; @@ -32,7 +32,7 @@ export class IncrementalIngestionEngine private manager: IncrementalIngestionDatabaseManager; - constructor(private options: IterationEngineOptions) { + constructor(private options: IterationEngineOptions) { this.manager = options.manager; this.restLength = Duration.fromObject(options.restLength); this.backoff = options.backoff ?? [ @@ -347,13 +347,11 @@ export class IncrementalIngestionEngine `incremental-engine: Received ${this.providerEventTopic} event`, ); - const payload = eventPayload as TInput; - if (!provider.deltaMapper) { return; } - const update = provider.deltaMapper(payload); + const update = provider.deltaMapper(eventPayload); if (update.delta) { if (update.delta.added.length > 0) { diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts index 2308db681b..e714568c4b 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/WrapperProviders.ts @@ -81,8 +81,8 @@ export class WrapperProviders { ).createRouter(); } - private async startProvider( - provider: IncrementalEntityProvider, + private async startProvider( + provider: IncrementalEntityProvider, providerOptions: IncrementalEntityProviderOptions, connection: EntityProviderConnection, ) { diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.test.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.test.ts index d196693db8..06cbcb010e 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.test.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.test.ts @@ -24,7 +24,7 @@ import { incrementalIngestionEntityProviderCatalogModule } from './incrementalIn describe('bitbucketServerEntityProviderCatalogModule', () => { it('should register provider at the catalog extension point', async () => { - const provider1: IncrementalEntityProvider = { + const provider1: IncrementalEntityProvider = { getProviderName: () => 'provider1', around: burst => burst(0), next: async (cursor, _context) => { diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts index 2ed7dda728..0ecb45292c 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/module/incrementalIngestionEntityProviderCatalogModule.ts @@ -38,7 +38,7 @@ export const incrementalIngestionEntityProviderCatalogModule = env, options: { providers: Array<{ - provider: IncrementalEntityProvider; + provider: IncrementalEntityProvider; options: IncrementalEntityProviderOptions; }>; }, diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/run.ts b/plugins/catalog-backend-module-incremental-ingestion/src/run.ts index 2dee5716c0..08d95f4417 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/run.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/run.ts @@ -36,7 +36,7 @@ import { incrementalIngestionEntityProviderCatalogModule, } from '.'; -const provider: IncrementalEntityProvider = { +const provider: IncrementalEntityProvider = { getProviderName: () => 'test-provider', around: burst => burst(0), next: async (_context, cursor) => { diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts b/plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts index e65698d4a4..6d2b3e2be5 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/service/IncrementalCatalogBuilder.ts @@ -68,8 +68,8 @@ export class IncrementalCatalogBuilder { return { incrementalAdminRouter }; } - addIncrementalEntityProvider( - provider: IncrementalEntityProvider, + addIncrementalEntityProvider( + provider: IncrementalEntityProvider, options: IncrementalEntityProviderOptions, ) { const { burstInterval, burstLength, restLength } = options; diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/types.ts b/plugins/catalog-backend-module-incremental-ingestion/src/types.ts index f1325f5cc9..24919e2647 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/types.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/types.ts @@ -46,7 +46,7 @@ import { IncrementalIngestionDatabaseManager } from './database/IncrementalInges * * @public */ -export interface IncrementalEntityProvider { +export interface IncrementalEntityProvider { /** * This name must be unique between all of the entity providers * operating in the catalog. @@ -80,7 +80,7 @@ export interface IncrementalEntityProvider { * If present, this method maps incoming payloads to apply updates * outside of the incremental ingestion schedule. */ - deltaMapper?: (payload: TInput) => { + deltaMapper?: (payload: unknown) => { delta: | { added: DeferredEntity[]; @@ -167,11 +167,11 @@ export interface IterationEngine { taskFn: TaskFunction; } -export interface IterationEngineOptions { +export interface IterationEngineOptions { logger: Logger; connection: EntityProviderConnection; manager: IncrementalIngestionDatabaseManager; - provider: IncrementalEntityProvider; + provider: IncrementalEntityProvider; restLength: DurationObjectUnits; ready: Promise; backoff?: IncrementalEntityProviderOptions['backoff']; From b7154302eac7f95c7e2ea346b80a8ab494e3d080 Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Thu, 12 Jan 2023 10:37:16 -0800 Subject: [PATCH 017/182] Fix typo Signed-off-by: Damon Kaswell --- .../src/engine/IncrementalIngestionEngine.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts b/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts index e4d11b5f0d..311af0f6de 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts @@ -41,7 +41,7 @@ export class IncrementalIngestionEngine { minutes: 30 }, { hours: 3 }, ]; - this.providerEventTopic = `${options.provider.getProviderName()}-delta`; + this.providerEventTopic = `${options.provider.getProviderName()}-push`; } async taskFn(signal: AbortSignal) { From c01b1f0ac527f4f76da29acca8b98846d09f00be Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Fri, 13 Jan 2023 08:33:46 -0800 Subject: [PATCH 018/182] Rename 'deltaMapper' to 'onEvent' Signed-off-by: Damon Kaswell --- .../api-report.md | 12 ++++++------ .../src/engine/IncrementalIngestionEngine.ts | 4 ++-- .../src/types.ts | 9 ++++++--- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/plugins/catalog-backend-module-incremental-ingestion/api-report.md b/plugins/catalog-backend-module-incremental-ingestion/api-report.md index c79d07ad26..1052afda74 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/api-report.md +++ b/plugins/catalog-backend-module-incremental-ingestion/api-report.md @@ -50,7 +50,12 @@ export class IncrementalCatalogBuilder { // @public export interface IncrementalEntityProvider { around(burst: (context: TContext) => Promise): Promise; - deltaMapper?: (payload: unknown) => { + getProviderName(): string; + next( + context: TContext, + cursor?: TCursor, + ): Promise>; + onEvent?: (payload: unknown) => { delta: | { added: DeferredEntity[]; @@ -60,11 +65,6 @@ export interface IncrementalEntityProvider { } | undefined; }; - getProviderName(): string; - next( - context: TContext, - cursor?: TCursor, - ): Promise>; } // @public (undocumented) diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts b/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts index 311af0f6de..a21092a838 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts @@ -347,11 +347,11 @@ export class IncrementalIngestionEngine `incremental-engine: Received ${this.providerEventTopic} event`, ); - if (!provider.deltaMapper) { + if (!provider.onEvent) { return; } - const update = provider.deltaMapper(eventPayload); + const update = provider.onEvent(eventPayload); if (update.delta) { if (update.delta.added.length > 0) { diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/types.ts b/plugins/catalog-backend-module-incremental-ingestion/src/types.ts index 24919e2647..315757d88a 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/types.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/types.ts @@ -77,10 +77,13 @@ export interface IncrementalEntityProvider { around(burst: (context: TContext) => Promise): Promise; /** - * If present, this method maps incoming payloads to apply updates - * outside of the incremental ingestion schedule. + * If present, this method accepts an incoming event, and maps the + * payload to an object containing a delta mutation. + * + * If a delta is present, the incremental entity provider will apply + * it automatically. */ - deltaMapper?: (payload: unknown) => { + onEvent?: (payload: unknown) => { delta: | { added: DeferredEntity[]; From aadbec389bb956b53c582b402802e951e6f40412 Mon Sep 17 00:00:00 2001 From: Sarah Medeiros Date: Fri, 13 Jan 2023 12:01:36 -0500 Subject: [PATCH 019/182] Add initial filter prop for lifecycle picker Signed-off-by: Sarah Medeiros --- .../EntityLifecyclePicker.test.tsx | 17 +++++++++++++++++ .../EntityLifecyclePicker.tsx | 10 ++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.test.tsx b/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.test.tsx index ef77f7615f..3ee610ddaf 100644 --- a/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.test.tsx +++ b/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.test.tsx @@ -210,4 +210,21 @@ describe('', () => { lifecycles: undefined, }); }); + it('responds to initialFilter prop', () => { + const updateFilters = jest.fn(); + render( + + + , + ); + expect(updateFilters).toHaveBeenLastCalledWith({ + lifecycles: new EntityLifecycleFilter(['production']), + }); + }); }); diff --git a/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.tsx b/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.tsx index b941f94531..913e800ec0 100644 --- a/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.tsx +++ b/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.tsx @@ -47,7 +47,13 @@ const icon = ; const checkedIcon = ; /** @public */ -export const EntityLifecyclePicker = () => { +export type EntityLifecyclePickerProps = { + initialFilter?: string[]; +}; + +/** @public */ +export const EntityLifecyclePicker = (props: EntityLifecyclePickerProps) => { + const { initialFilter } = props; const classes = useStyles(); const { updateFilters, @@ -64,7 +70,7 @@ export const EntityLifecyclePicker = () => { const [selectedLifecycles, setSelectedLifecycles] = useState( queryParamLifecycles.length ? queryParamLifecycles - : filters.lifecycles?.values ?? [], + : filters.lifecycles?.values ?? (initialFilter || []), ); // Set selected lifecycles on query parameter updates; this happens at initial page load and from From 929e1afe1b45621c1e0b13c059fbbdc989f59b63 Mon Sep 17 00:00:00 2001 From: Sarah Medeiros Date: Fri, 13 Jan 2023 12:35:05 -0500 Subject: [PATCH 020/182] Add changelog Signed-off-by: Sarah Medeiros --- .changeset/rare-melons-battle.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/rare-melons-battle.md diff --git a/.changeset/rare-melons-battle.md b/.changeset/rare-melons-battle.md new file mode 100644 index 0000000000..0b13793409 --- /dev/null +++ b/.changeset/rare-melons-battle.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-react': patch +--- + +Add initialFilter prop to EntityLifecyclePicker. This allows you to set an initial lifecycle for the catalog. From f1e0357974538f15835c58b2ca260c2aacbdc3ed Mon Sep 17 00:00:00 2001 From: Sarah Medeiros Date: Fri, 13 Jan 2023 12:55:04 -0500 Subject: [PATCH 021/182] Update API report. Simplify picker. Signed-off-by: Sarah Medeiros --- plugins/catalog-react/api-report.md | 9 ++++++++- .../EntityLifecyclePicker/EntityLifecyclePicker.tsx | 4 ++-- .../src/components/EntityLifecyclePicker/index.ts | 5 ++++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/plugins/catalog-react/api-report.md b/plugins/catalog-react/api-report.md index 9528c55326..48153a9c78 100644 --- a/plugins/catalog-react/api-report.md +++ b/plugins/catalog-react/api-report.md @@ -194,7 +194,14 @@ export class EntityLifecycleFilter implements EntityFilter { } // @public (undocumented) -export const EntityLifecyclePicker: () => JSX.Element | null; +export const EntityLifecyclePicker: ( + props: EntityLifecyclePickerProps, +) => JSX.Element | null; + +// @public (undocumented) +export type EntityLifecyclePickerProps = { + initialFilter?: string[]; +}; // @public export const EntityListContext: React_2.Context< diff --git a/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.tsx b/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.tsx index 913e800ec0..3ec0bf8497 100644 --- a/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.tsx +++ b/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.tsx @@ -53,7 +53,7 @@ export type EntityLifecyclePickerProps = { /** @public */ export const EntityLifecyclePicker = (props: EntityLifecyclePickerProps) => { - const { initialFilter } = props; + const { initialFilter = [] } = props; const classes = useStyles(); const { updateFilters, @@ -70,7 +70,7 @@ export const EntityLifecyclePicker = (props: EntityLifecyclePickerProps) => { const [selectedLifecycles, setSelectedLifecycles] = useState( queryParamLifecycles.length ? queryParamLifecycles - : filters.lifecycles?.values ?? (initialFilter || []), + : filters.lifecycles?.values ?? initialFilter, ); // Set selected lifecycles on query parameter updates; this happens at initial page load and from diff --git a/plugins/catalog-react/src/components/EntityLifecyclePicker/index.ts b/plugins/catalog-react/src/components/EntityLifecyclePicker/index.ts index def25eabf4..b1e2d73080 100644 --- a/plugins/catalog-react/src/components/EntityLifecyclePicker/index.ts +++ b/plugins/catalog-react/src/components/EntityLifecyclePicker/index.ts @@ -15,4 +15,7 @@ */ export { EntityLifecyclePicker } from './EntityLifecyclePicker'; -export type { CatalogReactEntityLifecyclePickerClassKey } from './EntityLifecyclePicker'; +export type { + CatalogReactEntityLifecyclePickerClassKey, + EntityLifecyclePickerProps, +} from './EntityLifecyclePicker'; From 0641029495c36cb4787fbff0c1541c550cc0c80a Mon Sep 17 00:00:00 2001 From: Sarah Medeiros Date: Fri, 13 Jan 2023 12:55:35 -0500 Subject: [PATCH 022/182] Update changelog message Signed-off-by: Sarah Medeiros --- .changeset/rare-melons-battle.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/rare-melons-battle.md b/.changeset/rare-melons-battle.md index 0b13793409..2cf096fad8 100644 --- a/.changeset/rare-melons-battle.md +++ b/.changeset/rare-melons-battle.md @@ -2,4 +2,4 @@ '@backstage/plugin-catalog-react': patch --- -Add initialFilter prop to EntityLifecyclePicker. This allows you to set an initial lifecycle for the catalog. +Add `initialFilter` prop to EntityLifecyclePicker. This allows you to set an initial lifecycle for the catalog. From c1fed63004b143c7be8084b5dd67643e3ad9a893 Mon Sep 17 00:00:00 2001 From: Sarah Medeiros Date: Fri, 13 Jan 2023 15:06:21 -0500 Subject: [PATCH 023/182] Define type inline & update api-report Signed-off-by: Sarah Medeiros --- plugins/catalog-react/api-report.md | 9 ++------- .../EntityLifecyclePicker/EntityLifecyclePicker.tsx | 7 +------ .../src/components/EntityLifecyclePicker/index.ts | 5 +---- 3 files changed, 4 insertions(+), 17 deletions(-) diff --git a/plugins/catalog-react/api-report.md b/plugins/catalog-react/api-report.md index 48153a9c78..095b6832a4 100644 --- a/plugins/catalog-react/api-report.md +++ b/plugins/catalog-react/api-report.md @@ -194,14 +194,9 @@ export class EntityLifecycleFilter implements EntityFilter { } // @public (undocumented) -export const EntityLifecyclePicker: ( - props: EntityLifecyclePickerProps, -) => JSX.Element | null; - -// @public (undocumented) -export type EntityLifecyclePickerProps = { +export const EntityLifecyclePicker: (props: { initialFilter?: string[]; -}; +}) => JSX.Element | null; // @public export const EntityListContext: React_2.Context< diff --git a/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.tsx b/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.tsx index 3ec0bf8497..77925598e1 100644 --- a/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.tsx +++ b/plugins/catalog-react/src/components/EntityLifecyclePicker/EntityLifecyclePicker.tsx @@ -47,12 +47,7 @@ const icon = ; const checkedIcon = ; /** @public */ -export type EntityLifecyclePickerProps = { - initialFilter?: string[]; -}; - -/** @public */ -export const EntityLifecyclePicker = (props: EntityLifecyclePickerProps) => { +export const EntityLifecyclePicker = (props: { initialFilter?: string[] }) => { const { initialFilter = [] } = props; const classes = useStyles(); const { diff --git a/plugins/catalog-react/src/components/EntityLifecyclePicker/index.ts b/plugins/catalog-react/src/components/EntityLifecyclePicker/index.ts index b1e2d73080..def25eabf4 100644 --- a/plugins/catalog-react/src/components/EntityLifecyclePicker/index.ts +++ b/plugins/catalog-react/src/components/EntityLifecyclePicker/index.ts @@ -15,7 +15,4 @@ */ export { EntityLifecyclePicker } from './EntityLifecyclePicker'; -export type { - CatalogReactEntityLifecyclePickerClassKey, - EntityLifecyclePickerProps, -} from './EntityLifecyclePicker'; +export type { CatalogReactEntityLifecyclePickerClassKey } from './EntityLifecyclePicker'; From 6e0b6a0d509113d032a41c06927d7d2c9ad42837 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Albin=20Halinen=20Wil=C3=A9n?= Date: Fri, 13 Jan 2023 15:36:52 +0800 Subject: [PATCH 024/182] Fixed techdocs-cli package publish CLI command missing awsBucketRootPath option MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Albin Halinen Wilén --- .changeset/quick-ladybugs-reply.md | 5 +++++ packages/techdocs-cli/cli-report.md | 2 +- packages/techdocs-cli/src/commands/index.ts | 8 ++++---- 3 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 .changeset/quick-ladybugs-reply.md diff --git a/.changeset/quick-ladybugs-reply.md b/.changeset/quick-ladybugs-reply.md new file mode 100644 index 0000000000..0e4f040b5d --- /dev/null +++ b/.changeset/quick-ladybugs-reply.md @@ -0,0 +1,5 @@ +--- +'@techdocs/cli': patch +--- + +Fixed publish command missing awsBucketRootPath option. diff --git a/packages/techdocs-cli/cli-report.md b/packages/techdocs-cli/cli-report.md index 9230fe9f1c..b5623b5cef 100644 --- a/packages/techdocs-cli/cli-report.md +++ b/packages/techdocs-cli/cli-report.md @@ -52,7 +52,6 @@ Options: --awsRoleArn --awsEndpoint --awsS3ForcePathStyle - --awsBucketRootPath --osCredentialId --osSecret --osAuthUrl @@ -79,6 +78,7 @@ Options: --awsEndpoint --awsS3sse --awsS3ForcePathStyle + --awsBucketRootPath --osCredentialId --osSecret --osAuthUrl diff --git a/packages/techdocs-cli/src/commands/index.ts b/packages/techdocs-cli/src/commands/index.ts index d4303ad306..894878c6b7 100644 --- a/packages/techdocs-cli/src/commands/index.ts +++ b/packages/techdocs-cli/src/commands/index.ts @@ -101,10 +101,6 @@ export function registerCommands(program: Command) { '--awsS3ForcePathStyle', 'Optional AWS S3 option to force path style.', ) - .option( - '--awsBucketRootPath', - 'Optional sub-directory to store files in Amazon S3', - ) .option( '--osCredentialId ', '(Required for OpenStack) specify when --publisher-type openStackSwift', @@ -177,6 +173,10 @@ export function registerCommands(program: Command) { '--awsS3ForcePathStyle', 'Optional AWS S3 option to force path style.', ) + .option( + '--awsBucketRootPath ', + 'Optional sub-directory to store files in Amazon S3', + ) .option( '--osCredentialId ', '(Required for OpenStack) specify when --publisher-type openStackSwift', From f39c417bbbed0226e5c12812c37fce3a4d671985 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Albin=20Halinen=20Wil=C3=A9n?= Date: Mon, 16 Jan 2023 09:55:03 +0800 Subject: [PATCH 025/182] Fixed techdocs-cli package publish CLI command having the gcsBucketRootPath option misconfigured, previously returning a boolean vs a string MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Albin Halinen Wilén --- .changeset/quick-ladybugs-reply.md | 1 + packages/techdocs-cli/cli-report.md | 2 +- packages/techdocs-cli/src/commands/index.ts | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.changeset/quick-ladybugs-reply.md b/.changeset/quick-ladybugs-reply.md index 0e4f040b5d..971f6f635b 100644 --- a/.changeset/quick-ladybugs-reply.md +++ b/.changeset/quick-ladybugs-reply.md @@ -3,3 +3,4 @@ --- Fixed publish command missing awsBucketRootPath option. +Fixed publish command having the gcsBucketRootPath option misconfigured, previously returning a boolean vs a string. diff --git a/packages/techdocs-cli/cli-report.md b/packages/techdocs-cli/cli-report.md index b5623b5cef..861dad4605 100644 --- a/packages/techdocs-cli/cli-report.md +++ b/packages/techdocs-cli/cli-report.md @@ -83,7 +83,7 @@ Options: --osSecret --osAuthUrl --osSwiftUrl - --gcsBucketRootPath + --gcsBucketRootPath --directory -h, --help ``` diff --git a/packages/techdocs-cli/src/commands/index.ts b/packages/techdocs-cli/src/commands/index.ts index 894878c6b7..caa7d8efbd 100644 --- a/packages/techdocs-cli/src/commands/index.ts +++ b/packages/techdocs-cli/src/commands/index.ts @@ -194,7 +194,7 @@ export function registerCommands(program: Command) { '(Required for OpenStack) specify when --publisher-type openStackSwift', ) .option( - '--gcsBucketRootPath', + '--gcsBucketRootPath ', 'Optional sub-directory to store files in Google cloud storage', ) .option( From ef8f19256fad88188411c59652895302ae0c28d6 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Mon, 16 Jan 2023 10:58:31 +0000 Subject: [PATCH 026/182] surface the cause of the json rules engine Signed-off-by: Brian Fletcher --- .../src/service/JsonRulesEngineFactChecker.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.ts b/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.ts index 8866b5badb..e29494be8b 100644 --- a/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.ts +++ b/plugins/tech-insights-backend-module-jsonfc/src/service/JsonRulesEngineFactChecker.ts @@ -141,7 +141,9 @@ export class JsonRulesEngineFactChecker ); } catch (e) { if (isError(e)) { - throw new Error(`Failed to run rules engine, ${e.message}`); + throw new Error(`Failed to run rules engine, ${e.message}`, { + cause: e, + }); } throw e; } From d6b912f963069a7e5d8a396d31921ab3dbb807e2 Mon Sep 17 00:00:00 2001 From: Brian Fletcher Date: Mon, 16 Jan 2023 11:02:28 +0000 Subject: [PATCH 027/182] add changeset Signed-off-by: Brian Fletcher --- .changeset/lazy-badgers-rule.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/lazy-badgers-rule.md diff --git a/.changeset/lazy-badgers-rule.md b/.changeset/lazy-badgers-rule.md new file mode 100644 index 0000000000..748b9dbadc --- /dev/null +++ b/.changeset/lazy-badgers-rule.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-tech-insights-backend-module-jsonfc': patch +--- + +Surface the cause of the json rules engine From cb167af97da67de6712deddd1ab6e59377ec5c74 Mon Sep 17 00:00:00 2001 From: Sarah Medeiros Date: Mon, 16 Jan 2023 13:16:29 -0500 Subject: [PATCH 028/182] Update .changeset/rare-melons-battle.md Co-authored-by: Johan Haals Signed-off-by: Sarah Medeiros --- .changeset/rare-melons-battle.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/rare-melons-battle.md b/.changeset/rare-melons-battle.md index 2cf096fad8..85fd72c763 100644 --- a/.changeset/rare-melons-battle.md +++ b/.changeset/rare-melons-battle.md @@ -1,5 +1,5 @@ --- -'@backstage/plugin-catalog-react': patch +'@backstage/plugin-catalog-react': minor --- Add `initialFilter` prop to EntityLifecyclePicker. This allows you to set an initial lifecycle for the catalog. From 516b5dcfae61f1cc583df05cc43fa9a023c58a7a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 17 Jan 2023 00:10:26 +0000 Subject: [PATCH 029/182] fix(deps): update dependency eslint-plugin-react to v7.32.1 Signed-off-by: Renovate Bot --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index b76056c0d6..821f7a5a17 100644 --- a/yarn.lock +++ b/yarn.lock @@ -22123,8 +22123,8 @@ __metadata: linkType: hard "eslint-plugin-react@npm:^7.28.0": - version: 7.32.0 - resolution: "eslint-plugin-react@npm:7.32.0" + version: 7.32.1 + resolution: "eslint-plugin-react@npm:7.32.1" dependencies: array-includes: ^3.1.6 array.prototype.flatmap: ^1.3.1 @@ -22143,7 +22143,7 @@ __metadata: string.prototype.matchall: ^4.0.8 peerDependencies: eslint: ^3 || ^4 || ^5 || ^6 || ^7 || ^8 - checksum: b81ce2623b50a936287d8e21997bd855094e643856c99b42a9f0c10e1c7b123e469c3d75f77df9eefb719fee2b47a763862f1cdca1e7cc26edc7cde2fb8cba87 + checksum: e20eab61161a3db6211c2bd1eb9be3e407fd14e72c06c5f39a078b6ac37427b2af6056ee70e3954249bca0a04088ae797a0c8ba909fb8802e29712de2a41262d languageName: node linkType: hard From ce38f17255f9fb8b9f9ed04c938638b94df0857c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 16 Jan 2023 15:23:07 +0100 Subject: [PATCH 030/182] backend-test-utils: refactor to export mockServices and mockFactories Signed-off-by: Patrik Oldsberg --- packages/backend-test-utils/api-report.md | 27 ++++++++++---- .../src/next/implementations/index.ts | 3 +- .../src/next/implementations/mockFactories.ts | 36 +++++++++++++++++++ .../{mockConfigService.ts => mockServices.ts} | 28 +++++++-------- .../src/next/wiring/TestBackend.ts | 4 +-- 5 files changed, 73 insertions(+), 25 deletions(-) create mode 100644 packages/backend-test-utils/src/next/implementations/mockFactories.ts rename packages/backend-test-utils/src/next/implementations/{mockConfigService.ts => mockServices.ts} (63%) diff --git a/packages/backend-test-utils/api-report.md b/packages/backend-test-utils/api-report.md index 8ff29f8545..ffcc0f3d35 100644 --- a/packages/backend-test-utils/api-report.md +++ b/packages/backend-test-utils/api-report.md @@ -5,6 +5,7 @@ ```ts import { Backend } from '@backstage/backend-app-api'; import { BackendFeature } from '@backstage/backend-plugin-api'; +import { ConfigReader } from '@backstage/config'; import { ConfigService } from '@backstage/backend-plugin-api'; import { ExtendedHttpServer } from '@backstage/backend-app-api'; import { ExtensionPoint } from '@backstage/backend-plugin-api'; @@ -17,13 +18,25 @@ import { ServiceRef } from '@backstage/backend-plugin-api'; export function isDockerDisabledForTests(): boolean; // @alpha (undocumented) -export const mockConfigFactory: ( - options?: - | { - data?: JsonObject | undefined; - } - | undefined, -) => ServiceFactory; +export namespace mockFactories { + const // (undocumented) + config: ( + options?: mockServices.config.Options | undefined, + ) => ServiceFactory; +} + +// @alpha (undocumented) +export namespace mockServices { + // (undocumented) + export namespace config { + // (undocumented) + export type Options = { + data?: JsonObject; + }; + } + // (undocumented) + export function config(options?: config.Options): ConfigReader; +} // @public export function setupRequestMockHandlers(worker: { diff --git a/packages/backend-test-utils/src/next/implementations/index.ts b/packages/backend-test-utils/src/next/implementations/index.ts index 55f417f8df..f962b392d8 100644 --- a/packages/backend-test-utils/src/next/implementations/index.ts +++ b/packages/backend-test-utils/src/next/implementations/index.ts @@ -13,4 +13,5 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { mockConfigFactory } from './mockConfigService'; +export { mockFactories } from './mockFactories'; +export { mockServices } from './mockServices'; diff --git a/packages/backend-test-utils/src/next/implementations/mockFactories.ts b/packages/backend-test-utils/src/next/implementations/mockFactories.ts new file mode 100644 index 0000000000..7d55588402 --- /dev/null +++ b/packages/backend-test-utils/src/next/implementations/mockFactories.ts @@ -0,0 +1,36 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + coreServices, + createServiceFactory, +} from '@backstage/backend-plugin-api'; +import { mockServices } from './mockServices'; + +/** + * @alpha + */ +export namespace mockFactories { + export const config = createServiceFactory( + (options?: mockServices.config.Options) => ({ + service: coreServices.config, + deps: {}, + async factory() { + return mockServices.config(options); + }, + }), + ); +} diff --git a/packages/backend-test-utils/src/next/implementations/mockConfigService.ts b/packages/backend-test-utils/src/next/implementations/mockServices.ts similarity index 63% rename from packages/backend-test-utils/src/next/implementations/mockConfigService.ts rename to packages/backend-test-utils/src/next/implementations/mockServices.ts index 1d04faad91..0ede7625c3 100644 --- a/packages/backend-test-utils/src/next/implementations/mockConfigService.ts +++ b/packages/backend-test-utils/src/next/implementations/mockServices.ts @@ -1,5 +1,5 @@ /* - * Copyright 2022 The Backstage Authors + * Copyright 2023 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,20 +14,18 @@ * limitations under the License. */ -import { - coreServices, - createServiceFactory, -} from '@backstage/backend-plugin-api'; import { ConfigReader } from '@backstage/config'; import { JsonObject } from '@backstage/types'; -/** @alpha */ -export const mockConfigFactory = createServiceFactory( - (options?: { data?: JsonObject }) => ({ - service: coreServices.config, - deps: {}, - async factory() { - return new ConfigReader(options?.data, 'mock-config'); - }, - }), -); +/** + * @alpha + */ +export namespace mockServices { + export namespace config { + export type Options = { data?: JsonObject }; + } + + export function config(options?: config.Options) { + return new ConfigReader(options?.data, 'mock-config'); + } +} diff --git a/packages/backend-test-utils/src/next/wiring/TestBackend.ts b/packages/backend-test-utils/src/next/wiring/TestBackend.ts index 78c2f78e5e..699dfa6396 100644 --- a/packages/backend-test-utils/src/next/wiring/TestBackend.ts +++ b/packages/backend-test-utils/src/next/wiring/TestBackend.ts @@ -41,7 +41,7 @@ import { coreServices, } from '@backstage/backend-plugin-api'; -import { mockConfigFactory } from '../implementations/mockConfigService'; +import { mockFactories } from '../implementations'; import { mockRootLoggerService } from '../implementations/mockRootLoggerService'; import { mockTokenManagerFactory } from '../implementations/mockTokenManagerService'; import { ConfigReader } from '@backstage/config'; @@ -89,7 +89,7 @@ const defaultServiceFactories = [ httpRouterFactory(), lifecycleFactory(), loggerFactory(), - mockConfigFactory(), + mockFactories.config(), mockRootLoggerService(), mockIdentityFactory(), mockTokenManagerFactory(), From 034116c3105ee6cfe7e9f9cd6cd8305e29c3faef Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 16 Jan 2023 15:34:55 +0100 Subject: [PATCH 031/182] backend-test-utils: move the rest of the mock services to new structure Signed-off-by: Patrik Oldsberg --- packages/backend-test-utils/api-report.md | 61 +++++++++++++++++++ .../src/next/implementations/mockFactories.ts | 48 +++++++++++++++ .../implementations/mockIdentityService.ts | 16 +---- .../implementations/mockRootLoggerService.ts | 27 +++----- .../src/next/implementations/mockServices.ts | 40 +++++++++++- .../mockTokenManagerService.ts | 40 ------------ .../src/next/wiring/TestBackend.ts | 35 ++++------- 7 files changed, 168 insertions(+), 99 deletions(-) delete mode 100644 packages/backend-test-utils/src/next/implementations/mockTokenManagerService.ts diff --git a/packages/backend-test-utils/api-report.md b/packages/backend-test-utils/api-report.md index ffcc0f3d35..6221a32d5b 100644 --- a/packages/backend-test-utils/api-report.md +++ b/packages/backend-test-utils/api-report.md @@ -5,14 +5,27 @@ ```ts import { Backend } from '@backstage/backend-app-api'; import { BackendFeature } from '@backstage/backend-plugin-api'; +import { CacheService } from '@backstage/backend-plugin-api'; import { ConfigReader } from '@backstage/config'; import { ConfigService } from '@backstage/backend-plugin-api'; +import { DatabaseService } from '@backstage/backend-plugin-api'; import { ExtendedHttpServer } from '@backstage/backend-app-api'; import { ExtensionPoint } from '@backstage/backend-plugin-api'; +import { HttpRouterFactoryOptions } from '@backstage/backend-app-api'; +import { HttpRouterService } from '@backstage/backend-plugin-api'; +import { IdentityService } from '@backstage/backend-plugin-api'; import { JsonObject } from '@backstage/types'; import { Knex } from 'knex'; +import { LifecycleService } from '@backstage/backend-plugin-api'; +import { LoggerService } from '@backstage/backend-plugin-api'; +import { PermissionsService } from '@backstage/backend-plugin-api'; +import { RootLifecycleService } from '@backstage/backend-plugin-api'; +import { RootLoggerService } from '@backstage/backend-plugin-api'; +import { SchedulerService } from '@backstage/backend-plugin-api'; import { ServiceFactory } from '@backstage/backend-plugin-api'; import { ServiceRef } from '@backstage/backend-plugin-api'; +import { TokenManagerService } from '@backstage/backend-plugin-api'; +import { UrlReaderService } from '@backstage/backend-plugin-api'; // @public (undocumented) export function isDockerDisabledForTests(): boolean; @@ -23,6 +36,34 @@ export namespace mockFactories { config: ( options?: mockServices.config.Options | undefined, ) => ServiceFactory; + const // (undocumented) + rootLogger: ( + options?: mockServices.rootLogger.Options | undefined, + ) => ServiceFactory; + const // (undocumented) + tokenManager: () => ServiceFactory; + const // (undocumented) + identity: () => ServiceFactory; + const // (undocumented) + cache: () => ServiceFactory; + const // (undocumented) + database: () => ServiceFactory; + const // (undocumented) + httpRouter: ( + options?: HttpRouterFactoryOptions | undefined, + ) => ServiceFactory; + const // (undocumented) + lifecycle: () => ServiceFactory; + const // (undocumented) + logger: () => ServiceFactory; + const // (undocumented) + permissions: () => ServiceFactory; + const // (undocumented) + rootLifecycle: () => ServiceFactory; + const // (undocumented) + scheduler: () => ServiceFactory; + const // (undocumented) + urlReader: () => ServiceFactory; } // @alpha (undocumented) @@ -36,6 +77,26 @@ export namespace mockServices { } // (undocumented) export function config(options?: config.Options): ConfigReader; + // (undocumented) + export function identity(): IdentityService; + // (undocumented) + export namespace rootLogger { + // (undocumented) + export type Options = { + levels: + | boolean + | { + error: boolean; + warn: boolean; + info: boolean; + debug: boolean; + }; + }; + } + // (undocumented) + export function rootLogger(options?: rootLogger.Options): LoggerService; + // (undocumented) + export function tokenManager(): TokenManagerService; } // @public diff --git a/packages/backend-test-utils/src/next/implementations/mockFactories.ts b/packages/backend-test-utils/src/next/implementations/mockFactories.ts index 7d55588402..d3bfe28c61 100644 --- a/packages/backend-test-utils/src/next/implementations/mockFactories.ts +++ b/packages/backend-test-utils/src/next/implementations/mockFactories.ts @@ -14,6 +14,17 @@ * limitations under the License. */ +import { + cacheFactory, + databaseFactory, + httpRouterFactory, + lifecycleFactory, + loggerFactory, + permissionsFactory, + rootLifecycleFactory, + schedulerFactory, + urlReaderFactory, +} from '@backstage/backend-app-api'; import { coreServices, createServiceFactory, @@ -33,4 +44,41 @@ export namespace mockFactories { }, }), ); + + export const rootLogger = createServiceFactory( + (options?: mockServices.rootLogger.Options) => ({ + service: coreServices.rootLogger, + deps: {}, + async factory() { + return mockServices.rootLogger(options); + }, + }), + ); + + export const tokenManager = createServiceFactory({ + service: coreServices.tokenManager, + deps: {}, + async factory() { + return mockServices.tokenManager(); + }, + }); + + export const identity = createServiceFactory({ + service: coreServices.identity, + deps: {}, + async factory() { + return mockServices.identity(); + }, + }); + + // For all other services, we just use the default implementations + export const cache = cacheFactory; + export const database = databaseFactory; + export const httpRouter = httpRouterFactory; + export const lifecycle = lifecycleFactory; + export const logger = loggerFactory; + export const permissions = permissionsFactory; + export const rootLifecycle = rootLifecycleFactory; + export const scheduler = schedulerFactory; + export const urlReader = urlReaderFactory; } diff --git a/packages/backend-test-utils/src/next/implementations/mockIdentityService.ts b/packages/backend-test-utils/src/next/implementations/mockIdentityService.ts index a838c8c879..ff424f79c2 100644 --- a/packages/backend-test-utils/src/next/implementations/mockIdentityService.ts +++ b/packages/backend-test-utils/src/next/implementations/mockIdentityService.ts @@ -13,17 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { - coreServices, - createServiceFactory, - IdentityService, -} from '@backstage/backend-plugin-api'; +import { IdentityService } from '@backstage/backend-plugin-api'; import { IdentityApiGetIdentityRequest, BackstageIdentityResponse, } from '@backstage/plugin-auth-node'; -class MockIdentityServiceImpl implements IdentityService { +export class MockIdentityService implements IdentityService { getIdentity( _options: IdentityApiGetIdentityRequest, ): Promise { @@ -37,11 +33,3 @@ class MockIdentityServiceImpl implements IdentityService { }); } } - -export const mockIdentityFactory = createServiceFactory({ - service: coreServices.identity, - deps: {}, - async factory() { - return new MockIdentityServiceImpl(); - }, -}); diff --git a/packages/backend-test-utils/src/next/implementations/mockRootLoggerService.ts b/packages/backend-test-utils/src/next/implementations/mockRootLoggerService.ts index bb3af6a9fb..e7450dfc9c 100644 --- a/packages/backend-test-utils/src/next/implementations/mockRootLoggerService.ts +++ b/packages/backend-test-utils/src/next/implementations/mockRootLoggerService.ts @@ -15,21 +15,14 @@ */ import { - coreServices, - createServiceFactory, LoggerService, LogMeta, RootLoggerService, } from '@backstage/backend-plugin-api'; +import type { mockServices } from './mockServices'; -interface MockLoggerOptions { - levels: - | boolean - | { error: boolean; warn: boolean; info: boolean; debug: boolean }; -} - -class MockLogger implements RootLoggerService { - #levels: Exclude; +export class MockLogger implements RootLoggerService { + #levels: Exclude; #meta: LogMeta; error(message: string, meta?: LogMeta | Error | undefined): void { @@ -52,7 +45,10 @@ class MockLogger implements RootLoggerService { return new MockLogger(this.#levels, { ...this.#meta, ...meta }); } - constructor(levels: MockLoggerOptions['levels'], meta: LogMeta) { + constructor( + levels: mockServices.rootLogger.Options['levels'], + meta: LogMeta, + ) { if (typeof levels === 'boolean') { this.#levels = { error: levels, @@ -79,12 +75,3 @@ class MockLogger implements RootLoggerService { } } } - -/** @alpha */ -export const mockRootLoggerService = createServiceFactory({ - service: coreServices.rootLogger, - deps: {}, - async factory(_deps) { - return new MockLogger(false, {}); - }, -}); diff --git a/packages/backend-test-utils/src/next/implementations/mockServices.ts b/packages/backend-test-utils/src/next/implementations/mockServices.ts index 0ede7625c3..8baed04daa 100644 --- a/packages/backend-test-utils/src/next/implementations/mockServices.ts +++ b/packages/backend-test-utils/src/next/implementations/mockServices.ts @@ -14,8 +14,15 @@ * limitations under the License. */ +import { + IdentityService, + LoggerService, + TokenManagerService, +} from '@backstage/backend-plugin-api'; import { ConfigReader } from '@backstage/config'; import { JsonObject } from '@backstage/types'; +import { MockIdentityService } from './mockIdentityService'; +import { MockLogger } from './mockRootLoggerService'; /** * @alpha @@ -24,8 +31,39 @@ export namespace mockServices { export namespace config { export type Options = { data?: JsonObject }; } - export function config(options?: config.Options) { return new ConfigReader(options?.data, 'mock-config'); } + + export namespace rootLogger { + export type Options = { + levels: + | boolean + | { error: boolean; warn: boolean; info: boolean; debug: boolean }; + }; + } + export function rootLogger(options?: rootLogger.Options): LoggerService { + return new MockLogger(options?.levels ?? false, {}); + } + + export function tokenManager(): TokenManagerService { + return { + async getToken(): Promise<{ token: string }> { + return { token: 'mock-token' }; + }, + async authenticate(token: string): Promise { + if (token !== 'mock-token') { + throw new Error('Invalid token'); + } + }, + }; + } + + export function identity(): IdentityService { + return new MockIdentityService(); + } + + // TODO(Rugvip): Not all core services have implementations available here yet. + // some may need a bit more refactoring for it to be simpler to + // re-implement functioning mock versions here. } diff --git a/packages/backend-test-utils/src/next/implementations/mockTokenManagerService.ts b/packages/backend-test-utils/src/next/implementations/mockTokenManagerService.ts deleted file mode 100644 index 2cfc15220e..0000000000 --- a/packages/backend-test-utils/src/next/implementations/mockTokenManagerService.ts +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2023 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { TokenManager } from '@backstage/backend-common'; -import { - coreServices, - createServiceFactory, -} from '@backstage/backend-plugin-api'; - -class TokenManagerMock implements TokenManager { - async getToken(): Promise<{ token: string }> { - return { token: 'mock-token' }; - } - async authenticate(token: string): Promise { - if (token !== 'mock-token') { - throw new Error('Invalid token'); - } - } -} - -/** @alpha */ -export const mockTokenManagerFactory = createServiceFactory({ - service: coreServices.tokenManager, - deps: {}, - async factory() { - return new TokenManagerMock(); - }, -}); diff --git a/packages/backend-test-utils/src/next/wiring/TestBackend.ts b/packages/backend-test-utils/src/next/wiring/TestBackend.ts index 699dfa6396..f2e7f45d08 100644 --- a/packages/backend-test-utils/src/next/wiring/TestBackend.ts +++ b/packages/backend-test-utils/src/next/wiring/TestBackend.ts @@ -17,15 +17,6 @@ import { Backend, createSpecializedBackend, - lifecycleFactory, - rootLifecycleFactory, - loggerFactory, - cacheFactory, - permissionsFactory, - schedulerFactory, - urlReaderFactory, - databaseFactory, - httpRouterFactory, MiddlewareFactory, createHttpServer, ExtendedHttpServer, @@ -42,11 +33,8 @@ import { } from '@backstage/backend-plugin-api'; import { mockFactories } from '../implementations'; -import { mockRootLoggerService } from '../implementations/mockRootLoggerService'; -import { mockTokenManagerFactory } from '../implementations/mockTokenManagerService'; import { ConfigReader } from '@backstage/config'; import express from 'express'; -import { mockIdentityFactory } from '../implementations/mockIdentityService'; /** @alpha */ export interface TestBackendOptions< @@ -84,19 +72,18 @@ export interface TestBackend extends Backend { } const defaultServiceFactories = [ - cacheFactory(), - databaseFactory(), - httpRouterFactory(), - lifecycleFactory(), - loggerFactory(), + mockFactories.cache(), + mockFactories.database(), + mockFactories.httpRouter(), + mockFactories.lifecycle(), + mockFactories.logger(), mockFactories.config(), - mockRootLoggerService(), - mockIdentityFactory(), - mockTokenManagerFactory(), - permissionsFactory(), - rootLifecycleFactory(), - schedulerFactory(), - urlReaderFactory(), + mockFactories.identity(), + mockFactories.tokenManager(), + mockFactories.permissions(), + mockFactories.rootLifecycle(), + mockFactories.scheduler(), + mockFactories.urlReader(), ]; const backendInstancesToCleanUp = new Array(); From d0901c9ba4a3c68ee78ed9625da5b1eaf1d9654e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 16 Jan 2023 15:38:23 +0100 Subject: [PATCH 032/182] changesets: added changeset for new mock exports from backend-test-utils Signed-off-by: Patrik Oldsberg --- .changeset/twenty-parents-relate.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/twenty-parents-relate.md diff --git a/.changeset/twenty-parents-relate.md b/.changeset/twenty-parents-relate.md new file mode 100644 index 0000000000..f1408bfa88 --- /dev/null +++ b/.changeset/twenty-parents-relate.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-test-utils': patch +--- + +All mock service factories and mock service implementations are now available via two new exports, `mockServices`, and `mockFactories`. From 846b310dbb995c971c84aef1bb593400ae1e1d1f Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 16 Jan 2023 17:13:34 +0100 Subject: [PATCH 033/182] backend-test-utils: merge mockFactories into mockServices Signed-off-by: Patrik Oldsberg --- .changeset/twenty-parents-relate.md | 2 +- packages/backend-test-utils/api-report.md | 127 ++++++++++++------ .../src/next/implementations/index.ts | 1 - .../src/next/implementations/mockFactories.ts | 84 ------------ .../src/next/implementations/mockServices.ts | 90 ++++++++++++- .../src/next/wiring/TestBackend.ts | 26 ++-- 6 files changed, 187 insertions(+), 143 deletions(-) delete mode 100644 packages/backend-test-utils/src/next/implementations/mockFactories.ts diff --git a/.changeset/twenty-parents-relate.md b/.changeset/twenty-parents-relate.md index f1408bfa88..a883873c16 100644 --- a/.changeset/twenty-parents-relate.md +++ b/.changeset/twenty-parents-relate.md @@ -2,4 +2,4 @@ '@backstage/backend-test-utils': patch --- -All mock service factories and mock service implementations are now available via two new exports, `mockServices`, and `mockFactories`. +All mock service factories and mock service implementations are now available via the new experimental `mockServices` export. diff --git a/packages/backend-test-utils/api-report.md b/packages/backend-test-utils/api-report.md index 6221a32d5b..232f7e8136 100644 --- a/packages/backend-test-utils/api-report.md +++ b/packages/backend-test-utils/api-report.md @@ -30,56 +30,84 @@ import { UrlReaderService } from '@backstage/backend-plugin-api'; // @public (undocumented) export function isDockerDisabledForTests(): boolean; -// @alpha (undocumented) -export namespace mockFactories { - const // (undocumented) - config: ( - options?: mockServices.config.Options | undefined, - ) => ServiceFactory; - const // (undocumented) - rootLogger: ( - options?: mockServices.rootLogger.Options | undefined, - ) => ServiceFactory; - const // (undocumented) - tokenManager: () => ServiceFactory; - const // (undocumented) - identity: () => ServiceFactory; - const // (undocumented) - cache: () => ServiceFactory; - const // (undocumented) - database: () => ServiceFactory; - const // (undocumented) - httpRouter: ( - options?: HttpRouterFactoryOptions | undefined, - ) => ServiceFactory; - const // (undocumented) - lifecycle: () => ServiceFactory; - const // (undocumented) - logger: () => ServiceFactory; - const // (undocumented) - permissions: () => ServiceFactory; - const // (undocumented) - rootLifecycle: () => ServiceFactory; - const // (undocumented) - scheduler: () => ServiceFactory; - const // (undocumented) - urlReader: () => ServiceFactory; -} - // @alpha (undocumented) export namespace mockServices { + // (undocumented) + export namespace cache { + const // (undocumented) + ref: ServiceRef; + const // (undocumented) + factory: () => ServiceFactory; + } + // (undocumented) + export function config(options?: config.Options): ConfigReader; // (undocumented) export namespace config { // (undocumented) export type Options = { data?: JsonObject; }; + const // (undocumented) + ref: ServiceRef; + const // (undocumented) + factory: (options?: Options | undefined) => ServiceFactory; } // (undocumented) - export function config(options?: config.Options): ConfigReader; + export namespace database { + const // (undocumented) + ref: ServiceRef; + const // (undocumented) + factory: () => ServiceFactory; + } + // (undocumented) + export namespace httpRouter { + const // (undocumented) + ref: ServiceRef; + const // (undocumented) + factory: ( + options?: HttpRouterFactoryOptions | undefined, + ) => ServiceFactory; + } // (undocumented) export function identity(): IdentityService; // (undocumented) + export namespace identity { + const // (undocumented) + ref: ServiceRef; + const // (undocumented) + factory: () => ServiceFactory; + } + // (undocumented) + export namespace lifecycle { + const // (undocumented) + ref: ServiceRef; + const // (undocumented) + factory: () => ServiceFactory; + } + // (undocumented) + export namespace logger { + const // (undocumented) + ref: ServiceRef; + const // (undocumented) + factory: () => ServiceFactory; + } + // (undocumented) + export namespace permissions { + const // (undocumented) + ref: ServiceRef; + const // (undocumented) + factory: () => ServiceFactory; + } + // (undocumented) + export namespace rootLifecycle { + const // (undocumented) + ref: ServiceRef; + const // (undocumented) + factory: () => ServiceFactory; + } + // (undocumented) + export function rootLogger(options?: rootLogger.Options): LoggerService; + // (undocumented) export namespace rootLogger { // (undocumented) export type Options = { @@ -92,11 +120,34 @@ export namespace mockServices { debug: boolean; }; }; + const // (undocumented) + ref: ServiceRef; + const // (undocumented) + factory: (options?: Options | undefined) => ServiceFactory; } // (undocumented) - export function rootLogger(options?: rootLogger.Options): LoggerService; + export namespace scheduler { + const // (undocumented) + ref: ServiceRef; + const // (undocumented) + factory: () => ServiceFactory; + } // (undocumented) export function tokenManager(): TokenManagerService; + // (undocumented) + export namespace tokenManager { + const // (undocumented) + ref: ServiceRef; + const // (undocumented) + factory: () => ServiceFactory; + } + // (undocumented) + export namespace urlReader { + const // (undocumented) + ref: ServiceRef; + const // (undocumented) + factory: () => ServiceFactory; + } } // @public diff --git a/packages/backend-test-utils/src/next/implementations/index.ts b/packages/backend-test-utils/src/next/implementations/index.ts index f962b392d8..f562150efa 100644 --- a/packages/backend-test-utils/src/next/implementations/index.ts +++ b/packages/backend-test-utils/src/next/implementations/index.ts @@ -13,5 +13,4 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { mockFactories } from './mockFactories'; export { mockServices } from './mockServices'; diff --git a/packages/backend-test-utils/src/next/implementations/mockFactories.ts b/packages/backend-test-utils/src/next/implementations/mockFactories.ts deleted file mode 100644 index d3bfe28c61..0000000000 --- a/packages/backend-test-utils/src/next/implementations/mockFactories.ts +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2023 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - cacheFactory, - databaseFactory, - httpRouterFactory, - lifecycleFactory, - loggerFactory, - permissionsFactory, - rootLifecycleFactory, - schedulerFactory, - urlReaderFactory, -} from '@backstage/backend-app-api'; -import { - coreServices, - createServiceFactory, -} from '@backstage/backend-plugin-api'; -import { mockServices } from './mockServices'; - -/** - * @alpha - */ -export namespace mockFactories { - export const config = createServiceFactory( - (options?: mockServices.config.Options) => ({ - service: coreServices.config, - deps: {}, - async factory() { - return mockServices.config(options); - }, - }), - ); - - export const rootLogger = createServiceFactory( - (options?: mockServices.rootLogger.Options) => ({ - service: coreServices.rootLogger, - deps: {}, - async factory() { - return mockServices.rootLogger(options); - }, - }), - ); - - export const tokenManager = createServiceFactory({ - service: coreServices.tokenManager, - deps: {}, - async factory() { - return mockServices.tokenManager(); - }, - }); - - export const identity = createServiceFactory({ - service: coreServices.identity, - deps: {}, - async factory() { - return mockServices.identity(); - }, - }); - - // For all other services, we just use the default implementations - export const cache = cacheFactory; - export const database = databaseFactory; - export const httpRouter = httpRouterFactory; - export const lifecycle = lifecycleFactory; - export const logger = loggerFactory; - export const permissions = permissionsFactory; - export const rootLifecycle = rootLifecycleFactory; - export const scheduler = schedulerFactory; - export const urlReader = urlReaderFactory; -} diff --git a/packages/backend-test-utils/src/next/implementations/mockServices.ts b/packages/backend-test-utils/src/next/implementations/mockServices.ts index 8baed04daa..5cb1371d95 100644 --- a/packages/backend-test-utils/src/next/implementations/mockServices.ts +++ b/packages/backend-test-utils/src/next/implementations/mockServices.ts @@ -15,35 +15,69 @@ */ import { + coreServices, + createServiceFactory, IdentityService, LoggerService, + ServiceFactory, + ServiceRef, TokenManagerService, } from '@backstage/backend-plugin-api'; +import { + cacheFactory, + databaseFactory, + httpRouterFactory, + lifecycleFactory, + loggerFactory, + permissionsFactory, + rootLifecycleFactory, + schedulerFactory, + urlReaderFactory, +} from '@backstage/backend-app-api'; import { ConfigReader } from '@backstage/config'; import { JsonObject } from '@backstage/types'; import { MockIdentityService } from './mockIdentityService'; import { MockLogger } from './mockRootLoggerService'; +function simpleFactory( + ref: ServiceRef, + factory: (...options: TOptions) => TService, +): (...options: TOptions) => ServiceFactory { + return createServiceFactory((options: unknown) => ({ + service: ref as ServiceRef, + deps: {}, + async factory() { + return (factory as any)(options); + }, + })); +} + /** * @alpha */ export namespace mockServices { - export namespace config { - export type Options = { data?: JsonObject }; - } export function config(options?: config.Options) { return new ConfigReader(options?.data, 'mock-config'); } + export namespace config { + export type Options = { data?: JsonObject }; + export const ref = coreServices.config; + export const factory = simpleFactory(ref, config); + } + + export function rootLogger(options?: rootLogger.Options): LoggerService { + return new MockLogger(options?.levels ?? false, {}); + } export namespace rootLogger { export type Options = { levels: | boolean | { error: boolean; warn: boolean; info: boolean; debug: boolean }; }; - } - export function rootLogger(options?: rootLogger.Options): LoggerService { - return new MockLogger(options?.levels ?? false, {}); + + export const ref = coreServices.rootLogger; + export const factory = simpleFactory(ref, rootLogger); } export function tokenManager(): TokenManagerService { @@ -58,12 +92,56 @@ export namespace mockServices { }, }; } + export namespace tokenManager { + export const ref = coreServices.tokenManager; + export const factory = simpleFactory(ref, tokenManager); + } export function identity(): IdentityService { return new MockIdentityService(); } + export namespace identity { + export const ref = coreServices.identity; + export const factory = simpleFactory(ref, identity); + } // TODO(Rugvip): Not all core services have implementations available here yet. // some may need a bit more refactoring for it to be simpler to // re-implement functioning mock versions here. + export namespace cache { + export const ref = coreServices.cache; + export const factory = cacheFactory; + } + export namespace database { + export const ref = coreServices.database; + export const factory = databaseFactory; + } + export namespace httpRouter { + export const ref = coreServices.httpRouter; + export const factory = httpRouterFactory; + } + export namespace lifecycle { + export const ref = coreServices.lifecycle; + export const factory = lifecycleFactory; + } + export namespace logger { + export const ref = coreServices.logger; + export const factory = loggerFactory; + } + export namespace permissions { + export const ref = coreServices.permissions; + export const factory = permissionsFactory; + } + export namespace rootLifecycle { + export const ref = coreServices.rootLifecycle; + export const factory = rootLifecycleFactory; + } + export namespace scheduler { + export const ref = coreServices.scheduler; + export const factory = schedulerFactory; + } + export namespace urlReader { + export const ref = coreServices.urlReader; + export const factory = urlReaderFactory; + } } diff --git a/packages/backend-test-utils/src/next/wiring/TestBackend.ts b/packages/backend-test-utils/src/next/wiring/TestBackend.ts index f2e7f45d08..279e64daac 100644 --- a/packages/backend-test-utils/src/next/wiring/TestBackend.ts +++ b/packages/backend-test-utils/src/next/wiring/TestBackend.ts @@ -32,7 +32,7 @@ import { coreServices, } from '@backstage/backend-plugin-api'; -import { mockFactories } from '../implementations'; +import { mockServices } from '../implementations'; import { ConfigReader } from '@backstage/config'; import express from 'express'; @@ -72,18 +72,18 @@ export interface TestBackend extends Backend { } const defaultServiceFactories = [ - mockFactories.cache(), - mockFactories.database(), - mockFactories.httpRouter(), - mockFactories.lifecycle(), - mockFactories.logger(), - mockFactories.config(), - mockFactories.identity(), - mockFactories.tokenManager(), - mockFactories.permissions(), - mockFactories.rootLifecycle(), - mockFactories.scheduler(), - mockFactories.urlReader(), + mockServices.cache.factory(), + mockServices.database.factory(), + mockServices.httpRouter.factory(), + mockServices.lifecycle.factory(), + mockServices.logger.factory(), + mockServices.config.factory(), + mockServices.identity.factory(), + mockServices.tokenManager.factory(), + mockServices.permissions.factory(), + mockServices.rootLifecycle.factory(), + mockServices.scheduler.factory(), + mockServices.urlReader.factory(), ]; const backendInstancesToCleanUp = new Array(); From cf2967561f9ef303947a2a9352c4e8744f91fbb1 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 17 Jan 2023 10:33:41 +0100 Subject: [PATCH 034/182] update tests to use mockServices Signed-off-by: Patrik Oldsberg --- packages/backend-defaults/src/CreateBackend.test.ts | 4 ++-- .../BitbucketCloudEntityProviderCatalogModule.test.ts | 7 ++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/packages/backend-defaults/src/CreateBackend.test.ts b/packages/backend-defaults/src/CreateBackend.test.ts index 35b3a7decf..2cfb92c3a6 100644 --- a/packages/backend-defaults/src/CreateBackend.test.ts +++ b/packages/backend-defaults/src/CreateBackend.test.ts @@ -21,7 +21,7 @@ import { createServiceRef, createSharedEnvironment, } from '@backstage/backend-plugin-api'; -import { mockConfigFactory } from '@backstage/backend-test-utils'; +import { mockServices } from '@backstage/backend-test-utils'; import { createBackend } from './CreateBackend'; const fooServiceRef = createServiceRef({ id: 'foo', scope: 'root' }); @@ -112,7 +112,7 @@ describe('createBackend', () => { }; }, }), - mockConfigFactory({ + mockServices.config.factory({ data: { root: 'root-env' }, }), createServiceFactory({ diff --git a/plugins/catalog-backend-module-bitbucket-cloud/src/service/BitbucketCloudEntityProviderCatalogModule.test.ts b/plugins/catalog-backend-module-bitbucket-cloud/src/service/BitbucketCloudEntityProviderCatalogModule.test.ts index d44281600c..1b18ab0ed9 100644 --- a/plugins/catalog-backend-module-bitbucket-cloud/src/service/BitbucketCloudEntityProviderCatalogModule.test.ts +++ b/plugins/catalog-backend-module-bitbucket-cloud/src/service/BitbucketCloudEntityProviderCatalogModule.test.ts @@ -19,10 +19,7 @@ import { PluginTaskScheduler, TaskScheduleDefinition, } from '@backstage/backend-tasks'; -import { - startTestBackend, - mockConfigFactory, -} from '@backstage/backend-test-utils'; +import { startTestBackend, mockServices } from '@backstage/backend-test-utils'; import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node'; import { eventsExtensionPoint } from '@backstage/plugin-events-node'; import { Duration } from 'luxon'; @@ -59,7 +56,7 @@ describe('bitbucketCloudEntityProviderCatalogModule', () => { [eventsExtensionPoint, eventsExtensionPointImpl], ], services: [ - mockConfigFactory({ + mockServices.config.factory({ data: { catalog: { providers: { From 8702e9f78d2ea48b1a39d8b8cdfb7efaeb62fa77 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 17 Jan 2023 10:35:25 +0100 Subject: [PATCH 035/182] backend-test-utils: add missing rootLogger factory to startTestBackend Signed-off-by: Patrik Oldsberg --- packages/backend-test-utils/src/next/wiring/TestBackend.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/backend-test-utils/src/next/wiring/TestBackend.ts b/packages/backend-test-utils/src/next/wiring/TestBackend.ts index 279e64daac..f87e0927e2 100644 --- a/packages/backend-test-utils/src/next/wiring/TestBackend.ts +++ b/packages/backend-test-utils/src/next/wiring/TestBackend.ts @@ -73,16 +73,17 @@ export interface TestBackend extends Backend { const defaultServiceFactories = [ mockServices.cache.factory(), + mockServices.config.factory(), mockServices.database.factory(), mockServices.httpRouter.factory(), + mockServices.identity.factory(), mockServices.lifecycle.factory(), mockServices.logger.factory(), - mockServices.config.factory(), - mockServices.identity.factory(), - mockServices.tokenManager.factory(), mockServices.permissions.factory(), mockServices.rootLifecycle.factory(), + mockServices.rootLogger.factory(), mockServices.scheduler.factory(), + mockServices.tokenManager.factory(), mockServices.urlReader.factory(), ]; From b284705dc01625f690af016ce5dbe69cf57af1a4 Mon Sep 17 00:00:00 2001 From: pitwegner Date: Tue, 17 Jan 2023 11:08:32 +0100 Subject: [PATCH 036/182] add template replace option Signed-off-by: pitwegner Signed-off-by: pitwegner --- .../src/scaffolder/actions/builtin/fetch/template.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.ts index 10a9d0c055..dc37b04f80 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.ts @@ -118,6 +118,12 @@ export function createFetchTemplateAction(options: { 'If set, only files with the given extension will be templated. If set to `true`, the default extension `.njk` is used.', type: ['string', 'boolean'], }, + replace: { + title: 'Replace files', + description: + 'If set, replace files in targetPath instead of skipping existing ones.', + type: 'boolean', + }, }, }, }, @@ -261,7 +267,7 @@ export function createFetchTemplateAction(options: { } const outputPath = resolveSafeChildPath(outputDir, localOutputPath); - if (fs.existsSync(outputPath)) { + if (fs.existsSync(outputPath) && !ctx.input.replace) { continue; } From ac44b2676321826ee76f34d217ce050b7a1d51d8 Mon Sep 17 00:00:00 2001 From: pitwegner Date: Tue, 17 Jan 2023 13:26:42 +0100 Subject: [PATCH 037/182] add replace to signature Signed-off-by: pitwegner Signed-off-by: pitwegner --- .../src/scaffolder/actions/builtin/fetch/template.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.ts index dc37b04f80..2ebcfd14c7 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.ts @@ -62,6 +62,7 @@ export function createFetchTemplateAction(options: { copyWithoutRender?: string[]; copyWithoutTemplating?: string[]; cookiecutterCompat?: boolean; + replace: boolean; }>({ id: 'fetch:template', description: From 0b2952ee4b3a05c2f7e9cebf9cd052ed1a629645 Mon Sep 17 00:00:00 2001 From: pitwegner Date: Tue, 17 Jan 2023 13:30:21 +0100 Subject: [PATCH 038/182] add changeset Signed-off-by: pitwegner Signed-off-by: pitwegner --- .changeset/perfect-cheetahs-serve.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/perfect-cheetahs-serve.md diff --git a/.changeset/perfect-cheetahs-serve.md b/.changeset/perfect-cheetahs-serve.md new file mode 100644 index 0000000000..2a031a65f7 --- /dev/null +++ b/.changeset/perfect-cheetahs-serve.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': patch +--- + +Added the option to overwrite files in the targetPath of the template:fetch action From 987d1f3f43cf0d356fda3dc134b77b73bf643f6e Mon Sep 17 00:00:00 2001 From: pitwegner Date: Tue, 17 Jan 2023 13:34:24 +0100 Subject: [PATCH 039/182] format keywords as code Signed-off-by: pitwegner Signed-off-by: pitwegner --- .changeset/perfect-cheetahs-serve.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/perfect-cheetahs-serve.md b/.changeset/perfect-cheetahs-serve.md index 2a031a65f7..bc1835e472 100644 --- a/.changeset/perfect-cheetahs-serve.md +++ b/.changeset/perfect-cheetahs-serve.md @@ -2,4 +2,4 @@ '@backstage/plugin-scaffolder-backend': patch --- -Added the option to overwrite files in the targetPath of the template:fetch action +Added the option to overwrite files in the `targetPath` of the `template:fetch` action From 6d435e5199de37838351406d19ce8f8e9d5a638f Mon Sep 17 00:00:00 2001 From: pitwegner Date: Tue, 17 Jan 2023 13:50:30 +0100 Subject: [PATCH 040/182] make arg optional Signed-off-by: pitwegner --- .../src/scaffolder/actions/builtin/fetch/template.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.ts index 2ebcfd14c7..81b3f4577c 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.ts @@ -62,7 +62,7 @@ export function createFetchTemplateAction(options: { copyWithoutRender?: string[]; copyWithoutTemplating?: string[]; cookiecutterCompat?: boolean; - replace: boolean; + replace?: boolean; }>({ id: 'fetch:template', description: From 9d1950d4a46c261676063a9a6f350f05765de998 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 17 Jan 2023 14:13:09 +0100 Subject: [PATCH 041/182] docs: tweaks to new backend system docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: Johan Haals Signed-off-by: Patrik Oldsberg --- docs/plugins/new-backend-system.md | 46 +++++++++++++++--------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/docs/plugins/new-backend-system.md b/docs/plugins/new-backend-system.md index e5c318d5c2..bfc6f79661 100644 --- a/docs/plugins/new-backend-system.md +++ b/docs/plugins/new-backend-system.md @@ -8,13 +8,13 @@ description: Details of the upcoming backend system ## Status -The new backend system is under active development, and only a small number of plugins and services have been migrated so far. It is possible to try it out, but it is not recommended to use this new system in production yet. +The new backend system is under active development, and only a small number of plugins have been migrated so far. It is possible to try it out, but it is not recommended to use this new system in production yet. -You can find an example backend setup at https://github.com/backstage/backstage/tree/master/packages/backend-next. +You can find an example backend setup in [the backend-next package](https://github.com/backstage/backstage/tree/master/packages/backend-next). ## Overview -The new Backstage backend system is being built to help make it simpler to install backend plugins and keep projects up to date. It also changes the foundation to one that makes it a lot easier to evolve plugins and the system itself. You can read more about the reasoning in the [original RFC](https://github.com/backstage/backstage/issues/11611). +The new Backstage backend system is being built to help make it simpler to install backend plugins and to keep projects up to date. It also changes the foundation to one that makes it a lot easier to evolve plugins and the system itself with minimal disruption or cause for breaking changes. You can read more about the reasoning in the [original RFC](https://github.com/backstage/backstage/issues/11611). One of the goals of the new system was to reduce the code needed for setting up a Backstage backend and installing plugins. This is an example of how you create, add features, and start up your backend in the new system: @@ -32,21 +32,21 @@ backend.add(catalogPlugin()); await backend.start(); ``` -One notable change that helped achieve this much slimmer backend setup is the introduction of dependency injection, with a system that is very similar to the one in the Backstage frontend. +One notable change that helped achieve this much slimmer backend setup is the introduction of a system for dependency injection, which is very similar to the one in the Backstage frontend. ## Building Blocks -This section introduces the high-level building blocks upon which this new system is built. These are all concepts that exist in our current system in one way or another, but the have all been lifted up to be first class concerns in the new system. +This section introduces the high-level building blocks upon which this new system is built. These are all concepts that exist in our current system in one way or another, but they have all been lifted up to be first class concerns in the new system. ### Backend -This is the backend instance itself, which you can think of as the unit of deployment. It does not have any functionality in itself, but is simply responsible for wiring things together. +This is the backend instance itself, which you can think of as the unit of deployment. It does not have any functionality in and of itself, but is simply responsible for wiring things together. It is up to you to decide how many different backends you want to deploy. You can have all features in a single one, or split things out into multiple smaller deployments. All depending on your need to scale and isolate individual features. ### Plugins -Plugins provide the actual features, just like in our existing system. They operate completely independently of each other. If plugins what to communicate with each other, they must do so over the wire. There can be no direct communication between plugins through code. Because of this constraints, each plugins can be considered to be its own microservice. +Plugins provide the actual features, just like in our existing system. They operate completely independently of each other. If plugins want to communicate with each other, they must do so over the wire. There can be no direct communication between plugins through code. Because of this constraint, each plugin can be considered to be its own microservice. ### Services @@ -77,6 +77,7 @@ Plugins are created using the `createBackendPlugin` function. All plugins must h ```ts import { configServiceRef, + coreServices, createBackendPlugin, } from '@backstage/backend-plugin-api'; @@ -89,7 +90,7 @@ export const examplePlugin = createBackendPlugin({ register(env) { env.registerInit({ deps: { - logger: loggerServiceRef, + logger: coreServices.logger, }, // logger is provided by the backend based on the dependency on loggerServiceRef above. async init({ logger }) { @@ -113,7 +114,7 @@ export const examplePlugin = createBackendPlugin({ id: 'example', register(env, options?: { silent?: boolean }) { env.registerInit({ - deps: { logger: loggerServiceRef }, + deps: { logger: coreServices.logger }, async init({ logger }) { if (!options?.silent) { logger.info('Hello from example plugin'); @@ -188,7 +189,7 @@ Extension points are registered by a plugin and extended by modules. ## Backend Services -The default backend provides several _services_ out of the box which includes access to configuration, logging, databases and more. +The default backend provides several [core services](https://github.com/backstage/backstage/blob/master/packages/backend-plugin-api/src/services/definitions/coreServices.ts) out of the box which includes access to configuration, logging, databases and more. Service dependencies are declared using their `ServiceRef`s in the `deps` section of the plugin or module, and the implementations are then forwarded to the `init` method of the plugin or module. ### Service References @@ -204,8 +205,7 @@ ServiceRefs contain a scope which is used to determine if the serviceFactory cre ```ts import { createServiceFactory, - pluginMetadataServiceRef, - loggerServiceRef, + coreServices, } from '@backstage/backend-plugin-api'; import { ExampleImpl } from './ExampleImpl'; @@ -223,11 +223,11 @@ export const exampleServiceRef = createServiceRef({ createServiceFactory({ service, deps: { - logger: loggerServiceRef, - plugin: pluginMetadataServiceRef, + logger: coreServices.logger, + plugin: coreServices.pluginMetadata, }, // Logger is available directly in the factory as it's a root scoped service and will be created once per backend instance. - async factory({ logger }) { + async factory({ logger, plugin }) { // plugin is available as it's a plugin scoped service and will be created once per plugin. return async ({ plugin }) => { // This block will be executed once for every plugin that depends on this service @@ -277,20 +277,20 @@ const backend = createBackend({ ## Testing Utilities for testing backend plugins and modules are available in `@backstage/backend-test-utils`. +`startTestBackend` returns the HTTP which can be used together with `supertest` to test the plugin. ```ts import { startTestBackend } from '@backstage/backend-test-utils'; +import request from 'supertest'; -describe('Example', () => { - it('should do something', async () => { - await startTestBackend({ - // mock services can be provided to the backend - services: [someServiceFactory], - // plugins and modules for testing - features: [testModule()], +describe('My plugin tests', () => { + it('should return 200', async () => { + const { server } = await startTestBackend({ + features: [myPlugin()], }); - // assertions + const response = await request(server).get('/api/example/hello'); + expect(response.status).toBe(200); }); }); ``` From e4f3c15dbe1ca0c53a2ceece9a6f70a84b15fb46 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 17 Jan 2023 15:49:06 +0100 Subject: [PATCH 042/182] docs: new structure for backend system docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: Johan Haals Signed-off-by: Patrik Oldsberg --- docs/backend-system/architecture/01-index.md | 70 ++++++++++++++ .../architecture/02-backends.md | 27 ++++++ .../architecture/03-services.md | 94 +++++++++++++++++++ .../backend-system/architecture/04-plugins.md | 68 ++++++++++++++ .../architecture/05-extension-points.md | 30 ++++++ .../backend-system/architecture/06-modules.md | 40 ++++++++ .../building-backends/01-index.md | 12 +++ .../building-backends/08-migrating.md | 9 ++ .../building-plugins-and-modules/01-index.md | 12 +++ .../02-testing.md | 26 +++++ .../08-migrating.md | 9 ++ docs/backend-system/core-services/01-index.md | 75 +++++++++++++++ docs/backend-system/index.md | 15 +++ microsite/sidebars.json | 40 +++++++- 14 files changed, 526 insertions(+), 1 deletion(-) create mode 100644 docs/backend-system/architecture/01-index.md create mode 100644 docs/backend-system/architecture/02-backends.md create mode 100644 docs/backend-system/architecture/03-services.md create mode 100644 docs/backend-system/architecture/04-plugins.md create mode 100644 docs/backend-system/architecture/05-extension-points.md create mode 100644 docs/backend-system/architecture/06-modules.md create mode 100644 docs/backend-system/building-backends/01-index.md create mode 100644 docs/backend-system/building-backends/08-migrating.md create mode 100644 docs/backend-system/building-plugins-and-modules/01-index.md create mode 100644 docs/backend-system/building-plugins-and-modules/02-testing.md create mode 100644 docs/backend-system/building-plugins-and-modules/08-migrating.md create mode 100644 docs/backend-system/core-services/01-index.md create mode 100644 docs/backend-system/index.md diff --git a/docs/backend-system/architecture/01-index.md b/docs/backend-system/architecture/01-index.md new file mode 100644 index 0000000000..5e6fb60ee6 --- /dev/null +++ b/docs/backend-system/architecture/01-index.md @@ -0,0 +1,70 @@ +--- +id: index +title: Backend System Architecture +sidebar_label: System Architecture +# prettier-ignore +description: The structure and architecture of the new Backend System and its component parts +--- + +# Overview + +This section introduces the high-level building blocks upon which this new +system is built. These are all concepts that exist in our current system in one +way or another, but they have all been lifted up to be first class concerns in +the new system. + +## Building Blocks + +This section introduces the high-level building blocks upon which this new system is built. These are all concepts that exist in our current system in one way or another, but they have all been lifted up to be first class concerns in the new system. + + + +### Backend + +This is the backend instance itself, which you can think of as the unit of deployment. It does not have any functionality in and of itself, but is simply responsible for wiring things together. + +It is up to you to decide how many different backends you want to deploy. You can have all features in a single one, or split things out into multiple smaller deployments. All depending on your need to scale and isolate individual features. + +### Plugins + +Plugins provide the actual features, just like in our existing system. They operate completely independently of each other. If plugins want to communicate with each other, they must do so over the wire. There can be no direct communication between plugins through code. Because of this constraint, each plugin can be considered to be its own microservice. + +### Services + +Services provide utilities to help make it simpler to implement plugins, so that each plugin doesn't need to implement everything from scratch. There are both many built-in services, like the ones for logging, database access, and reading configuration, but you can also import third-party services, or create your own. + +Services are also a customization point for individual backend installations. You can both override services with your own implementations, as well as make smaller customizations to existing services. + +### Extension Points + +Many plugins have ways in which you can extend them, for example entity providers for the Catalog, or custom actions for the Scaffolder. These extension patterns are now encoded into Extension Points. + +Extension Points look a little bit like services, since you depended on them just like you would a service. A key difference is that extension points are registered and provided by plugins themselves, based on what customizations each individual plugin wants to expose. + +Extension Points are also exported separately from the plugin instance itself, and a single plugin can also expose multiple different extension points at once. This makes it easier to evolve and deprecated individual Extension Points over time, rather than dealing with a single large API surface. + +### Modules + +Modules use the plugin Extension Points to add new features for plugins. They might for example add an individual Catalog Entity Provider, or one or more Scaffolder Actions. Modules are basically plugins for plugins. + +Each module may only extend a single plugin, and the module must be deployed together with that plugin in the same backend instance. Modules may however only communicate with their plugin through its registered extension points. + +Just like plugins, modules also have access to services and can depend on their own service implementations. They will however share services with the plugin that they extend, there are no module-specific service implementations. + +## Package structure + +A detailed explanation of the package architecture can be found in the +[Backstage Architecture +Overview](../../overview/architecture-overview.md#package-architecture). The +most important packages to consider for this system are `backend`, +`plugin--backend`, `plugin--node`, and +`plugin--backend-module-`. + +- `plugin--backend` houses the implementation of the backend plugins + themselves. +- `plugin--node` houses the extension points and any other utilities + that modules or other plugins might need. +- `plugin--backend-module-` houses the modules that extend + the plugins via the extension points. +- `backend` is the backend itself that wires everything together to something + that you can deploy. diff --git a/docs/backend-system/architecture/02-backends.md b/docs/backend-system/architecture/02-backends.md new file mode 100644 index 0000000000..c73a899863 --- /dev/null +++ b/docs/backend-system/architecture/02-backends.md @@ -0,0 +1,27 @@ +--- +id: backends +title: Backend Instances +sidebar_label: Backend +# prettier-ignore +description: Service APIs for backend plugins +--- + +The new Backstage backend system is being built to help make it simpler to install backend plugins and to keep projects up to date. It also changes the foundation to one that makes it a lot easier to evolve plugins and the system itself with minimal disruption or cause for breaking changes. You can read more about the reasoning in the [original RFC](https://github.com/backstage/backstage/issues/11611). + +One of the goals of the new system was to reduce the code needed for setting up a Backstage backend and installing plugins. This is an example of how you create, add features, and start up your backend in the new system: + +```ts +import { createBackend } from '@backstage/backend-defaults'; +import { catalogPlugin } from '@backstage/plugin-catalog-backend'; + +// Create your backend instance +const backend = createBackend(); + +// Install all desired features +backend.add(catalogPlugin()); + +// Start up the backend +await backend.start(); +``` + +One notable change that helped achieve this much slimmer backend setup is the introduction of a system for dependency injection, which is very similar to the one in the Backstage frontend. diff --git a/docs/backend-system/architecture/03-services.md b/docs/backend-system/architecture/03-services.md new file mode 100644 index 0000000000..adedbc6383 --- /dev/null +++ b/docs/backend-system/architecture/03-services.md @@ -0,0 +1,94 @@ +--- +id: services +title: Backend Service APIs +sidebar_label: Service APIs +# prettier-ignore +description: Service APIs for backend plugins +--- + +## Backend Services + +The default backend provides several [core services](https://github.com/backstage/backstage/blob/master/packages/backend-plugin-api/src/services/definitions/coreServices.ts) out of the box which includes access to configuration, logging, databases and more. +Service dependencies are declared using their `ServiceRef`s in the `deps` section of the plugin or module, and the implementations are then forwarded to the `init` method of the plugin or module. + +### Service References + +A `ServiceRef` is a named reference to an interface which are later used to resolve the concrete service implementation. Conceptually this is very similar to `ApiRef`s in the frontend. +Services is what provides common utilities that previously resided in the `PluginEnvironment` such as Config, Logging and Database. + +On startup the backend will make sure that the services are initialized before being passed to the plugin/module that depend on them. +ServiceRefs contain a scope which is used to determine if the serviceFactory creating the service will create a new instance scoped per plugin/module or if it will be shared. `plugin` scoped services will be created once per plugin/module and `root` scoped services will be created once per backend instance. + +#### Defining a Service + +```ts +import { + createServiceFactory, + coreServices, +} from '@backstage/backend-plugin-api'; +import { ExampleImpl } from './ExampleImpl'; + +export interface ExampleApi { + doSomething(): Promise; +} + +export const exampleServiceRef = createServiceRef({ + id: 'example', + scope: 'plugin', // can be 'root' or 'plugin' + + // The defaultFactory is optional to implement but it will be used if no other factory is provided to the backend. + // This is allows for the backend to provide a default implementation of the service without having to wire it beforehand. + defaultFactory: async service => + createServiceFactory({ + service, + deps: { + logger: coreServices.logger, + plugin: coreServices.pluginMetadata, + }, + // Logger is available directly in the factory as it's a root scoped service and will be created once per backend instance. + async factory({ logger, plugin }) { + // plugin is available as it's a plugin scoped service and will be created once per plugin. + return async ({ plugin }) => { + // This block will be executed once for every plugin that depends on this service + logger.info('Initializing example service plugin instance'); + return new ExampleImpl({ logger, plugin }); + }; + }, + }), +}); +``` + +### Overriding Services + +In this example we replace the default root logger service implementation with a custom one that streams logs to GCP. The `rootLoggerServiceRef` has a `'root'` scope, meaning there are no plugin-specific instances of this service. + +```ts +import { + createServiceFactory, + rootLoggerServiceRef, + LoggerService, +} from '@backstage/backend-plugin-api'; + +// This custom implementation would typically live separately from +// the backend setup code, either nearby such as in +// packages/backend/src/services/logger/GoogleCloudLogger.ts +// Or you can let it live in its own library package. +class GoogleCloudLogger implements LoggerService { + static factory = createServiceFactory({ + service: rootLoggerServiceRef, + deps: {}, + factory() { + return new GoogleCloudLogger(); + }, + }); + // custom implementation here ... +} + +// packages/backend/src/index.ts +const backend = createBackend({ + services: [ + // supplies additional or replacement services to the backend + GoogleCloudLogger.factory(), + ], +}); +``` diff --git a/docs/backend-system/architecture/04-plugins.md b/docs/backend-system/architecture/04-plugins.md new file mode 100644 index 0000000000..6f077a0d2d --- /dev/null +++ b/docs/backend-system/architecture/04-plugins.md @@ -0,0 +1,68 @@ +--- +id: plugins +title: Backend Plugins +sidebar_label: Plugins +# prettier-ignore +description: Backend plugins +--- + +## Creating Plugins + +Plugins are created using the `createBackendPlugin` function. All plugins must have an ID and a register method. Plugins may also accept an options object, which can be either optional or required. The options are passed to the second parameter of the register method, and the options type is inferred and forwarded to the returned plugin factory function. + +```ts +import { + configServiceRef, + coreServices, + createBackendPlugin, +} from '@backstage/backend-plugin-api'; + +// export type ExamplePluginOptions = { exampleOption: boolean }; +export const examplePlugin = createBackendPlugin({ + // unique id for the plugin + id: 'example', + // It's possible to provide options to the plugin + // register(env, options: ExamplePluginOptions) { + register(env) { + env.registerInit({ + deps: { + logger: coreServices.logger, + }, + // logger is provided by the backend based on the dependency on loggerServiceRef above. + async init({ logger }) { + logger.info('Hello from example plugin'); + }, + }); + }, +}); +``` + +The plugin can then be installed in the backend using the returned plugin factory function: + +```ts +backend.add(examplePlugin()); +``` + +If we wanted our plugin to accept options as well, we'd accept the options as the second parameter of the register method: + +```ts +export const examplePlugin = createBackendPlugin({ + id: 'example', + register(env, options?: { silent?: boolean }) { + env.registerInit({ + deps: { logger: coreServices.logger }, + async init({ logger }) { + if (!options?.silent) { + logger.info('Hello from example plugin'); + } + }, + }); + }, +}); +``` + +Passing the option to the plugin during installation looks like this: + +```ts +backend.add(examplePlugin({ silent: true })); +``` diff --git a/docs/backend-system/architecture/05-extension-points.md b/docs/backend-system/architecture/05-extension-points.md new file mode 100644 index 0000000000..d74a4690c9 --- /dev/null +++ b/docs/backend-system/architecture/05-extension-points.md @@ -0,0 +1,30 @@ +--- +id: extension-points +title: Backend Plugin Extension Points +sidebar_label: Extension Points +# prettier-ignore +description: Extension points of backend plugins +--- + +### Extension Points + +Modules depend on extension points just as a regular dependency by specifying it in the `deps` section. + +#### Defining an Extension Point + +```ts +import { createExtensionPoint } from '@backstage/backend-plugin-api'; + +export interface ScaffolderActionsExtensionPoint { + addAction(action: ScaffolderAction): void; +} + +export const scaffolderActionsExtensionPoint = + createExtensionPoint({ + id: 'scaffolder.actions', + }); +``` + +#### Registering an Extension Point + +Extension points are registered by a plugin and extended by modules. diff --git a/docs/backend-system/architecture/06-modules.md b/docs/backend-system/architecture/06-modules.md new file mode 100644 index 0000000000..428e51fafa --- /dev/null +++ b/docs/backend-system/architecture/06-modules.md @@ -0,0 +1,40 @@ +--- +id: modules +title: Plugin Modules +sidebar_label: Modules +# prettier-ignore +description: Modules for backend plugins +--- + +## Creating Modules + +Some facts about modules + +- A Module is able to extend a plugin with additional functionality using the `ExtensionPoint`s registered by the plugin. +- A module can only extend one plugin but can interact with multiple `ExtensionPoint`s registered by that plugin. +- A module is always initialized before the plugin it extends. + +A module depends on the `ExtensionPoint`s exported by the target plugin's library package, for example `@backstage/plugin-catalog-node`, and does not directly declare a dependency on the plugin package itself. + +Here's an example on how to create a module that adds a new processor using the `catalogProcessingExtensionPoint`: + +```ts +import { createBackendModule } from '@backstage/backend-plugin-api'; +import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node'; +import { MyCustomProcessor } from './processor'; + +export const exampleCustomProcessorCatalogModule = createBackendModule({ + moduleId: 'exampleCustomProcessor', + pluginId: 'catalog', + register(env) { + env.registerInit({ + deps: { + catalog: catalogProcessingExtensionPoint, + }, + async init({ catalog }) { + catalog.addProcessor(new MyCustomProcessor()); + }, + }); + }, +}); +``` diff --git a/docs/backend-system/building-backends/01-index.md b/docs/backend-system/building-backends/01-index.md new file mode 100644 index 0000000000..b9879f4b24 --- /dev/null +++ b/docs/backend-system/building-backends/01-index.md @@ -0,0 +1,12 @@ +--- +id: index +title: Building Backends +sidebar_label: Overview +# prettier-ignore +description: Building backends using the new backend system +--- + +> NOTE: If you have an existing backend that is not yet using the new backend +> system, see [migrating](./08-migrating.md). + +# Overview diff --git a/docs/backend-system/building-backends/08-migrating.md b/docs/backend-system/building-backends/08-migrating.md new file mode 100644 index 0000000000..84f801f15d --- /dev/null +++ b/docs/backend-system/building-backends/08-migrating.md @@ -0,0 +1,9 @@ +--- +id: migrating +title: Migrating your Backend to the New Backend System +sidebar_label: Migration Guide +# prettier-ignore +description: How to migrate existing backends to the new backend system +--- + +# Overview diff --git a/docs/backend-system/building-plugins-and-modules/01-index.md b/docs/backend-system/building-plugins-and-modules/01-index.md new file mode 100644 index 0000000000..fff82286ad --- /dev/null +++ b/docs/backend-system/building-plugins-and-modules/01-index.md @@ -0,0 +1,12 @@ +--- +id: index +title: Building Backend Plugins and Modules +sidebar_label: Overview +# prettier-ignore +description: Building backend plugins and modules using the new backend system +--- + +> NOTE: If you have an existing backend and/or backend plugins that are not yet +> using the new backend system, see [migrating](./08-migrating.md). + +# Overview diff --git a/docs/backend-system/building-plugins-and-modules/02-testing.md b/docs/backend-system/building-plugins-and-modules/02-testing.md new file mode 100644 index 0000000000..e8a8619f72 --- /dev/null +++ b/docs/backend-system/building-plugins-and-modules/02-testing.md @@ -0,0 +1,26 @@ +--- +id: testing +title: Testing Backend Plugins and Modules +sidebar_label: Testing +# prettier-ignore +description: Learn how to test your backend plugins and modules +--- + +Utilities for testing backend plugins and modules are available in `@backstage/backend-test-utils`. +`startTestBackend` returns a server which can be used together with `supertest` to test the plugins. + +```ts +import { startTestBackend } from '@backstage/backend-test-utils'; +import request from 'supertest'; + +describe('My plugin tests', () => { + it('should return 200', async () => { + const { server } = await startTestBackend({ + features: [myPlugin()], + }); + + const response = await request(server).get('/api/example/hello'); + expect(response.status).toBe(200); + }); +}); +``` diff --git a/docs/backend-system/building-plugins-and-modules/08-migrating.md b/docs/backend-system/building-plugins-and-modules/08-migrating.md new file mode 100644 index 0000000000..2f3fd5ebcc --- /dev/null +++ b/docs/backend-system/building-plugins-and-modules/08-migrating.md @@ -0,0 +1,9 @@ +--- +id: migrating +title: Migrating your Backend Plugin to the New Backend System +sidebar_label: Migration Guide +# prettier-ignore +description: How to migrate existing backend plugins to the new backend system +--- + +# Overview diff --git a/docs/backend-system/core-services/01-index.md b/docs/backend-system/core-services/01-index.md new file mode 100644 index 0000000000..875227f0c4 --- /dev/null +++ b/docs/backend-system/core-services/01-index.md @@ -0,0 +1,75 @@ +--- +id: index +title: Core Backend Service APIs +sidebar_label: Core Services +# prettier-ignore +description: Core backend service APIs +--- + +The default backend provides several [core services](https://github.com/backstage/backstage/blob/master/packages/backend-plugin-api/src/services/definitions/coreServices.ts) out of the box which includes access to configuration, logging, URL Readers, databases and more. + +All core services are available through the `coreServices` namespace in the `@backstage/backend-plugin-api` package. + +```ts +import { coreServices } from '@backstage/backend-plugin-api'; +``` + +## HTTP Router Service + +One of the most common services is the HTTP router service which is used to expose HTTP endpoints for other plugins to consume. + +The following example shows how to register a HTTP router for the `example` plugin. +This single route will be available at the `/api/example/hello` path. + +```ts +import { + coreServices, + createBackendPlugin, +} from '@backstage/backend-plugin-api'; +import { Router } from 'express'; + +createBackendPlugin({ + id: 'example', + register(env) { + env.registerInit({ + deps: { http: coreServices.httpRouter }, + async init({ http }) { + const router = Router(); + router.get('/hello', (_req, res) => { + res.status(200).json({ hello: 'world' }); + }); + http.use(router); + }, + }); + }, +}); +``` + +## Logging and Configuration Service + +It is common for plugins to need access to configuration values and log. + +```ts +import { + coreServices, + createBackendPlugin, +} from '@backstage/backend-plugin-api'; +import { Router } from 'express'; + +createBackendPlugin({ + id: 'example', + register(env) { + env.registerInit({ + deps: { + logger: coreServices.logger, + config: coreServices.config, + }, + + async init({ config, logger }) { + const url = config.getString('backend.baseUrl'); + c; + }, + }); + }, +}); +``` diff --git a/docs/backend-system/index.md b/docs/backend-system/index.md new file mode 100644 index 0000000000..ca13bc188c --- /dev/null +++ b/docs/backend-system/index.md @@ -0,0 +1,15 @@ +--- +id: index +title: The Backend System +sidebar_label: Overview +# prettier-ignore +description: The backend system +--- + +> **DISCLAIMER: The new backend system is under active development and is not considered stable** + +## Status + +The new backend system is under active development, and only a small number of plugins have been migrated so far. It is possible to try it out, but it is not recommended to use this new system in production yet. + +You can find an example backend setup in [the backend-next package](https://github.com/backstage/backstage/tree/master/packages/backend-next). diff --git a/microsite/sidebars.json b/microsite/sidebars.json index f8746c776c..ab8c29c9a4 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -362,6 +362,44 @@ "architecture-decisions/adrs-adr012", "architecture-decisions/adrs-adr013" ], - "FAQ": ["FAQ"] + "FAQ": ["FAQ"], + "Experimental Backend System": [ + "backend-system/index", + { + "type": "subcategory", + "label": "Architecture", + "ids": [ + "backend-system/architecture/index", + "backend-system/architecture/services", + "backend-system/architecture/plugins", + "backend-system/architecture/extension-points", + "backend-system/architecture/modules" + ] + }, + { + "type": "subcategory", + "label": "Building Backends", + "ids": [ + "backend-system/building-backends/index", + "backend-system/building-backends/migrating" + ] + }, + { + "type": "subcategory", + "label": "Building Plugins & Modules", + "ids": [ + "backend-system/building-plugins-and-modules/index", + "backend-system/building-plugins-and-modules/testing", + "backend-system/building-plugins-and-modules/migrating" + ] + }, + { + "type": "subcategory", + "label": "Core Services", + "ids": [ + "backend-system/core-services/index" + ] + } + ] } } From c5b529ed164b3d387e420db997120e55a048c76a Mon Sep 17 00:00:00 2001 From: pitwegner Date: Tue, 17 Jan 2023 15:55:47 +0100 Subject: [PATCH 043/182] add tests and apireport Signed-off-by: pitwegner --- plugins/scaffolder-backend/api-report.md | 1 + .../actions/builtin/fetch/template.test.ts | 77 +++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index 1a3c1db03f..7f8ab0ad79 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -127,6 +127,7 @@ export function createFetchTemplateAction(options: { copyWithoutRender?: string[] | undefined; copyWithoutTemplating?: string[] | undefined; cookiecutterCompat?: boolean | undefined; + replace?: boolean | undefined; }>; // @public diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.test.ts index 6b03290e1e..6658e1915d 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.test.ts @@ -669,4 +669,81 @@ describe('fetch:template', () => { ).resolves.toEqual('test-project: 1234'); }); }); + + describe('with replacement of existing files', () => { + let context: ActionContext; + + beforeEach(async () => { + context = mockContext({ + values: { + name: 'test-project', + count: 1234, + }, + replace: true, + }); + + mockFetchContents.mockImplementation(({ outputPath }) => { + mockFs({ + ...realFiles, + [joinPath(workspacePath, 'target')]: { + 'static-content.txt': 'static-content', + }, + [outputPath]: { + 'static-content.txt': '${{ values.name }}: ${{ values.count }}', + }, + }); + + return Promise.resolve(); + }); + + await action.handler(context); + }); + + afterEach(() => { + mockFs.restore(); + }); + + it('overwrites existing file', async () => { + await expect( + fs.readFile(`${workspacePath}/target/static-content.txt`, 'utf-8'), + ).resolves.toEqual('test-project: 1234'); + }); + }); + + describe('without replacement of existing files', () => { + let context: ActionContext; + + beforeEach(async () => { + context = mockContext({ + values: { + name: 'test-project', + count: 1234, + }, + targetPath: './target', + replace: false, + }); + + mockFetchContents.mockImplementation(({ outputPath }) => { + mockFs({ + ...realFiles, + [joinPath(workspacePath, 'target')]: { + 'static-content.txt': 'static-content', + }, + [outputPath]: { + 'static-content.txt': '${{ values.name }}: ${{ values.count }}', + }, + }); + + return Promise.resolve(); + }); + + await action.handler(context); + }); + + it('keeps existing file', async () => { + await expect( + fs.readFile(`${workspacePath}/target/static-content.txt`, 'utf-8'), + ).resolves.toEqual('static-content'); + }); + }); }); From 7a2417b9c2a45f89ef4197cd1208ca4352ba07f8 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 17 Jan 2023 16:02:44 +0100 Subject: [PATCH 044/182] docs: hide backend system docs for now MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: Johan Haals Signed-off-by: Patrik Oldsberg --- microsite/sidebars.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/microsite/sidebars.json b/microsite/sidebars.json index ab8c29c9a4..01d2b95b6b 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -363,7 +363,7 @@ "architecture-decisions/adrs-adr013" ], "FAQ": ["FAQ"], - "Experimental Backend System": [ + "Experimental Backend System": [{"_hidden_": [ "backend-system/index", { "type": "subcategory", @@ -400,6 +400,6 @@ "backend-system/core-services/index" ] } - ] + ]}] } } From 219ddcf6a87b2a79b28f2b8e15b91a50082612c2 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 17 Jan 2023 16:03:16 +0100 Subject: [PATCH 045/182] docs/backend-system: update service example to match reality MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: Johan Haals Signed-off-by: Patrik Oldsberg --- .../architecture/03-services.md | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/docs/backend-system/architecture/03-services.md b/docs/backend-system/architecture/03-services.md index adedbc6383..27fea27db9 100644 --- a/docs/backend-system/architecture/03-services.md +++ b/docs/backend-system/architecture/03-services.md @@ -36,8 +36,10 @@ export const exampleServiceRef = createServiceRef({ id: 'example', scope: 'plugin', // can be 'root' or 'plugin' - // The defaultFactory is optional to implement but it will be used if no other factory is provided to the backend. - // This is allows for the backend to provide a default implementation of the service without having to wire it beforehand. + // The defaultFactory is optional to implement but it will be used if no + // other factory is provided to the backend. This allows for the backend + // to provide a default implementation of the service without having to wire + // it beforehand. defaultFactory: async service => createServiceFactory({ service, @@ -45,14 +47,21 @@ export const exampleServiceRef = createServiceRef({ logger: coreServices.logger, plugin: coreServices.pluginMetadata, }, - // Logger is available directly in the factory as it's a root scoped service and will be created once per backend instance. - async factory({ logger, plugin }) { - // plugin is available as it's a plugin scoped service and will be created once per plugin. - return async ({ plugin }) => { - // This block will be executed once for every plugin that depends on this service - logger.info('Initializing example service plugin instance'); - return new ExampleImpl({ logger, plugin }); - }; + // This root context method is only available for plugin scoped services. + // It's only called once per backend instance. + // Logger is available at the root context level as it's also a root + // scoped service. + createRootContext({ logger }) { + return new ExampleImplFactory({ logger }); + }, + // Plugin is available as it's a plugin scoped service and will be + // created once per plugin. The logger can be had here too if needed. + // Both this anc the root context can also be async. + factory({ logger, plugin }, rootContext) { + // This block will be executed once for every plugin that depends on + // this service + logger.info('Initializing example service plugin instance'); + return rootContext.forPlugin(plugin.getId()); }, }), }); From f47c82f16e15b92d2f4bb7aab8429fd29f7b4f4c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 17 Jan 2023 16:12:48 +0100 Subject: [PATCH 046/182] docs/backend-system: core service doc tweaks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: Johan Haals Signed-off-by: Patrik Oldsberg --- docs/backend-system/core-services/01-index.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/backend-system/core-services/01-index.md b/docs/backend-system/core-services/01-index.md index 875227f0c4..cb1ec157ae 100644 --- a/docs/backend-system/core-services/01-index.md +++ b/docs/backend-system/core-services/01-index.md @@ -38,6 +38,7 @@ createBackendPlugin({ router.get('/hello', (_req, res) => { res.status(200).json({ hello: 'world' }); }); + // Registers the router at the /api/example path http.use(router); }, }); @@ -47,7 +48,7 @@ createBackendPlugin({ ## Logging and Configuration Service -It is common for plugins to need access to configuration values and log. +It is common for plugins to need access to configuration values and log messages. ```ts import { @@ -61,13 +62,13 @@ createBackendPlugin({ register(env) { env.registerInit({ deps: { - logger: coreServices.logger, + log: coreServices.logger, config: coreServices.config, }, - - async init({ config, logger }) { + async init({ config, log }) { + log.warn('Brace yourself for more log output'); const url = config.getString('backend.baseUrl'); - c; + log.info(`Backend URL is running on ${url}`); }, }); }, From 6762810d076e4ce91365608a7f6a52103bec62ba Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 17 Jan 2023 16:13:03 +0100 Subject: [PATCH 047/182] docs/backend-system: remove old doc from sidebar MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fredrik Adelöw Co-authored-by: Johan Haals Signed-off-by: Patrik Oldsberg --- microsite/sidebars.json | 3 +-- mkdocs.yml | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/microsite/sidebars.json b/microsite/sidebars.json index 01d2b95b6b..6008dfed56 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -238,8 +238,7 @@ "plugins/proxying", "plugins/backend-plugin", "plugins/call-existing-api", - "plugins/url-reader", - "plugins/new-backend-system" + "plugins/url-reader" ] }, { diff --git a/mkdocs.yml b/mkdocs.yml index f3194f0d61..38369f4748 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -138,7 +138,6 @@ nav: - Backend plugin: 'plugins/backend-plugin.md' - Call existing API: 'plugins/call-existing-api.md' - URL Reader: 'plugins/url-reader.md' - - New Backend System: 'plugins/new-backend-system.md' - Testing: - Testing with Jest: 'plugins/testing.md' - Publishing: From 9673364f56d3a171d46133ca951fea0c3deda060 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 17 Jan 2023 16:26:58 +0100 Subject: [PATCH 048/182] microsite: format sidebars.json Signed-off-by: Patrik Oldsberg --- microsite/sidebars.json | 72 +++++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 35 deletions(-) diff --git a/microsite/sidebars.json b/microsite/sidebars.json index 6008dfed56..acc83a6871 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -362,43 +362,45 @@ "architecture-decisions/adrs-adr013" ], "FAQ": ["FAQ"], - "Experimental Backend System": [{"_hidden_": [ - "backend-system/index", + "Experimental Backend System": [ { - "type": "subcategory", - "label": "Architecture", - "ids": [ - "backend-system/architecture/index", - "backend-system/architecture/services", - "backend-system/architecture/plugins", - "backend-system/architecture/extension-points", - "backend-system/architecture/modules" - ] - }, - { - "type": "subcategory", - "label": "Building Backends", - "ids": [ - "backend-system/building-backends/index", - "backend-system/building-backends/migrating" - ] - }, - { - "type": "subcategory", - "label": "Building Plugins & Modules", - "ids": [ - "backend-system/building-plugins-and-modules/index", - "backend-system/building-plugins-and-modules/testing", - "backend-system/building-plugins-and-modules/migrating" - ] - }, - { - "type": "subcategory", - "label": "Core Services", - "ids": [ - "backend-system/core-services/index" + "_hidden_": [ + "backend-system/index", + { + "type": "subcategory", + "label": "Architecture", + "ids": [ + "backend-system/architecture/index", + "backend-system/architecture/services", + "backend-system/architecture/plugins", + "backend-system/architecture/extension-points", + "backend-system/architecture/modules" + ] + }, + { + "type": "subcategory", + "label": "Building Backends", + "ids": [ + "backend-system/building-backends/index", + "backend-system/building-backends/migrating" + ] + }, + { + "type": "subcategory", + "label": "Building Plugins & Modules", + "ids": [ + "backend-system/building-plugins-and-modules/index", + "backend-system/building-plugins-and-modules/testing", + "backend-system/building-plugins-and-modules/migrating" + ] + }, + { + "type": "subcategory", + "label": "Core Services", + "ids": ["backend-system/core-services/index"] + } ] } - ]}] + ] } } From 70febd490d052d9f1ec45491308e39665f9c9e0f Mon Sep 17 00:00:00 2001 From: Damon Kaswell Date: Tue, 17 Jan 2023 07:46:15 -0800 Subject: [PATCH 049/182] Better event handling Signed-off-by: Damon Kaswell --- .../api-report.md | 23 +++--- .../src/engine/IncrementalIngestionEngine.ts | 74 ++++++++++--------- .../src/router/routes.ts | 2 +- .../src/types.ts | 28 ++++--- 4 files changed, 68 insertions(+), 59 deletions(-) diff --git a/plugins/catalog-backend-module-incremental-ingestion/api-report.md b/plugins/catalog-backend-module-incremental-ingestion/api-report.md index 1052afda74..3fc932ad2d 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/api-report.md +++ b/plugins/catalog-backend-module-incremental-ingestion/api-report.md @@ -10,6 +10,7 @@ import { CatalogBuilder } from '@backstage/plugin-catalog-backend'; import type { Config } from '@backstage/config'; import type { DeferredEntity } from '@backstage/plugin-catalog-backend'; import type { DurationObjectUnits } from 'luxon'; +import { EventParams } from '@backstage/plugin-events-node'; import type { Logger } from 'winston'; import type { PermissionEvaluator } from '@backstage/plugin-permission-common'; import type { PluginDatabaseManager } from '@backstage/backend-common'; @@ -55,16 +56,18 @@ export interface IncrementalEntityProvider { context: TContext, cursor?: TCursor, ): Promise>; - onEvent?: (payload: unknown) => { - delta: - | { - added: DeferredEntity[]; - removed: { - entityRef: string; - }[]; - } - | undefined; - }; + onEvent?: (params: EventParams) => + | { + delta: + | { + added: DeferredEntity[]; + removed: { + entityRef: string; + }[]; + } + | undefined; + } + | undefined; } // @public (undocumented) diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts b/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts index a21092a838..c1a62b7a73 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/engine/IncrementalIngestionEngine.ts @@ -336,14 +336,14 @@ export class IncrementalIngestionEngine } async onEvent(params: EventParams): Promise { - const { topic, eventPayload } = params; + const { topic } = params; if (topic !== this.providerEventTopic) { return; } const { logger, provider, connection } = this.options; const providerName = provider.getProviderName(); - logger.info( + logger.debug( `incremental-engine: Received ${this.providerEventTopic} event`, ); @@ -351,48 +351,50 @@ export class IncrementalIngestionEngine return; } - const update = provider.onEvent(eventPayload); + const update = provider.onEvent(params); - if (update.delta) { - if (update.delta.added.length > 0) { - const ingestionRecord = await this.manager.getCurrentIngestionRecord( - providerName, - ); - - if (!ingestionRecord) { - logger.debug( - `incremental-engine: Skipping delta addition because incremental ingestion is restarting.`, + if (update) { + if (update.delta) { + if (update.delta.added.length > 0) { + const ingestionRecord = await this.manager.getCurrentIngestionRecord( + providerName, ); - } else { - const mark = - ingestionRecord.status === 'resting' - ? await this.manager.getLastMark(ingestionRecord.id) - : await this.manager.getFirstMark(ingestionRecord.id); - if (!mark) { - throw new Error( - `Cannot apply delta, page records are missing! Please re-run incremental ingestion for ${providerName}.`, + if (!ingestionRecord) { + logger.debug( + `incremental-engine: Skipping delta addition because incremental ingestion is restarting.`, ); + } else { + const mark = + ingestionRecord.status === 'resting' + ? await this.manager.getLastMark(ingestionRecord.id) + : await this.manager.getFirstMark(ingestionRecord.id); + + if (!mark) { + throw new Error( + `Cannot apply delta, page records are missing! Please re-run incremental ingestion for ${providerName}.`, + ); + } + await this.manager.createMarkEntities(mark.id, update.delta.added); } - await this.manager.createMarkEntities(mark.id, update.delta.added); } - } - if (update.delta.removed.length > 0) { - await this.manager.deleteEntityRecordsByRef(update.delta.removed); - } + if (update.delta.removed.length > 0) { + await this.manager.deleteEntityRecordsByRef(update.delta.removed); + } - await connection.applyMutation({ - type: 'delta', - ...update.delta, - }); - logger.info( - `incremental-engine: Processed ${this.providerEventTopic} event`, - ); - } else { - logger.warn( - `incremental-engine: Rejected ${this.providerEventTopic} event - empty or invalid`, - ); + await connection.applyMutation({ + type: 'delta', + ...update.delta, + }); + logger.debug( + `incremental-engine: Processed ${this.providerEventTopic} event`, + ); + } else { + logger.warn( + `incremental-engine: Rejected ${this.providerEventTopic} event - empty or invalid`, + ); + } } } diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts b/plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts index d03619d5a4..b3e3ae5792 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/router/routes.ts @@ -239,7 +239,7 @@ export class IncrementalProviderRouter implements EventPublisher { }); }); - router.post(`${PROVIDER_BASE_PATH}/delta`, async (req, res) => { + router.post(`${PROVIDER_BASE_PATH}/event`, async (req, res) => { const { provider } = req.params; const topic = `${provider}-push`; diff --git a/plugins/catalog-backend-module-incremental-ingestion/src/types.ts b/plugins/catalog-backend-module-incremental-ingestion/src/types.ts index 315757d88a..efadee3a76 100644 --- a/plugins/catalog-backend-module-incremental-ingestion/src/types.ts +++ b/plugins/catalog-backend-module-incremental-ingestion/src/types.ts @@ -27,6 +27,7 @@ import type { DeferredEntity, EntityProviderConnection, } from '@backstage/plugin-catalog-backend'; +import { EventParams } from '@backstage/plugin-events-node'; import type { PermissionEvaluator } from '@backstage/plugin-permission-common'; import type { DurationObjectUnits } from 'luxon'; import type { Logger } from 'winston'; @@ -77,20 +78,23 @@ export interface IncrementalEntityProvider { around(burst: (context: TContext) => Promise): Promise; /** - * If present, this method accepts an incoming event, and maps the - * payload to an object containing a delta mutation. + * This method accepts an incoming event for the provider, and + * (optionally) maps the payload to an object containing a delta + * mutation. * - * If a delta is present, the incremental entity provider will apply - * it automatically. + * If a valid delta is returned by this method, it will be ingested + * automatically by the provider. */ - onEvent?: (payload: unknown) => { - delta: - | { - added: DeferredEntity[]; - removed: { entityRef: string }[]; - } - | undefined; - }; + onEvent?: (params: EventParams) => + | { + delta: + | { + added: DeferredEntity[]; + removed: { entityRef: string }[]; + } + | undefined; + } + | undefined; } /** From 2ca355e4913ed7b2446b1a5d515a85d9be63f99d Mon Sep 17 00:00:00 2001 From: Paul Cowan Date: Thu, 8 Dec 2022 12:41:26 +0000 Subject: [PATCH 050/182] Add SecurityTab Signed-off-by: Paul Cowan --- .../components/SecurityTab/SecurityTab.tsx | 25 +++++++++++++++++++ .../app/src/components/SecurityTab/index.ts | 17 +++++++++++++ .../components/SecurityTab/showSecurityTab.ts | 22 ++++++++++++++++ .../app/src/components/catalog/EntityPage.tsx | 10 ++++++++ .../examples/all-components.yaml | 24 +++++++++--------- .../components/wayback-search-component.yaml | 2 ++ 6 files changed, 88 insertions(+), 12 deletions(-) create mode 100644 packages/app/src/components/SecurityTab/SecurityTab.tsx create mode 100644 packages/app/src/components/SecurityTab/index.ts create mode 100644 packages/app/src/components/SecurityTab/showSecurityTab.ts diff --git a/packages/app/src/components/SecurityTab/SecurityTab.tsx b/packages/app/src/components/SecurityTab/SecurityTab.tsx new file mode 100644 index 0000000000..2adca3b0ef --- /dev/null +++ b/packages/app/src/components/SecurityTab/SecurityTab.tsx @@ -0,0 +1,25 @@ +/* eslint-disable no-console */ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import { useEntity } from '@backstage/plugin-catalog-react'; + +export function SecurityTab(): JSX.Element | null { + const entity = useEntity(); + console.log(entity); + return

Security

; +} diff --git a/packages/app/src/components/SecurityTab/index.ts b/packages/app/src/components/SecurityTab/index.ts new file mode 100644 index 0000000000..2c7908e958 --- /dev/null +++ b/packages/app/src/components/SecurityTab/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export { SecurityTab } from './SecurityTab'; +export { showSecurityTab } from './showSecurityTab'; diff --git a/packages/app/src/components/SecurityTab/showSecurityTab.ts b/packages/app/src/components/SecurityTab/showSecurityTab.ts new file mode 100644 index 0000000000..364436e44e --- /dev/null +++ b/packages/app/src/components/SecurityTab/showSecurityTab.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Entity } from '@backstage/catalog-model'; + +export const SECURITY_ANNOTATION = 'myorg.org/security'; + +/** @public */ +export const showSecurityTab = (entity: Entity) => + Boolean(entity.metadata.annotations?.[SECURITY_ANNOTATION]); diff --git a/packages/app/src/components/catalog/EntityPage.tsx b/packages/app/src/components/catalog/EntityPage.tsx index 02b00f038c..6288dc86ee 100644 --- a/packages/app/src/components/catalog/EntityPage.tsx +++ b/packages/app/src/components/catalog/EntityPage.tsx @@ -154,6 +154,8 @@ import { ReportIssue, } from '@backstage/plugin-techdocs-module-addons-contrib'; import { EntityCostInsightsContent } from '@backstage/plugin-cost-insights'; +import { SecurityTab } from '../SecurityTab/SecurityTab'; +import { showSecurityTab } from '../SecurityTab'; const customEntityFilterKind = ['Component', 'API', 'System']; @@ -400,6 +402,12 @@ const overviewContent = ( ); +const securityContent = ( + + + +); + const serviceEntityPage = ( @@ -440,6 +448,8 @@ const serviceEntityPage = ( {techdocsContent} + {securityContent} + Date: Thu, 8 Dec 2022 14:46:43 +0000 Subject: [PATCH 051/182] move TemplateWizardPage content into separate component Signed-off-by: Paul Cowan --- .../components/SecurityTab/SecurityTab.tsx | 31 ++++- plugins/scaffolder/src/next/Router/Router.tsx | 2 + plugins/scaffolder/src/next/Router/index.ts | 2 +- .../TemplateWizardContent.tsx | 112 ++++++++++++++++++ .../src/next/TemplateWizardContent/index.ts | 16 +++ .../TemplateWizardPage/TemplateWizardPage.tsx | 84 ++----------- plugins/scaffolder/src/next/index.ts | 1 + 7 files changed, 169 insertions(+), 79 deletions(-) create mode 100644 plugins/scaffolder/src/next/TemplateWizardContent/TemplateWizardContent.tsx create mode 100644 plugins/scaffolder/src/next/TemplateWizardContent/index.ts diff --git a/packages/app/src/components/SecurityTab/SecurityTab.tsx b/packages/app/src/components/SecurityTab/SecurityTab.tsx index 2adca3b0ef..af4d66320d 100644 --- a/packages/app/src/components/SecurityTab/SecurityTab.tsx +++ b/packages/app/src/components/SecurityTab/SecurityTab.tsx @@ -1,4 +1,3 @@ -/* eslint-disable no-console */ /* * Copyright 2022 The Backstage Authors * @@ -16,10 +15,30 @@ */ import React from 'react'; -import { useEntity } from '@backstage/plugin-catalog-react'; +import { + TemplateWizardContent, + useGetCustomFields, +} from '@backstage/plugin-scaffolder'; -export function SecurityTab(): JSX.Element | null { - const entity = useEntity(); - console.log(entity); - return

Security

; +interface SecurityTabProps { + customExtensionsElement: React.ReactNode; +} + +export function SecurityTab(props: SecurityTabProps): JSX.Element | null { + // eslint-disable-next-line no-alert + const onComplete = async () => alert('success!!!!'); + const onError = (error: Error | undefined) => ( +

{error?.message ?? 'Houston we have a problem.'}

+ ); + const fieldExtensions = useGetCustomFields(props.customExtensionsElement); + + return ( + + ); } diff --git a/plugins/scaffolder/src/next/Router/Router.tsx b/plugins/scaffolder/src/next/Router/Router.tsx index 784e17dd55..a7bf16e033 100644 --- a/plugins/scaffolder/src/next/Router/Router.tsx +++ b/plugins/scaffolder/src/next/Router/Router.tsx @@ -52,7 +52,9 @@ export type NextRouterProps = { */ export const Router = (props: PropsWithChildren) => { const { components: { TemplateCardComponent } = {} } = props; + const outlet = useOutlet() || props.children; + const customFieldExtensions = useCustomFieldExtensions(outlet); const fieldExtensions = [ diff --git a/plugins/scaffolder/src/next/Router/index.ts b/plugins/scaffolder/src/next/Router/index.ts index dac1db7b3a..b4df375d1d 100644 --- a/plugins/scaffolder/src/next/Router/index.ts +++ b/plugins/scaffolder/src/next/Router/index.ts @@ -13,5 +13,5 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { Router } from './Router'; +export { Router, useGetCustomFields } from './Router'; export type { NextRouterProps } from './Router'; diff --git a/plugins/scaffolder/src/next/TemplateWizardContent/TemplateWizardContent.tsx b/plugins/scaffolder/src/next/TemplateWizardContent/TemplateWizardContent.tsx new file mode 100644 index 0000000000..4dfc9be9e5 --- /dev/null +++ b/plugins/scaffolder/src/next/TemplateWizardContent/TemplateWizardContent.tsx @@ -0,0 +1,112 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import React, { useEffect } from 'react'; +import { + Content, + Header, + InfoCard, + MarkdownContent, + Page, + Progress, +} from '@backstage/core-components'; +import { stringifyEntityRef } from '@backstage/catalog-model'; +import { useTemplateParameterSchema } from '../TemplateWizardPage/TemplateWizardPage'; +import { NextFieldExtensionOptions } from '../../extensions'; +import type { ErrorTransformer } from '@rjsf/utils'; +import type { JsonValue } from '@backstage/types'; +import { makeStyles } from '@material-ui/core'; +import { BackstageTheme } from '@backstage/theme'; +import { errorApiRef, useApi } from '@backstage/core-plugin-api'; +import { Stepper } from '../TemplateWizardPage/Stepper'; + +const useStyles = makeStyles(() => ({ + markdown: { + /** to make the styles for React Markdown not leak into the description */ + '& :first-child': { + marginTop: 0, + }, + '& :last-child': { + marginBottom: 0, + }, + }, +})); + +export interface TemplateWizardContentProps { + namespace: string; + templateName: string; + customFieldExtensions: NextFieldExtensionOptions[]; + transformErrors?: ErrorTransformer; + onComplete: (values: Record) => Promise; + onError(error: Error | undefined): JSX.Element | null; +} + +export const TemplateWizardContent = ( + props: TemplateWizardContentProps, +): JSX.Element | null => { + const styles = useStyles(); + const templateRef = stringifyEntityRef({ + kind: 'Template', + namespace: props.namespace, + name: props.templateName, + }); + + const errorApi = useApi(errorApiRef); + + const { loading, manifest, error } = useTemplateParameterSchema(templateRef); + + useEffect(() => { + if (error) { + errorApi.post(new Error(`Failed to load template, ${error}`)); + } + }, [error, errorApi]); + + if (error) { + return props.onError(error); + } + + return ( + +
+ + {loading && } + {manifest && ( + + } + noPadding + titleTypographyProps={{ component: 'h2' }} + > + + + )} + + + ); +}; diff --git a/plugins/scaffolder/src/next/TemplateWizardContent/index.ts b/plugins/scaffolder/src/next/TemplateWizardContent/index.ts new file mode 100644 index 0000000000..3fda3efb4f --- /dev/null +++ b/plugins/scaffolder/src/next/TemplateWizardContent/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export { TemplateWizardContent } from './TemplateWizardContent'; diff --git a/plugins/scaffolder/src/next/TemplateWizardPage/TemplateWizardPage.tsx b/plugins/scaffolder/src/next/TemplateWizardPage/TemplateWizardPage.tsx index c4e8a51e1f..4ff47e41e9 100644 --- a/plugins/scaffolder/src/next/TemplateWizardPage/TemplateWizardPage.tsx +++ b/plugins/scaffolder/src/next/TemplateWizardPage/TemplateWizardPage.tsx @@ -13,20 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import React, { useEffect } from 'react'; -import { - Page, - Header, - Content, - Progress, - InfoCard, - MarkdownContent, -} from '@backstage/core-components'; +import React from 'react'; import { Navigate, useNavigate } from 'react-router-dom'; import { stringifyEntityRef } from '@backstage/catalog-model'; import { AnalyticsContext, - errorApiRef, useApi, useRouteRef, useRouteRefParams, @@ -34,37 +25,21 @@ import { import { scaffolderApiRef, useTemplateSecrets, -} from '@backstage/plugin-scaffolder-react'; -import useAsync from 'react-use/lib/useAsync'; -import { makeStyles } from '@material-ui/core'; -import { BackstageTheme } from '@backstage/theme'; -import { - Stepper, NextFieldExtensionOptions, } from '@backstage/plugin-scaffolder-react'; +import useAsync from 'react-use/lib/useAsync'; import { JsonValue } from '@backstage/types'; import { FormProps } from '../types'; import { nextRouteRef } from '../routes'; import { scaffolderTaskRouteRef, selectedTemplateRouteRef } from '../../routes'; +import { TemplateWizardContent } from '../TemplateWizardContent/TemplateWizardContent'; type TemplateWizardPageProps = { customFieldExtensions: NextFieldExtensionOptions[]; FormProps?: FormProps; }; -const useStyles = makeStyles(() => ({ - markdown: { - /** to make the styles for React Markdown not leak into the description */ - '& :first-child': { - marginTop: 0, - }, - '& :last-child': { - marginBottom: 0, - }, - }, -})); - -const useTemplateParameterSchema = (templateRef: string) => { +export const useTemplateParameterSchema = (templateRef: string) => { const scaffolderApi = useApi(scaffolderApiRef); const { value, loading, error } = useAsync( () => scaffolderApi.getTemplateParameterSchema(templateRef), @@ -75,7 +50,6 @@ const useTemplateParameterSchema = (templateRef: string) => { }; export const TemplateWizardPage = (props: TemplateWizardPageProps) => { - const styles = useStyles(); const rootRef = useRouteRef(nextRouteRef); const taskRoute = useRouteRef(scaffolderTaskRouteRef); const { secrets } = useTemplateSecrets(); @@ -91,9 +65,6 @@ export const TemplateWizardPage = (props: TemplateWizardPageProps) => { name: templateName, }); - const errorApi = useApi(errorApiRef); - const { loading, manifest, error } = useTemplateParameterSchema(templateRef); - const onComplete = async (values: Record) => { const { taskId } = await scaffolderApi.scaffold({ templateRef, @@ -104,48 +75,17 @@ export const TemplateWizardPage = (props: TemplateWizardPageProps) => { navigate(taskRoute({ taskId })); }; - useEffect(() => { - if (error) { - errorApi.post(new Error(`Failed to load template, ${error}`)); - } - }, [error, errorApi]); - - if (error) { - return ; - } + const onError = () => ; return ( - -
- - {loading && } - {manifest && ( - - } - noPadding - titleTypographyProps={{ component: 'h2' }} - > - - - )} - - + ); }; diff --git a/plugins/scaffolder/src/next/index.ts b/plugins/scaffolder/src/next/index.ts index 1ca9096459..fd88d57551 100644 --- a/plugins/scaffolder/src/next/index.ts +++ b/plugins/scaffolder/src/next/index.ts @@ -18,3 +18,4 @@ export * from './TemplateListPage'; export * from './TemplateWizardPage'; export * from './types'; export * from './routes'; +export * from './TemplateWizardContent'; From a647ff761ee28567caada8027ff8cd378f535507 Mon Sep 17 00:00:00 2001 From: Paul Cowan Date: Thu, 8 Dec 2022 14:53:17 +0000 Subject: [PATCH 052/182] useGetCustomFields Signed-off-by: Paul Cowan --- .../app/src/components/SecurityTab/SecurityTab.tsx | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/app/src/components/SecurityTab/SecurityTab.tsx b/packages/app/src/components/SecurityTab/SecurityTab.tsx index af4d66320d..3a40f9c1d7 100644 --- a/packages/app/src/components/SecurityTab/SecurityTab.tsx +++ b/packages/app/src/components/SecurityTab/SecurityTab.tsx @@ -21,16 +21,20 @@ import { } from '@backstage/plugin-scaffolder'; interface SecurityTabProps { - customExtensionsElement: React.ReactNode; + customExtensionsElement?: React.ReactNode; } -export function SecurityTab(props: SecurityTabProps): JSX.Element | null { +export function SecurityTab({ + customExtensionsElement = <>, +}: SecurityTabProps): JSX.Element | null { // eslint-disable-next-line no-alert const onComplete = async () => alert('success!!!!'); + const onError = (error: Error | undefined) => (

{error?.message ?? 'Houston we have a problem.'}

); - const fieldExtensions = useGetCustomFields(props.customExtensionsElement); + + const fieldExtensions = useGetCustomFields(customExtensionsElement); return ( Date: Thu, 8 Dec 2022 15:12:43 +0000 Subject: [PATCH 053/182] wrap TemplateWizardContent in Signed-off-by: Paul Cowan --- .../app/src/components/SecurityTab/SecurityTab.tsx | 4 ++-- .../TemplateWizardContent/TemplateWizardContent.tsx | 10 ++++++++++ .../scaffolder/src/next/TemplateWizardContent/index.ts | 5 ++++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/packages/app/src/components/SecurityTab/SecurityTab.tsx b/packages/app/src/components/SecurityTab/SecurityTab.tsx index 3a40f9c1d7..69c2455ea5 100644 --- a/packages/app/src/components/SecurityTab/SecurityTab.tsx +++ b/packages/app/src/components/SecurityTab/SecurityTab.tsx @@ -16,7 +16,7 @@ import React from 'react'; import { - TemplateWizardContent, + TemplateContent, useGetCustomFields, } from '@backstage/plugin-scaffolder'; @@ -37,7 +37,7 @@ export function SecurityTab({ const fieldExtensions = useGetCustomFields(customExtensionsElement); return ( - (() => ({ markdown: { @@ -110,3 +114,9 @@ export const TemplateWizardContent = ( ); }; + +export const TemplateContent = (props: TemplateWizardContentProps) => ( + + + +); diff --git a/plugins/scaffolder/src/next/TemplateWizardContent/index.ts b/plugins/scaffolder/src/next/TemplateWizardContent/index.ts index 3fda3efb4f..fdb0589cc2 100644 --- a/plugins/scaffolder/src/next/TemplateWizardContent/index.ts +++ b/plugins/scaffolder/src/next/TemplateWizardContent/index.ts @@ -13,4 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { TemplateWizardContent } from './TemplateWizardContent'; +export { + TemplateWizardContent, + TemplateContent, +} from './TemplateWizardContent'; From e2789aea3d0bd0d170e3b5cff1fab2a7fce3d6fa Mon Sep 17 00:00:00 2001 From: Paul Cowan Date: Thu, 8 Dec 2022 15:59:28 +0000 Subject: [PATCH 054/182] prefill template content Signed-off-by: Paul Cowan --- .../app/src/components/SecurityTab/SecurityTab.tsx | 14 ++++++++++++-- packages/app/src/components/catalog/EntityPage.tsx | 2 +- .../TemplateWizardContent.tsx | 2 ++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/packages/app/src/components/SecurityTab/SecurityTab.tsx b/packages/app/src/components/SecurityTab/SecurityTab.tsx index 69c2455ea5..87311ed34f 100644 --- a/packages/app/src/components/SecurityTab/SecurityTab.tsx +++ b/packages/app/src/components/SecurityTab/SecurityTab.tsx @@ -21,10 +21,14 @@ import { } from '@backstage/plugin-scaffolder'; interface SecurityTabProps { + namespace: string; + templateName: string; customExtensionsElement?: React.ReactNode; } export function SecurityTab({ + namespace, + templateName, customExtensionsElement = <>, }: SecurityTabProps): JSX.Element | null { // eslint-disable-next-line no-alert @@ -38,11 +42,17 @@ export function SecurityTab({ return ( ); } diff --git a/packages/app/src/components/catalog/EntityPage.tsx b/packages/app/src/components/catalog/EntityPage.tsx index 6288dc86ee..25f9b4fdc8 100644 --- a/packages/app/src/components/catalog/EntityPage.tsx +++ b/packages/app/src/components/catalog/EntityPage.tsx @@ -404,7 +404,7 @@ const overviewContent = ( const securityContent = ( - + ); diff --git a/plugins/scaffolder/src/next/TemplateWizardContent/TemplateWizardContent.tsx b/plugins/scaffolder/src/next/TemplateWizardContent/TemplateWizardContent.tsx index ced1be5fdc..72d3b6c921 100644 --- a/plugins/scaffolder/src/next/TemplateWizardContent/TemplateWizardContent.tsx +++ b/plugins/scaffolder/src/next/TemplateWizardContent/TemplateWizardContent.tsx @@ -55,6 +55,7 @@ export interface TemplateWizardContentProps { transformErrors?: ErrorTransformer; onComplete: (values: Record) => Promise; onError(error: Error | undefined): JSX.Element | null; + initialFormState?: Record; } export const TemplateWizardContent = ( @@ -107,6 +108,7 @@ export const TemplateWizardContent = ( extensions={props.customFieldExtensions} onComplete={props.onComplete} transformErrors={props.transformErrors} + initialFormState={props.initialFormState} /> )} From 5fb19bf12aad8bec8bb482448eac3943c61bff09 Mon Sep 17 00:00:00 2001 From: Paul Cowan Date: Thu, 8 Dec 2022 17:18:55 +0000 Subject: [PATCH 055/182] EmbeddedScaffolderWorkflow component Signed-off-by: Paul Cowan --- .../EmbeddedScaffolderWorkflow.tsx | 80 +++++++++++++++++++ .../components/SecurityTab/SecurityTab.tsx | 45 +++++------ .../app/src/components/catalog/EntityPage.tsx | 2 +- .../TemplateWizardContent.tsx | 63 +++++++-------- .../TemplateWizardPage/TemplateWizardPage.tsx | 2 + 5 files changed, 129 insertions(+), 63 deletions(-) create mode 100644 packages/app/src/components/EmbeddedScaffolderWorkflow/EmbeddedScaffolderWorkflow.tsx diff --git a/packages/app/src/components/EmbeddedScaffolderWorkflow/EmbeddedScaffolderWorkflow.tsx b/packages/app/src/components/EmbeddedScaffolderWorkflow/EmbeddedScaffolderWorkflow.tsx new file mode 100644 index 0000000000..0c06836bbe --- /dev/null +++ b/packages/app/src/components/EmbeddedScaffolderWorkflow/EmbeddedScaffolderWorkflow.tsx @@ -0,0 +1,80 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React, { useState, useCallback } from 'react'; +import type { ReactNode } from 'react'; +import { + TemplateContent, + useGetCustomFields, +} from '@backstage/plugin-scaffolder'; +import { Button } from '@material-ui/core'; +import type { JsonValue } from '@backstage/types'; +import { FormProps } from '@backstage/plugin-scaffolder-react'; + +interface WorkflowProps { + frontPage: ReactNode; + namespace: string; + templateName: string; + customExtensionsElement?: React.ReactNode; + initialFormState?: Record; + onComplete: (values: Record) => Promise; + onError(error: Error | undefined): JSX.Element | null; + FormProps: FormProps +} + +export function EmbeddedScaffolderWorkflow({ + namespace, + templateName, + customExtensionsElement = <>, + frontPage, + onComplete, + onError, +}: WorkflowProps): JSX.Element { + const [showTemplateContent, setShowTemplateContent] = useState(false); + const fieldExtensions = useGetCustomFields(customExtensionsElement); + + const showContent = !showTemplateContent; + + const startTemplate = useCallback(() => setShowTemplateContent(true), []); + + return ( + <> + {showContent && ( + <> + {frontPage} + + + )} + {showTemplateContent && ( + + )} + + ); +} diff --git a/packages/app/src/components/SecurityTab/SecurityTab.tsx b/packages/app/src/components/SecurityTab/SecurityTab.tsx index 87311ed34f..6d53e542f6 100644 --- a/packages/app/src/components/SecurityTab/SecurityTab.tsx +++ b/packages/app/src/components/SecurityTab/SecurityTab.tsx @@ -15,22 +15,11 @@ */ import React from 'react'; -import { - TemplateContent, - useGetCustomFields, -} from '@backstage/plugin-scaffolder'; +import { EmbeddedScaffolderWorkflow } from '../EmbeddedScaffolderWorkflow/EmbeddedScaffolderWorkflow'; -interface SecurityTabProps { - namespace: string; - templateName: string; - customExtensionsElement?: React.ReactNode; -} +interface SecurityTabProps {} -export function SecurityTab({ - namespace, - templateName, - customExtensionsElement = <>, -}: SecurityTabProps): JSX.Element | null { +export function SecurityTab({}: SecurityTabProps): JSX.Element | null { // eslint-disable-next-line no-alert const onComplete = async () => alert('success!!!!'); @@ -38,21 +27,25 @@ export function SecurityTab({

{error?.message ?? 'Houston we have a problem.'}

); - const fieldExtensions = useGetCustomFields(customExtensionsElement); - return ( - +

Security Insights

+

+ Security insights actionable advice to improve security posture of + your application +

+

+ You must complete on-boarding process to activate security insights + on this project. +

+ + } /> ); } diff --git a/packages/app/src/components/catalog/EntityPage.tsx b/packages/app/src/components/catalog/EntityPage.tsx index 25f9b4fdc8..6288dc86ee 100644 --- a/packages/app/src/components/catalog/EntityPage.tsx +++ b/packages/app/src/components/catalog/EntityPage.tsx @@ -404,7 +404,7 @@ const overviewContent = ( const securityContent = ( - + ); diff --git a/plugins/scaffolder/src/next/TemplateWizardContent/TemplateWizardContent.tsx b/plugins/scaffolder/src/next/TemplateWizardContent/TemplateWizardContent.tsx index 72d3b6c921..744f0a6d6d 100644 --- a/plugins/scaffolder/src/next/TemplateWizardContent/TemplateWizardContent.tsx +++ b/plugins/scaffolder/src/next/TemplateWizardContent/TemplateWizardContent.tsx @@ -16,25 +16,23 @@ import React, { useEffect } from 'react'; import { Content, - Header, InfoCard, MarkdownContent, - Page, Progress, } from '@backstage/core-components'; import { stringifyEntityRef } from '@backstage/catalog-model'; import { useTemplateParameterSchema } from '../TemplateWizardPage/TemplateWizardPage'; -import { NextFieldExtensionOptions } from '../../extensions'; import type { ErrorTransformer } from '@rjsf/utils'; import type { JsonValue } from '@backstage/types'; import { makeStyles } from '@material-ui/core'; import { BackstageTheme } from '@backstage/theme'; import { errorApiRef, useApi } from '@backstage/core-plugin-api'; -import { Stepper } from '../TemplateWizardPage/Stepper'; import { - SecretsContext, SecretsContextProvider, -} from '../../components/secrets/SecretsContext'; + Stepper, + type NextFieldExtensionOptions, +} from '@backstage/plugin-scaffolder-react'; +import { type FormProps } from '@backstage/plugin-scaffolder-react'; const useStyles = makeStyles(() => ({ markdown: { @@ -56,6 +54,7 @@ export interface TemplateWizardContentProps { onComplete: (values: Record) => Promise; onError(error: Error | undefined): JSX.Element | null; initialFormState?: Record; + FormProps?: FormProps; } export const TemplateWizardContent = ( @@ -83,37 +82,29 @@ export const TemplateWizardContent = ( } return ( - -
- - {loading && } - {manifest && ( - - } - noPadding - titleTypographyProps={{ component: 'h2' }} - > - + {loading && } + {manifest && ( + - - )} - - + } + noPadding + titleTypographyProps={{ component: 'h2' }} + > + + + )} + ); }; diff --git a/plugins/scaffolder/src/next/TemplateWizardPage/TemplateWizardPage.tsx b/plugins/scaffolder/src/next/TemplateWizardPage/TemplateWizardPage.tsx index 4ff47e41e9..e0be68dbe0 100644 --- a/plugins/scaffolder/src/next/TemplateWizardPage/TemplateWizardPage.tsx +++ b/plugins/scaffolder/src/next/TemplateWizardPage/TemplateWizardPage.tsx @@ -33,6 +33,7 @@ import { FormProps } from '../types'; import { nextRouteRef } from '../routes'; import { scaffolderTaskRouteRef, selectedTemplateRouteRef } from '../../routes'; import { TemplateWizardContent } from '../TemplateWizardContent/TemplateWizardContent'; +import { Header, Page } from '@backstage/core-components'; type TemplateWizardPageProps = { customFieldExtensions: NextFieldExtensionOptions[]; @@ -85,6 +86,7 @@ export const TemplateWizardPage = (props: TemplateWizardPageProps) => { onComplete={onComplete} onError={onError} customFieldExtensions={props.customFieldExtensions} + FormProps={props.FormProps} /> ); From fa79ce2b594b2b9d2e6d7f9cbf0fb55807b9aa28 Mon Sep 17 00:00:00 2001 From: Paul Cowan Date: Fri, 9 Dec 2022 16:02:03 +0000 Subject: [PATCH 056/182] allow ReviewState component to be replaced as a prop Signed-off-by: Paul Cowan --- .../EmbeddedScaffolderWorkflow.tsx | 99 ++++++++++++------- .../components/SecurityTab/SecurityTab.tsx | 27 ++++- .../components/ReviewState/ReviewState.tsx | 2 +- .../src/next/components/Stepper/Stepper.tsx | 20 ++-- plugins/scaffolder/src/index.ts | 7 ++ plugins/scaffolder/src/next/Router/Router.tsx | 2 +- .../TemplateWizardContent.tsx | 24 +++-- .../src/next/TemplateWizardContent/index.ts | 1 + .../TemplateWizardPage/TemplateWizardPage.tsx | 3 +- 9 files changed, 126 insertions(+), 59 deletions(-) diff --git a/packages/app/src/components/EmbeddedScaffolderWorkflow/EmbeddedScaffolderWorkflow.tsx b/packages/app/src/components/EmbeddedScaffolderWorkflow/EmbeddedScaffolderWorkflow.tsx index 0c06836bbe..909c8bda12 100644 --- a/packages/app/src/components/EmbeddedScaffolderWorkflow/EmbeddedScaffolderWorkflow.tsx +++ b/packages/app/src/components/EmbeddedScaffolderWorkflow/EmbeddedScaffolderWorkflow.tsx @@ -20,61 +20,86 @@ import { TemplateContent, useGetCustomFields, } from '@backstage/plugin-scaffolder'; -import { Button } from '@material-ui/core'; +import type { FormProps } from '@backstage/plugin-scaffolder-react'; +import type { TemplateContentProps } from '@backstage/plugin-scaffolder'; +import { Box, Button } from '@material-ui/core'; import type { JsonValue } from '@backstage/types'; -import { FormProps } from '@backstage/plugin-scaffolder-react'; -interface WorkflowProps { - frontPage: ReactNode; - namespace: string; - templateName: string; +type WorkflowProps = Omit & { customExtensionsElement?: React.ReactNode; initialFormState?: Record; onComplete: (values: Record) => Promise; onError(error: Error | undefined): JSX.Element | null; FormProps: FormProps -} + frontPage: ReactNode; + finishPage: ReactNode; +}; + +type Display = 'front' | 'workflow' | 'finish'; + +type DisplayComponents = Record; + +type OnCompleteArgs = Parameters[0]; export function EmbeddedScaffolderWorkflow({ namespace, templateName, customExtensionsElement = <>, frontPage, - onComplete, + finishPage, + onComplete = async (_values: OnCompleteArgs) => void 0, onError, + title, + description, + ReviewStateWrapper, }: WorkflowProps): JSX.Element { - const [showTemplateContent, setShowTemplateContent] = useState(false); + const [display, setDisplay] = useState('front'); const fieldExtensions = useGetCustomFields(customExtensionsElement); - const showContent = !showTemplateContent; + const startTemplate = useCallback(() => setDisplay('workflow'), []); - const startTemplate = useCallback(() => setShowTemplateContent(true), []); + const onWorkFlowComplete = useCallback( + async (values: OnCompleteArgs) => { + setDisplay('finish'); - return ( - <> - {showContent && ( - <> - {frontPage} - - - )} - {showTemplateContent && ( - - )} - + await onComplete(values); + }, + [onComplete], ); + + const DisplayElements: DisplayComponents = { + front: ( + + {frontPage} + + + ), + workflow: ( + + ), + finish: ( + + {finishPage} + + ), + }; + + return <>{DisplayElements[display]}; } diff --git a/packages/app/src/components/SecurityTab/SecurityTab.tsx b/packages/app/src/components/SecurityTab/SecurityTab.tsx index 6d53e542f6..1841a4357e 100644 --- a/packages/app/src/components/SecurityTab/SecurityTab.tsx +++ b/packages/app/src/components/SecurityTab/SecurityTab.tsx @@ -16,12 +16,19 @@ import React from 'react'; import { EmbeddedScaffolderWorkflow } from '../EmbeddedScaffolderWorkflow/EmbeddedScaffolderWorkflow'; +import { Box } from '@material-ui/core'; -interface SecurityTabProps {} +const ReviewWrapper = () => { + return ( + +

This is a different wrapper for the review page

+ + ); +}; -export function SecurityTab({}: SecurityTabProps): JSX.Element | null { - // eslint-disable-next-line no-alert - const onComplete = async () => alert('success!!!!'); +export function SecurityTab(): JSX.Element | null { + // eslint-disable-next-line no-console + const onComplete = async () => console.log('onComplete called from '); const onError = (error: Error | undefined) => (

{error?.message ?? 'Houston we have a problem.'}

@@ -29,6 +36,11 @@ export function SecurityTab({}: SecurityTabProps): JSX.Element | null { return ( } + finishPage={ + <> +

Security Insights

+

Congratulations, this application is complete!

+ + } + ReviewStateWrapper={ReviewWrapper} /> ); } diff --git a/plugins/scaffolder-react/src/next/components/ReviewState/ReviewState.tsx b/plugins/scaffolder-react/src/next/components/ReviewState/ReviewState.tsx index 52ad80c4c2..11fb28b29c 100644 --- a/plugins/scaffolder-react/src/next/components/ReviewState/ReviewState.tsx +++ b/plugins/scaffolder-react/src/next/components/ReviewState/ReviewState.tsx @@ -23,7 +23,7 @@ import { ParsedTemplateSchema } from '../../hooks/useTemplateSchema'; * The props for the {@link ReviewState} component. * @alpha */ -export type ReviewStateProps = { +export interface ReviewStateProps { schemas: ParsedTemplateSchema[]; formState: JsonObject; }; diff --git a/plugins/scaffolder-react/src/next/components/Stepper/Stepper.tsx b/plugins/scaffolder-react/src/next/components/Stepper/Stepper.tsx index cde299da7e..638a1fb41c 100644 --- a/plugins/scaffolder-react/src/next/components/Stepper/Stepper.tsx +++ b/plugins/scaffolder-react/src/next/components/Stepper/Stepper.tsx @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { useAnalytics, useApiHolder } from '@backstage/core-plugin-api'; +import { useAnalytics, useApiHolder, useRouteRefParams } from '@backstage/core-plugin-api'; import { JsonValue } from '@backstage/types'; import { Stepper as MuiStepper, @@ -28,12 +28,12 @@ import React, { useCallback, useMemo, useState } from 'react'; import { NextFieldExtensionOptions } from '../../extensions'; import { TemplateParameterSchema } from '../../../types'; import { createAsyncValidators } from './createAsyncValidators'; +import type { FormProps } from '../../types'; +import { ReviewState, type ReviewStateProps } from '../ReviewState'; import { useTemplateSchema } from '../../hooks/useTemplateSchema'; -import { ReviewState } from '../ReviewState'; +import { useFormDataFromQuery } from '../../hooks/useFormDataFromQuery'; import validator from '@rjsf/validator-ajv6'; -import { useFormDataFromQuery } from '../../hooks'; -import { FormProps } from '../../types'; const useStyles = makeStyles(theme => ({ backButton: { @@ -62,7 +62,9 @@ export type StepperProps = { initialState?: Record; onComplete: (values: Record) => Promise; -}; + initialFormState?: Record; + ReviewStateWrapper?: (props: ReviewStateProps) => JSX.Element; +} // TODO(blam): We require here, as the types in this package depend on @rjsf/core explicitly // which is what we're using here as the default types, it needs to depend on @rjsf/core-v5 because @@ -73,7 +75,11 @@ const Form = withTheme(require('@rjsf/material-ui-v5').Theme); * The `Stepper` component is the Wizard that is rendered when a user selects a template * @alpha */ -export const Stepper = (props: StepperProps) => { + +export const Stepper = ({ + ReviewStateWrapper = ReviewState, + ...props +}: StepperProps) => { const analytics = useAnalytics(); const { steps } = useTemplateSchema(props.manifest); const apiHolder = useApiHolder(); @@ -183,7 +189,7 @@ export const Stepper = (props: StepperProps) => { ) : ( <> - +
- - ), - workflow: ( - - ), - finish: ( - - {finishPage} - - ), - }; - - return <>{DisplayElements[display]}; -} diff --git a/plugins/scaffolder/src/next/TemplateWizardPage/TemplateWizardPage.tsx b/plugins/scaffolder/src/next/TemplateWizardPage/TemplateWizardPage.tsx index eb23284649..8fb6f0a31e 100644 --- a/plugins/scaffolder/src/next/TemplateWizardPage/TemplateWizardPage.tsx +++ b/plugins/scaffolder/src/next/TemplateWizardPage/TemplateWizardPage.tsx @@ -27,29 +27,18 @@ import { useTemplateSecrets, NextFieldExtensionOptions, } from '@backstage/plugin-scaffolder-react'; -import useAsync from 'react-use/lib/useAsync'; import { JsonValue } from '@backstage/types'; import { FormProps } from '@backstage/plugin-scaffolder-react'; import { nextRouteRef } from '../routes'; import { scaffolderTaskRouteRef, selectedTemplateRouteRef } from '../../routes'; import { Header, Page } from '@backstage/core-components'; -import { Workflow } from '../Workflow/Workflow'; +import { Workflow } from '@backstage/plugin-scaffolder-react'; type TemplateWizardPageProps = { customFieldExtensions: NextFieldExtensionOptions[]; FormProps?: FormProps; }; -export const useTemplateParameterSchema = (templateRef: string) => { - const scaffolderApi = useApi(scaffolderApiRef); - const { value, loading, error } = useAsync( - () => scaffolderApi.getTemplateParameterSchema(templateRef), - [scaffolderApi, templateRef], - ); - - return { manifest: value, loading, error }; -}; - export const TemplateWizardPage = (props: TemplateWizardPageProps) => { const rootRef = useRouteRef(nextRouteRef); const taskRoute = useRouteRef(scaffolderTaskRouteRef); diff --git a/plugins/scaffolder/src/next/index.ts b/plugins/scaffolder/src/next/index.ts index c03e7f9aa4..1ca9096459 100644 --- a/plugins/scaffolder/src/next/index.ts +++ b/plugins/scaffolder/src/next/index.ts @@ -18,7 +18,3 @@ export * from './TemplateListPage'; export * from './TemplateWizardPage'; export * from './types'; export * from './routes'; -export * from './Workflow'; -export * from './EmbeddedScaffolderWorkflow'; -export type { WorkflowProps } from './Workflow'; - From b965a83a1f0134250a1d3e585f83664992ea194b Mon Sep 17 00:00:00 2001 From: Paul Cowan Date: Thu, 12 Jan 2023 17:02:21 +0000 Subject: [PATCH 073/182] add tests to scaffoler-react Signed-off-by: Paul Cowan --- plugins/scaffolder-react/package.json | 2 +- .../components/ReviewState/ReviewState.tsx | 2 +- .../next/components/Stepper/Stepper.test.tsx | 38 ++++++ .../src/next/components/Stepper/Stepper.tsx | 9 +- .../components/Workflow/Workflow.test.tsx | 116 ++++++++++++++++++ .../src/next/components/Workflow/Workflow.tsx | 2 +- yarn.lock | 1 + 7 files changed, 164 insertions(+), 6 deletions(-) create mode 100644 plugins/scaffolder-react/src/next/components/Workflow/Workflow.test.tsx diff --git a/plugins/scaffolder-react/package.json b/plugins/scaffolder-react/package.json index d08edb75fb..e44a1d94ee 100644 --- a/plugins/scaffolder-react/package.json +++ b/plugins/scaffolder-react/package.json @@ -57,8 +57,8 @@ "json-schema": "^0.4.0", "json-schema-library": "^7.3.9", "lodash": "^4.17.21", - "react-use": "^17.2.4", "qs": "^6.9.4", + "react-use": "^17.2.4", "zen-observable": "^0.10.0", "zod": "~3.18.0", "zod-to-json-schema": "~3.18.0" diff --git a/plugins/scaffolder-react/src/next/components/ReviewState/ReviewState.tsx b/plugins/scaffolder-react/src/next/components/ReviewState/ReviewState.tsx index 11fb28b29c..56abc52469 100644 --- a/plugins/scaffolder-react/src/next/components/ReviewState/ReviewState.tsx +++ b/plugins/scaffolder-react/src/next/components/ReviewState/ReviewState.tsx @@ -26,7 +26,7 @@ import { ParsedTemplateSchema } from '../../hooks/useTemplateSchema'; export interface ReviewStateProps { schemas: ParsedTemplateSchema[]; formState: JsonObject; -}; +} /** * The component used by the {@link Stepper} to render the review step. diff --git a/plugins/scaffolder-react/src/next/components/Stepper/Stepper.test.tsx b/plugins/scaffolder-react/src/next/components/Stepper/Stepper.test.tsx index 786f45b05d..e460afbdfc 100644 --- a/plugins/scaffolder-react/src/next/components/Stepper/Stepper.test.tsx +++ b/plugins/scaffolder-react/src/next/components/Stepper/Stepper.test.tsx @@ -352,4 +352,42 @@ describe('Stepper', () => { // flush promises return new Promise(process.nextTick); }); + + it('should override the Create and Review button text', async () => { + const manifest: TemplateParameterSchema = { + title: 'Custom Fields', + steps: [ + { + title: 'Test', + schema: { + properties: { + name: { + type: 'string', + }, + }, + }, + }, + ], + }; + + const { getByRole } = await renderInTestApp( + , + ); + + await act(async () => { + await fireEvent.click(getByRole('button', { name: 'Inspect' })); + }); + + expect(getByRole('button', { name: 'Make' })).toBeInTheDocument(); + + await act(async () => { + await fireEvent.click(getByRole('button', { name: 'Make' })); + }); + }); }); diff --git a/plugins/scaffolder-react/src/next/components/Stepper/Stepper.tsx b/plugins/scaffolder-react/src/next/components/Stepper/Stepper.tsx index 6bd3ec8c71..5d0ad29513 100644 --- a/plugins/scaffolder-react/src/next/components/Stepper/Stepper.tsx +++ b/plugins/scaffolder-react/src/next/components/Stepper/Stepper.tsx @@ -61,8 +61,9 @@ export type StepperProps = { initialState?: Record; onComplete: (values: Record) => Promise; - initialFormState?: Record; ReviewStateWrapper?: (props: ReviewStateProps) => JSX.Element; + createButtonText?: string; + reviewButtonText?: string; }; // TODO(blam): We require here, as the types in this package depend on @rjsf/core explicitly @@ -77,6 +78,8 @@ const Form = withTheme(require('@rjsf/material-ui-v5').Theme); export const Stepper = ({ ReviewStateWrapper = ReviewState, + createButtonText = 'Create', + reviewButtonText = 'Review', ...props }: StepperProps) => { const analytics = useAnalytics(); @@ -182,7 +185,7 @@ export const Stepper = ({ Back
@@ -211,7 +214,7 @@ export const Stepper = ({ ); }} > - Create + {createButtonText} diff --git a/plugins/scaffolder-react/src/next/components/Workflow/Workflow.test.tsx b/plugins/scaffolder-react/src/next/components/Workflow/Workflow.test.tsx new file mode 100644 index 0000000000..8be6d58407 --- /dev/null +++ b/plugins/scaffolder-react/src/next/components/Workflow/Workflow.test.tsx @@ -0,0 +1,116 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ApiProvider } from '@backstage/core-app-api'; +import { + MockAnalyticsApi, + renderInTestApp, + TestApiRegistry, +} from '@backstage/test-utils'; +import { act, fireEvent } from '@testing-library/react'; +import React from 'react'; +import { Workflow } from './Workflow'; +import { analyticsApiRef } from '@backstage/core-plugin-api'; +import { ScaffolderApi } from '../../../api/types'; +import { scaffolderApiRef } from '../../../api/ref'; + +const scaffolderApiMock: jest.Mocked = { + scaffold: jest.fn(), + getTemplateParameterSchema: jest.fn(), + getIntegrationsList: jest.fn(), + getTask: jest.fn(), + streamLogs: jest.fn(), + listActions: jest.fn(), + listTasks: jest.fn(), +}; + +const analyticsMock = new MockAnalyticsApi(); +const apis = TestApiRegistry.from( + [scaffolderApiRef, scaffolderApiMock], + [analyticsApiRef, analyticsMock], +); + +describe('', () => { + it('should complete a workflow', async () => { + const onComplete = jest.fn(); + const onError = jest.fn(); + scaffolderApiMock.scaffold.mockResolvedValue({ taskId: 'xyz' }); + + scaffolderApiMock.getTemplateParameterSchema.mockResolvedValue({ + steps: [ + { + title: 'Step 1', + schema: { + properties: { + name: { + type: 'string', + }, + }, + }, + }, + ], + title: 'React JSON Schema Form Test', + }); + + const { getByRole, getAllByRole, getByText } = await renderInTestApp( + + ( +

This is a different wrapper for the review page

+ )} + customFieldExtensions={[]} + /> + , + ); + + // Test template title is overriden + expect(getByRole('heading', { level: 2 }).innerHTML).toBe( + 'Different title than template', + ); + + const input = getByRole('textbox') as HTMLInputElement; + + expect(input).toBeInTheDocument(); + + expect(input.value).toBe('prefilled-name'); + + await act(async () => { + fireEvent.click(getByRole('button', { name: 'Review' })); + }); + + expect( + getByText('This is a different wrapper for the review page'), + ).toBeDefined(); + + await act(async () => { + fireEvent.click(getAllByRole('button')[1] as HTMLButtonElement); + }); + + expect(onComplete).toHaveBeenCalledWith({ name: 'prefilled-name' }); + }); +}); diff --git a/plugins/scaffolder-react/src/next/components/Workflow/Workflow.tsx b/plugins/scaffolder-react/src/next/components/Workflow/Workflow.tsx index 783c504cc8..095ff49fe8 100644 --- a/plugins/scaffolder-react/src/next/components/Workflow/Workflow.tsx +++ b/plugins/scaffolder-react/src/next/components/Workflow/Workflow.tsx @@ -113,7 +113,7 @@ export const Workflow = ({ extensions={props.customFieldExtensions} onComplete={props.onComplete} FormProps={FormProps} - initialFormState={props.initialFormState} + initialState={props.initialFormState} ReviewStateWrapper={ReviewStateWrapper} /> diff --git a/yarn.lock b/yarn.lock index f8df88cb37..6a6a81f40b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7459,6 +7459,7 @@ __metadata: json-schema-library: ^7.3.9 lodash: ^4.17.21 qs: ^6.9.4 + react-use: ^17.2.4 zen-observable: ^0.10.0 zod: ~3.18.0 zod-to-json-schema: ~3.18.0 From 693695ee468f1eefde944e3ed8f470f3653a6fdf Mon Sep 17 00:00:00 2001 From: Paul Cowan Date: Thu, 12 Jan 2023 17:54:29 +0000 Subject: [PATCH 074/182] run api-report Signed-off-by: Paul Cowan --- .changeset/pink-falcons-serve.md | 1 + packages/app/src/App.tsx | 13 +-- plugins/scaffolder-react/api-report.md | 62 +++++++++- plugins/scaffolder/api-report.md | 108 +----------------- plugins/scaffolder/src/index.ts | 2 + plugins/scaffolder/src/next/Router/Router.tsx | 2 - 6 files changed, 69 insertions(+), 119 deletions(-) diff --git a/.changeset/pink-falcons-serve.md b/.changeset/pink-falcons-serve.md index 73ceff0ddc..8cc6859c25 100644 --- a/.changeset/pink-falcons-serve.md +++ b/.changeset/pink-falcons-serve.md @@ -1,5 +1,6 @@ --- '@backstage/plugin-scaffolder': minor +'@backstage/plugin-scaffolder-react': minor --- Embed scaffolder workflow in other components diff --git a/packages/app/src/App.tsx b/packages/app/src/App.tsx index ccf3d98149..f223924294 100644 --- a/packages/app/src/App.tsx +++ b/packages/app/src/App.tsx @@ -27,7 +27,7 @@ import { RELATION_PROVIDES_API, } from '@backstage/catalog-model'; import { createApp } from '@backstage/app-defaults'; -import { FlatRoutes } from '@backstage/core-app-api'; +import { AppRouter, FlatRoutes } from '@backstage/core-app-api'; import { AlertDisplay, OAuthRequestDialog, @@ -149,9 +149,6 @@ const app = createApp({ }, }); -const AppProvider = app.getProvider(); -const AppRouter = app.getRouter(); - const routes = ( } /> @@ -282,14 +279,12 @@ const routes = ( ); -const App = () => ( - +export default app.createRoot( + <> {routes} - + , ); - -export default App; diff --git a/plugins/scaffolder-react/api-report.md b/plugins/scaffolder-react/api-report.md index adc097775b..97f05dfb16 100644 --- a/plugins/scaffolder-react/api-report.md +++ b/plugins/scaffolder-react/api-report.md @@ -8,6 +8,7 @@ import { ApiHolder } from '@backstage/core-plugin-api'; import { ApiRef } from '@backstage/core-plugin-api'; import { Dispatch } from 'react'; +import type { ErrorTransformer } from '@rjsf/utils'; import { Extension } from '@backstage/core-plugin-api'; import { FieldProps } from '@rjsf/core'; import { FieldProps as FieldProps_2 } from '@rjsf/utils'; @@ -79,6 +80,9 @@ export type CustomFieldValidator = ( }, ) => void | Promise; +// @alpha (undocumented) +export const EmbeddableWorkflow: (props: WorkflowProps) => JSX.Element; + // @alpha export const extractSchemaFromStep: (inputStep: JsonObject) => { uiSchema: UiSchema; @@ -186,10 +190,12 @@ export interface ParsedTemplateSchema { export const ReviewState: (props: ReviewStateProps) => JSX.Element; // @alpha -export type ReviewStateProps = { - schemas: ParsedTemplateSchema[]; +export interface ReviewStateProps { + // (undocumented) formState: JsonObject; -}; + // (undocumented) + schemas: ParsedTemplateSchema[]; +} // @public export interface ScaffolderApi { @@ -342,7 +348,12 @@ export const SecretsContextProvider: ({ }: PropsWithChildren<{}>) => JSX.Element; // @alpha -export const Stepper: (props: StepperProps) => JSX.Element; +export const Stepper: ({ + ReviewStateWrapper, + createButtonText, + reviewButtonText, + ...props +}: StepperProps) => JSX.Element; // @alpha export type StepperProps = { @@ -352,6 +363,9 @@ export type StepperProps = { FormProps?: FormProps; initialState?: Record; onComplete: (values: Record) => Promise; + ReviewStateWrapper?: (props: ReviewStateProps) => JSX.Element; + createButtonText?: string; + reviewButtonText?: string; }; // @alpha @@ -418,6 +432,13 @@ export const useFormDataFromQuery: ( initialState?: Record, ) => [Record, Dispatch>>]; +// @alpha (undocumented) +export const useTemplateParameterSchema: (templateRef: string) => { + manifest: TemplateParameterSchema | undefined; + loading: boolean; + error: Error | undefined; +}; + // @alpha export const useTemplateSchema: (manifest: TemplateParameterSchema) => { steps: ParsedTemplateSchema[]; @@ -426,5 +447,38 @@ export const useTemplateSchema: (manifest: TemplateParameterSchema) => { // @public export const useTemplateSecrets: () => ScaffolderUseTemplateSecrets; +// @alpha (undocumented) +export const Workflow: ({ + ReviewStateWrapper, + FormProps, + ...props +}: WorkflowProps) => JSX.Element | null; + +// @alpha (undocumented) +export interface WorkflowProps { + // (undocumented) + customFieldExtensions: NextFieldExtensionOptions[]; + // (undocumented) + description?: string; + // (undocumented) + FormProps?: FormProps; + // (undocumented) + initialFormState?: Record; + // (undocumented) + namespace: string; + // (undocumented) + onComplete: (values: Record) => Promise; + // (undocumented) + onError(error: Error | undefined): JSX.Element | null; + // (undocumented) + ReviewStateWrapper?: (props: ReviewStateProps) => JSX.Element; + // (undocumented) + templateName: string; + // (undocumented) + title?: string; + // (undocumented) + transformErrors?: ErrorTransformer; +} + // (No @packageDocumentation comment for this package) ``` diff --git a/plugins/scaffolder/api-report.md b/plugins/scaffolder/api-report.md index 2fd5fb464e..45d8a77f1e 100644 --- a/plugins/scaffolder/api-report.md +++ b/plugins/scaffolder/api-report.md @@ -22,7 +22,8 @@ import { FieldExtensionComponentProps as FieldExtensionComponentProps_2 } from ' import { FieldExtensionOptions as FieldExtensionOptions_2 } from '@backstage/plugin-scaffolder-react'; import { FieldValidation } from '@rjsf/core'; import type { FormProps as FormProps_2 } from '@rjsf/core'; -import type { FormProps as FormProps_3 } from '@rjsf/core-v5'; +import type { FormProps as FormProps_3 } from '@backstage/plugin-scaffolder-react'; +import type { FormProps as FormProps_4 } from '@rjsf/core-v5'; import { IdentityApi } from '@backstage/core-plugin-api'; import { JsonObject } from '@backstage/types'; import { ListActionsResponse as ListActionsResponse_2 } from '@backstage/plugin-scaffolder-react'; @@ -31,11 +32,7 @@ import { Observable } from '@backstage/types'; import { PathParams } from '@backstage/core-plugin-api'; import { PropsWithChildren } from 'react'; import { default as React_2 } from 'react'; -<<<<<<< HEAD import { ReactNode } from 'react'; -======= -import type { ReactNode } from 'react'; ->>>>>>> 1a12ad8e8d (fix api-reports) import { RouteRef } from '@backstage/core-plugin-api'; import { ScaffolderApi as ScaffolderApi_2 } from '@backstage/plugin-scaffolder-react'; import { ScaffolderDryRunOptions as ScaffolderDryRunOptions_2 } from '@backstage/plugin-scaffolder-react'; @@ -73,45 +70,7 @@ export type CustomFieldValidator = CustomFieldValidator_2; // @public -<<<<<<< HEAD export const EntityNamePickerFieldExtension: FieldExtensionComponent_2< -======= -export type CustomFieldValidator = ( - data: TFieldReturnValue, - field: FieldValidation, - context: { - apiHolder: ApiHolder; - }, -) => void | Promise; - -// @alpha -export function EmbeddedScaffolderWorkflow({ - namespace, - templateName, - customExtensionsElement, - frontPage, - finishPage, - onComplete, - onError, - title, - description, - ReviewStateWrapper, - initialFormState, -}: EmbeddedScaffolderWorkflowProps): JSX.Element; - -// @alpha (undocumented) -export type EmbeddedScaffolderWorkflowProps = Omit< - WorkflowProps, - 'customFieldExtensions' | 'onComplete' -> & { - customExtensionsElement?: React_2.ReactNode; - frontPage: ReactNode; - finishPage: ReactNode; -} & Partial>; - -// @public -export const EntityNamePickerFieldExtension: FieldExtensionComponent< ->>>>>>> b2747a3f35 (api-reports) string, {} >; @@ -199,7 +158,7 @@ export interface FieldSchema { // @alpha @deprecated export type FormProps = Pick< - FormProps_3, + FormProps_4, 'transformErrors' | 'noHtml5Validate' >; @@ -249,7 +208,7 @@ export type NextRouterProps = { TaskPageComponent?: React_2.ComponentType<{}>; }; groups?: TemplateGroupFilter[]; - FormProps?: FormProps; + FormProps?: FormProps_3; }; // @alpha @@ -324,20 +283,6 @@ export const OwnerPickerFieldSchema: FieldSchema< // @public export type OwnerPickerUiOptions = typeof OwnerPickerFieldSchema.uiOptionsType; -// @alpha (undocumented) -export interface ParsedTemplateSchema { - // (undocumented) - description?: string; - // (undocumented) - mergedSchema: JsonObject; - // (undocumented) - schema: JsonObject; - // (undocumented) - title: string; - // (undocumented) - uiSchema: UiSchema; -} - // @public export const repoPickerValidation: ( value: string, @@ -403,14 +348,6 @@ export const RepoUrlPickerFieldSchema: FieldSchema< export type RepoUrlPickerUiOptions = typeof RepoUrlPickerFieldSchema.uiOptionsType; -// @alpha (undocumented) -export interface ReviewStateProps { - // (undocumented) - formState: JsonObject; - // (undocumented) - schemas: ParsedTemplateSchema[]; -} - // @public export type ReviewStepProps = { disableButtons: boolean; @@ -590,43 +527,6 @@ export type TemplateParameterSchema = TemplateParameterSchema_2; // @public export const TemplateTypePicker: () => JSX.Element | null; -<<<<<<< HEAD -<<<<<<< HEAD // @public @deprecated (undocumented) export const useTemplateSecrets: () => ScaffolderUseTemplateSecrets_2; -======= -// @alpha -export const useGetCustomFields: ( - element: React_2.ReactNode, -) => NextFieldExtensionOptions[]; - -======= ->>>>>>> 1a12ad8e8d (fix api-reports) -// @public -export const useTemplateSecrets: () => ScaffolderUseTemplateSecrets; - -// @alpha (undocumented) -export interface WorkflowProps { - // (undocumented) - customFieldExtensions: NextFieldExtensionOptions[]; - // (undocumented) - description?: string; - // (undocumented) - initialFormState?: Record; - // (undocumented) - namespace: string; - // (undocumented) - onComplete: (values: Record) => Promise; - // (undocumented) - onError(error: Error | undefined): JSX.Element | null; - // (undocumented) - ReviewStateWrapper?: (props: ReviewStateProps) => JSX.Element; - // (undocumented) - templateName: string; - // (undocumented) - title?: string; - // (undocumented) - transformErrors?: ErrorTransformer; -} ->>>>>>> b2747a3f35 (api-reports) ``` diff --git a/plugins/scaffolder/src/index.ts b/plugins/scaffolder/src/index.ts index 60b2dfeeb2..0e2b293982 100644 --- a/plugins/scaffolder/src/index.ts +++ b/plugins/scaffolder/src/index.ts @@ -49,5 +49,7 @@ export { nextRouteRef, nextScaffolderTaskRouteRef, nextSelectedTemplateRouteRef, + type TemplateGroupFilter, + type NextRouterProps, type FormProps, } from './next'; diff --git a/plugins/scaffolder/src/next/Router/Router.tsx b/plugins/scaffolder/src/next/Router/Router.tsx index 77e0423fd6..0db23d1cd4 100644 --- a/plugins/scaffolder/src/next/Router/Router.tsx +++ b/plugins/scaffolder/src/next/Router/Router.tsx @@ -52,9 +52,7 @@ export type NextRouterProps = { */ export const Router = (props: PropsWithChildren) => { const { components: { TemplateCardComponent } = {} } = props; - const outlet = useOutlet() || props.children; - const customFieldExtensions = useCustomFieldExtensions(outlet); const fieldExtensions = [ From e78ba09f8b24c04991bee4f49ca5f8d677130f44 Mon Sep 17 00:00:00 2001 From: Paul Cowan Date: Thu, 12 Jan 2023 18:39:00 +0000 Subject: [PATCH 075/182] FormProps export Signed-off-by: Paul Cowan --- plugins/scaffolder-react/src/next/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/scaffolder-react/src/next/index.ts b/plugins/scaffolder-react/src/next/index.ts index a18649f506..ff3d502677 100644 --- a/plugins/scaffolder-react/src/next/index.ts +++ b/plugins/scaffolder-react/src/next/index.ts @@ -15,6 +15,6 @@ */ export * from './components'; export * from './extensions'; -export * from './types'; +export { type FormProps } from './types'; export * from './lib'; export * from './hooks'; From aeff4e6b5a428c08a3b8abd62e3e8722cdc16179 Mon Sep 17 00:00:00 2001 From: Paul Cowan Date: Thu, 12 Jan 2023 18:41:01 +0000 Subject: [PATCH 076/182] FormProps export Signed-off-by: Paul Cowan --- plugins/scaffolder-react/src/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/scaffolder-react/src/index.ts b/plugins/scaffolder-react/src/index.ts index 0b053e6bf4..41a1298b37 100644 --- a/plugins/scaffolder-react/src/index.ts +++ b/plugins/scaffolder-react/src/index.ts @@ -21,3 +21,4 @@ export * from './api'; export * from './hooks'; export * from './next'; +export type { FormProps } from './next'; From 2f0c9d9ca0743690f27bea04a47cd55c142de911 Mon Sep 17 00:00:00 2001 From: Paul Cowan Date: Thu, 12 Jan 2023 19:10:10 +0000 Subject: [PATCH 077/182] make FormProps public Signed-off-by: Paul Cowan --- plugins/scaffolder-react/src/next/index.ts | 2 +- plugins/scaffolder-react/src/next/types.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/scaffolder-react/src/next/index.ts b/plugins/scaffolder-react/src/next/index.ts index ff3d502677..ea009ea556 100644 --- a/plugins/scaffolder-react/src/next/index.ts +++ b/plugins/scaffolder-react/src/next/index.ts @@ -15,6 +15,6 @@ */ export * from './components'; export * from './extensions'; -export { type FormProps } from './types'; +export type { FormProps } from './types'; export * from './lib'; export * from './hooks'; diff --git a/plugins/scaffolder-react/src/next/types.ts b/plugins/scaffolder-react/src/next/types.ts index 32d0fe85f0..a6f699791b 100644 --- a/plugins/scaffolder-react/src/next/types.ts +++ b/plugins/scaffolder-react/src/next/types.ts @@ -18,7 +18,7 @@ import type { FormProps as SchemaFormProps } from '@rjsf/core-v5'; /** * Any `@rjsf/core` form properties that are publicly exposed to the `NextScaffolderpage` * - * @alpha + * @public */ export type FormProps = Pick< SchemaFormProps, From 9081955021b58c67016c0acb66fb35e55e9ed329 Mon Sep 17 00:00:00 2001 From: Paul Cowan Date: Thu, 12 Jan 2023 19:14:40 +0000 Subject: [PATCH 078/182] revert FormProps imports Signed-off-by: Paul Cowan --- plugins/scaffolder-react/src/next/types.ts | 2 +- plugins/scaffolder/src/next/Router/Router.tsx | 2 +- .../src/next/TemplateWizardPage/TemplateWizardPage.tsx | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/scaffolder-react/src/next/types.ts b/plugins/scaffolder-react/src/next/types.ts index a6f699791b..32d0fe85f0 100644 --- a/plugins/scaffolder-react/src/next/types.ts +++ b/plugins/scaffolder-react/src/next/types.ts @@ -18,7 +18,7 @@ import type { FormProps as SchemaFormProps } from '@rjsf/core-v5'; /** * Any `@rjsf/core` form properties that are publicly exposed to the `NextScaffolderpage` * - * @public + * @alpha */ export type FormProps = Pick< SchemaFormProps, diff --git a/plugins/scaffolder/src/next/Router/Router.tsx b/plugins/scaffolder/src/next/Router/Router.tsx index 0db23d1cd4..f91e56bba7 100644 --- a/plugins/scaffolder/src/next/Router/Router.tsx +++ b/plugins/scaffolder/src/next/Router/Router.tsx @@ -26,7 +26,7 @@ import { import { TemplateEntityV1beta3 } from '@backstage/plugin-scaffolder-common'; import { TemplateGroupFilter } from '../TemplateListPage/TemplateGroups'; import { DEFAULT_SCAFFOLDER_FIELD_EXTENSIONS } from '../../extensions/default'; -import type { FormProps } from '@backstage/plugin-scaffolder-react'; +import { type FormProps } from '../types'; import { nextSelectedTemplateRouteRef } from '../routes'; /** diff --git a/plugins/scaffolder/src/next/TemplateWizardPage/TemplateWizardPage.tsx b/plugins/scaffolder/src/next/TemplateWizardPage/TemplateWizardPage.tsx index 8fb6f0a31e..01a1468c90 100644 --- a/plugins/scaffolder/src/next/TemplateWizardPage/TemplateWizardPage.tsx +++ b/plugins/scaffolder/src/next/TemplateWizardPage/TemplateWizardPage.tsx @@ -28,7 +28,7 @@ import { NextFieldExtensionOptions, } from '@backstage/plugin-scaffolder-react'; import { JsonValue } from '@backstage/types'; -import { FormProps } from '@backstage/plugin-scaffolder-react'; +import { type FormProps } from '../types'; import { nextRouteRef } from '../routes'; import { scaffolderTaskRouteRef, selectedTemplateRouteRef } from '../../routes'; import { Header, Page } from '@backstage/core-components'; From 72e9ee90ae1614075cbab6de65da95ea693ba2ef Mon Sep 17 00:00:00 2001 From: Paul Cowan Date: Thu, 12 Jan 2023 19:50:41 +0000 Subject: [PATCH 079/182] run api-report Signed-off-by: Paul Cowan --- plugins/scaffolder/api-report.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/plugins/scaffolder/api-report.md b/plugins/scaffolder/api-report.md index 45d8a77f1e..82a8dfd8d3 100644 --- a/plugins/scaffolder/api-report.md +++ b/plugins/scaffolder/api-report.md @@ -22,8 +22,7 @@ import { FieldExtensionComponentProps as FieldExtensionComponentProps_2 } from ' import { FieldExtensionOptions as FieldExtensionOptions_2 } from '@backstage/plugin-scaffolder-react'; import { FieldValidation } from '@rjsf/core'; import type { FormProps as FormProps_2 } from '@rjsf/core'; -import type { FormProps as FormProps_3 } from '@backstage/plugin-scaffolder-react'; -import type { FormProps as FormProps_4 } from '@rjsf/core-v5'; +import type { FormProps as FormProps_3 } from '@rjsf/core-v5'; import { IdentityApi } from '@backstage/core-plugin-api'; import { JsonObject } from '@backstage/types'; import { ListActionsResponse as ListActionsResponse_2 } from '@backstage/plugin-scaffolder-react'; @@ -158,7 +157,7 @@ export interface FieldSchema { // @alpha @deprecated export type FormProps = Pick< - FormProps_4, + FormProps_3, 'transformErrors' | 'noHtml5Validate' >; @@ -208,7 +207,7 @@ export type NextRouterProps = { TaskPageComponent?: React_2.ComponentType<{}>; }; groups?: TemplateGroupFilter[]; - FormProps?: FormProps_3; + FormProps?: FormProps; }; // @alpha From 7e6fdc0146e291b7c78a44217edb8ce94d8ac9ff Mon Sep 17 00:00:00 2001 From: Paul Cowan Date: Sun, 15 Jan 2023 08:14:51 +0000 Subject: [PATCH 080/182] rename ReviewStateWrapper to ReviewStateComponent Signed-off-by: Paul Cowan --- plugins/scaffolder-react/api-report.md | 25 ++++++------------- .../components/ReviewState/ReviewState.tsx | 4 +-- .../src/next/components/Stepper/Stepper.tsx | 18 +++++++------ .../components/Workflow/Workflow.test.tsx | 2 +- .../src/next/components/Workflow/Workflow.tsx | 16 ++++++------ 5 files changed, 29 insertions(+), 36 deletions(-) diff --git a/plugins/scaffolder-react/api-report.md b/plugins/scaffolder-react/api-report.md index 97f05dfb16..3c03bf2d15 100644 --- a/plugins/scaffolder-react/api-report.md +++ b/plugins/scaffolder-react/api-report.md @@ -190,12 +190,10 @@ export interface ParsedTemplateSchema { export const ReviewState: (props: ReviewStateProps) => JSX.Element; // @alpha -export interface ReviewStateProps { - // (undocumented) - formState: JsonObject; - // (undocumented) +export type ReviewStateProps = { schemas: ParsedTemplateSchema[]; -} + formState: JsonObject; +}; // @public export interface ScaffolderApi { @@ -348,12 +346,7 @@ export const SecretsContextProvider: ({ }: PropsWithChildren<{}>) => JSX.Element; // @alpha -export const Stepper: ({ - ReviewStateWrapper, - createButtonText, - reviewButtonText, - ...props -}: StepperProps) => JSX.Element; +export const Stepper: (stepperProps: StepperProps) => JSX.Element; // @alpha export type StepperProps = { @@ -363,7 +356,7 @@ export type StepperProps = { FormProps?: FormProps; initialState?: Record; onComplete: (values: Record) => Promise; - ReviewStateWrapper?: (props: ReviewStateProps) => JSX.Element; + ReviewStateComponent?: (props: ReviewStateProps) => JSX.Element; createButtonText?: string; reviewButtonText?: string; }; @@ -448,11 +441,7 @@ export const useTemplateSchema: (manifest: TemplateParameterSchema) => { export const useTemplateSecrets: () => ScaffolderUseTemplateSecrets; // @alpha (undocumented) -export const Workflow: ({ - ReviewStateWrapper, - FormProps, - ...props -}: WorkflowProps) => JSX.Element | null; +export const Workflow: (workflowProps: WorkflowProps) => JSX.Element | null; // @alpha (undocumented) export interface WorkflowProps { @@ -471,7 +460,7 @@ export interface WorkflowProps { // (undocumented) onError(error: Error | undefined): JSX.Element | null; // (undocumented) - ReviewStateWrapper?: (props: ReviewStateProps) => JSX.Element; + ReviewStateComponent?: (props: ReviewStateProps) => JSX.Element; // (undocumented) templateName: string; // (undocumented) diff --git a/plugins/scaffolder-react/src/next/components/ReviewState/ReviewState.tsx b/plugins/scaffolder-react/src/next/components/ReviewState/ReviewState.tsx index 56abc52469..52ad80c4c2 100644 --- a/plugins/scaffolder-react/src/next/components/ReviewState/ReviewState.tsx +++ b/plugins/scaffolder-react/src/next/components/ReviewState/ReviewState.tsx @@ -23,10 +23,10 @@ import { ParsedTemplateSchema } from '../../hooks/useTemplateSchema'; * The props for the {@link ReviewState} component. * @alpha */ -export interface ReviewStateProps { +export type ReviewStateProps = { schemas: ParsedTemplateSchema[]; formState: JsonObject; -} +}; /** * The component used by the {@link Stepper} to render the review step. diff --git a/plugins/scaffolder-react/src/next/components/Stepper/Stepper.tsx b/plugins/scaffolder-react/src/next/components/Stepper/Stepper.tsx index 5d0ad29513..37e86d1afa 100644 --- a/plugins/scaffolder-react/src/next/components/Stepper/Stepper.tsx +++ b/plugins/scaffolder-react/src/next/components/Stepper/Stepper.tsx @@ -61,7 +61,7 @@ export type StepperProps = { initialState?: Record; onComplete: (values: Record) => Promise; - ReviewStateWrapper?: (props: ReviewStateProps) => JSX.Element; + ReviewStateComponent?: (props: ReviewStateProps) => JSX.Element; createButtonText?: string; reviewButtonText?: string; }; @@ -76,12 +76,14 @@ const Form = withTheme(require('@rjsf/material-ui-v5').Theme); * @alpha */ -export const Stepper = ({ - ReviewStateWrapper = ReviewState, - createButtonText = 'Create', - reviewButtonText = 'Review', - ...props -}: StepperProps) => { +export const Stepper = (stepperProps: StepperProps) => { + const { + ReviewStateComponent = ReviewState, + createButtonText = 'Create', + reviewButtonText = 'Review', + ...props + } = stepperProps; + const analytics = useAnalytics(); const { steps } = useTemplateSchema(props.manifest); const apiHolder = useApiHolder(); @@ -191,7 +193,7 @@ export const Stepper = ({ ) : ( <> - +