diff --git a/.changeset/new-cougars-beam.md b/.changeset/new-cougars-beam.md new file mode 100644 index 0000000000..c1c03ce848 --- /dev/null +++ b/.changeset/new-cougars-beam.md @@ -0,0 +1,30 @@ +--- +'@backstage/plugin-catalog-backend-module-msgraph': minor +--- + +**BREAKING**: `MicrosoftGraphOrgEntityProvider.fromConfig` now requires a `schedule` field in its options, which simplifies scheduling. If you want to retain the old behavior of calling its `run()` method manually, you can set the new field value to the string `'manual'`. But you may prefer to instead give it a scheduled task runner from the backend tasks package: + +```diff + // packages/backend/src/plugins/catalog.ts ++import { Duration } from 'luxon'; ++import { MicrosoftGraphOrgEntityProvider } from '@backstage/plugin-catalog-backend-module-msgraph'; + + export default async function createPlugin( + env: PluginEnvironment, + ): Promise { + const builder = await CatalogBuilder.create(env); + ++ // The target parameter below needs to match one of the providers' target ++ // value specified in your app-config. ++ builder.addEntityProvider( ++ MicrosoftGraphOrgEntityProvider.fromConfig(env.config, { ++ id: 'production', ++ target: 'https://graph.microsoft.com/v1.0', ++ logger: env.logger, ++ schedule: env.scheduler.createScheduledTaskRunner({ ++ frequency: Duration.fromObject({ minutes: 5 }), ++ timeout: Duration.fromObject({ minutes: 3 }), ++ }), ++ }), ++ ); +``` diff --git a/plugins/catalog-backend-module-msgraph/README.md b/plugins/catalog-backend-module-msgraph/README.md index d9488486ad..741e378fd4 100644 --- a/plugins/catalog-backend-module-msgraph/README.md +++ b/plugins/catalog-backend-module-msgraph/README.md @@ -92,26 +92,31 @@ yarn add @backstage/plugin-catalog-backend-module-msgraph 4. The `MicrosoftGraphOrgEntityProvider` is not registered by default, so you have to register it in the catalog plugin. Pass the target to reference a - provider from the configuration. As entity providers are not part of the - entity refresh loop, you have to run them manually. + provider from the configuration. -```typescript -// packages/backend/src/plugins/catalog.ts -const msGraphOrgEntityProvider = MicrosoftGraphOrgEntityProvider.fromConfig( - env.config, - { - id: 'https://graph.microsoft.com/v1.0', - target: 'https://graph.microsoft.com/v1.0', - logger: env.logger, - }, -); -builder.addEntityProvider(msGraphOrgEntityProvider); +```diff + // packages/backend/src/plugins/catalog.ts ++import { Duration } from 'luxon'; ++import { MicrosoftGraphOrgEntityProvider } from '@backstage/plugin-catalog-backend-module-msgraph'; -// Trigger a read every 5 minutes -useHotCleanup( - module, - runPeriodically(() => msGraphOrgEntityProvider.read(), 5 * 60 * 1000), -); + export default async function createPlugin( + env: PluginEnvironment, + ): Promise { + const builder = await CatalogBuilder.create(env); + ++ // The target parameter below needs to match one of the providers' target ++ // value specified in your app-config (see above). ++ builder.addEntityProvider( ++ MicrosoftGraphOrgEntityProvider.fromConfig(env.config, { ++ id: 'production', ++ target: 'https://graph.microsoft.com/v1.0', ++ logger: env.logger, ++ schedule: env.scheduler.createScheduledTaskRunner({ ++ frequency: Duration.fromObject({ minutes: 5 }), ++ timeout: Duration.fromObject({ minutes: 3 }), ++ }), ++ }), ++ ); ``` ### Using the Processor diff --git a/plugins/catalog-backend-module-msgraph/api-report.md b/plugins/catalog-backend-module-msgraph/api-report.md index e3840dc6f7..2e741e65fe 100644 --- a/plugins/catalog-backend-module-msgraph/api-report.md +++ b/plugins/catalog-backend-module-msgraph/api-report.md @@ -14,6 +14,7 @@ import { Logger } from 'winston'; import * as MicrosoftGraph from '@microsoft/microsoft-graph-types'; import * as msal from '@azure/msal-node'; import { Response as Response_2 } from 'node-fetch'; +import { TaskRunner } from '@backstage/backend-tasks'; import { UserEntity } from '@backstage/catalog-model'; // @public @@ -119,19 +120,23 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { connect(connection: EntityProviderConnection): Promise; // (undocumented) static fromConfig( - config: Config, - options: { - id: string; - target: string; - logger: Logger; - userTransformer?: UserTransformer; - groupTransformer?: GroupTransformer; - organizationTransformer?: OrganizationTransformer; - }, + configRoot: Config, + options: MicrosoftGraphOrgEntityProviderOptions, ): MicrosoftGraphOrgEntityProvider; // (undocumented) getProviderName(): string; - read(): Promise; + read(options?: { logger?: Logger }): Promise; +} + +// @public +export interface MicrosoftGraphOrgEntityProviderOptions { + groupTransformer?: GroupTransformer; + id: string; + logger: Logger; + organizationTransformer?: OrganizationTransformer; + schedule: 'manual' | TaskRunner; + target: string; + userTransformer?: UserTransformer; } // @public diff --git a/plugins/catalog-backend-module-msgraph/package.json b/plugins/catalog-backend-module-msgraph/package.json index 58ff3b2b3d..119fc5004c 100644 --- a/plugins/catalog-backend-module-msgraph/package.json +++ b/plugins/catalog-backend-module-msgraph/package.json @@ -34,6 +34,7 @@ }, "dependencies": { "@azure/msal-node": "^1.1.0", + "@backstage/backend-tasks": "^0.2.0", "@backstage/catalog-model": "^0.13.0", "@backstage/config": "^0.1.15", "@backstage/plugin-catalog-backend": "^0.24.0", @@ -42,6 +43,7 @@ "lodash": "^4.17.21", "node-fetch": "^2.6.7", "p-limit": "^3.0.2", + "uuid": "^8.0.0", "winston": "^3.2.1", "qs": "^6.9.4" }, diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts index 71aabaeef2..dfd0a7782e 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { TaskRunner } from '@backstage/backend-tasks'; import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, @@ -25,6 +26,7 @@ import { EntityProviderConnection, } from '@backstage/plugin-catalog-backend'; import { merge } from 'lodash'; +import * as uuid from 'uuid'; import { Logger } from 'winston'; import { GroupTransformer, @@ -39,6 +41,62 @@ import { UserTransformer, } from '../microsoftGraph'; +/** + * Options for {@link MicrosoftGraphOrgEntityProvider}. + * + * @public + */ +export interface MicrosoftGraphOrgEntityProviderOptions { + /** + * A unique, stable identifier for this provider. + * + * @example "production" + */ + id: string; + + /** + * The target that this provider should consume. + * + * Should exactly match the "target" field of one of the providers + * configuration entries. + */ + target: string; + + /** + * The logger to use. + */ + logger: Logger; + + /** + * The refresh schedule to use. + * + * @remarks + * + * If you pass in 'manual', you are responsible for calling the `read` method + * manually at some interval. + * + * But more commonly you will pass in the result of + * {@link @backstage/backend-tasks#PluginTaskScheduler.createScheduledTaskRunner} + * to enable automatic scheduling of tasks. + */ + schedule: 'manual' | TaskRunner; + + /** + * The function that transforms a user entry in msgraph to an entity. + */ + userTransformer?: UserTransformer; + + /** + * The function that transforms a group entry in msgraph to an entity. + */ + groupTransformer?: GroupTransformer; + + /** + * The function that transforms an organization entry in msgraph to an entity. + */ + organizationTransformer?: OrganizationTransformer; +} + /** * Reads user and group entries out of Microsoft Graph, and provides them as * User and Group entities for the catalog. @@ -47,25 +105,21 @@ import { */ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { private connection?: EntityProviderConnection; + private scheduleFn?: () => Promise; static fromConfig( - config: Config, - options: { - id: string; - target: string; - logger: Logger; - userTransformer?: UserTransformer; - groupTransformer?: GroupTransformer; - organizationTransformer?: OrganizationTransformer; - }, + configRoot: Config, + options: MicrosoftGraphOrgEntityProviderOptions, ) { - const c = config.getOptionalConfig('catalog.processors.microsoftGraphOrg'); - const providers = c ? readMicrosoftGraphConfig(c) : []; + const config = configRoot.getOptionalConfig( + 'catalog.processors.microsoftGraphOrg', + ); + const providers = config ? readMicrosoftGraphConfig(config) : []; const provider = providers.find(p => options.target.startsWith(p.target)); if (!provider) { throw new Error( - `There is no Microsoft Graph Org provider that matches ${options.target}. Please add a configuration entry for it under catalog.processors.microsoftGraphOrg.providers.`, + `There is no Microsoft Graph Org provider that matches "${options.target}". Please add a configuration entry for it under "catalog.processors.microsoftGraphOrg.providers".`, ); } @@ -73,7 +127,7 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { target: options.target, }); - return new MicrosoftGraphOrgEntityProvider({ + const result = new MicrosoftGraphOrgEntityProvider({ id: options.id, userTransformer: options.userTransformer, groupTransformer: options.groupTransformer, @@ -81,6 +135,10 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { logger, provider, }); + + result.schedule(options.schedule); + + return result; } constructor( @@ -102,19 +160,21 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */ async connect(connection: EntityProviderConnection) { this.connection = connection; + await this.scheduleFn?.(); } /** * Runs one complete ingestion loop. Call this method regularly at some * appropriate cadence. */ - async read() { + async read(options?: { logger?: Logger }) { if (!this.connection) { throw new Error('Not initialized'); } + const logger = options?.logger ?? this.options.logger; const provider = this.options.provider; - const { markReadComplete } = trackProgress(this.options.logger); + const { markReadComplete } = trackProgress(logger); const client = MicrosoftGraphClient.create(this.options.provider); const { users, groups } = await readMicrosoftGraphOrg( @@ -130,7 +190,7 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { groupTransformer: this.options.groupTransformer, userTransformer: this.options.userTransformer, organizationTransformer: this.options.organizationTransformer, - logger: this.options.logger, + logger: logger, }, ); @@ -146,6 +206,34 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider { markCommitComplete(); } + + private schedule( + schedule: MicrosoftGraphOrgEntityProviderOptions['schedule'], + ) { + if (schedule === 'manual') { + return; + } + + this.scheduleFn = async () => { + const id = `${this.getProviderName()}:refresh`; + await schedule.run({ + id, + fn: async () => { + const logger = this.options.logger.child({ + class: MicrosoftGraphOrgEntityProvider.prototype.constructor.name, + taskId: id, + taskInstanceId: uuid.v4(), + }); + + try { + await this.read({ logger }); + } catch (error) { + logger.error(error); + } + }, + }); + }; + } } // Helps wrap the timing and logging behaviors @@ -173,12 +261,12 @@ function trackProgress(logger: Logger) { // Makes sure that emitted entities have a proper location based on their uuid export function withLocations(providerId: string, entity: Entity): Entity { - const uuid = + const uid = entity.metadata.annotations?.[MICROSOFT_GRAPH_USER_ID_ANNOTATION] || entity.metadata.annotations?.[MICROSOFT_GRAPH_GROUP_ID_ANNOTATION] || entity.metadata.annotations?.[MICROSOFT_GRAPH_TENANT_ID_ANNOTATION] || entity.metadata.name; - const location = `msgraph:${providerId}/${encodeURIComponent(uuid)}`; + const location = `msgraph:${providerId}/${encodeURIComponent(uid)}`; return merge( { metadata: { diff --git a/plugins/catalog-backend-module-msgraph/src/processors/index.ts b/plugins/catalog-backend-module-msgraph/src/processors/index.ts index 7f70bde994..7bc3559c8b 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/index.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/index.ts @@ -15,4 +15,5 @@ */ export { MicrosoftGraphOrgEntityProvider } from './MicrosoftGraphOrgEntityProvider'; +export type { MicrosoftGraphOrgEntityProviderOptions } from './MicrosoftGraphOrgEntityProvider'; export { MicrosoftGraphOrgReaderProcessor } from './MicrosoftGraphOrgReaderProcessor';