schedule msgraph org ingestion too

Signed-off-by: Fredrik Adelöw <freben@gmail.com>

plugins/catalog-backend-module-msgraph/README.md

@@ -92,26 +92,31 @@ yarn add @backstage/plugin-catalog-backend-module-msgraph
 
 4. The `MicrosoftGraphOrgEntityProvider` is not registered by default, so you
    have to register it in the catalog plugin. Pass the target to reference a
-   provider from the configuration. As entity providers are not part of the
-   entity refresh loop, you have to run them manually.
+   provider from the configuration.
 
-```typescript
-// packages/backend/src/plugins/catalog.ts
-const msGraphOrgEntityProvider = MicrosoftGraphOrgEntityProvider.fromConfig(
-  env.config,
-  {
-    id: 'https://graph.microsoft.com/v1.0',
-    target: 'https://graph.microsoft.com/v1.0',
-    logger: env.logger,
-  },
-);
-builder.addEntityProvider(msGraphOrgEntityProvider);
+```diff
+ // packages/backend/src/plugins/catalog.ts
++import { Duration } from 'luxon';
++import { MicrosoftGraphOrgEntityProvider } from '@backstage/plugin-catalog-backend-module-msgraph';
 
-// Trigger a read every 5 minutes
-useHotCleanup(
-  module,
-  runPeriodically(() => msGraphOrgEntityProvider.read(), 5 * 60 * 1000),
-);
+ export default async function createPlugin(
+   env: PluginEnvironment,
+ ): Promise<Router> {
+   const builder = await CatalogBuilder.create(env);
+
++  // The target parameter below needs to match one of the providers' target
++  // value specified in your app-config (see above).
++  builder.addEntityProvider(
++    MicrosoftGraphOrgEntityProvider.fromConfig(env.config, {
++      id: 'production',
++      target: 'https://graph.microsoft.com/v1.0',
++      logger: env.logger,
++      schedule: env.scheduler.createScheduledTaskRunner({
++        frequency: Duration.fromObject({ minutes: 5 }),
++        timeout: Duration.fromObject({ minutes: 3 }),
++      }),
++    }),
++  );
 ```
 
 ### Using the Processor

plugins/catalog-backend-module-msgraph/api-report.md

@@ -14,6 +14,7 @@ import { Logger } from 'winston';
 import * as MicrosoftGraph from '@microsoft/microsoft-graph-types';
 import * as msal from '@azure/msal-node';
 import { Response as Response_2 } from 'node-fetch';
+import { TaskRunner } from '@backstage/backend-tasks';
 import { UserEntity } from '@backstage/catalog-model';
 
 // @public
@@ -119,19 +120,23 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider {
   connect(connection: EntityProviderConnection): Promise<void>;
   // (undocumented)
   static fromConfig(
-    config: Config,
-    options: {
-      id: string;
-      target: string;
-      logger: Logger;
-      userTransformer?: UserTransformer;
-      groupTransformer?: GroupTransformer;
-      organizationTransformer?: OrganizationTransformer;
-    },
+    configRoot: Config,
+    options: MicrosoftGraphOrgEntityProviderOptions,
   ): MicrosoftGraphOrgEntityProvider;
   // (undocumented)
   getProviderName(): string;
-  read(): Promise<void>;
+  read(options?: { logger?: Logger }): Promise<void>;
+}
+
+// @public
+export interface MicrosoftGraphOrgEntityProviderOptions {
+  groupTransformer?: GroupTransformer;
+  id: string;
+  logger: Logger;
+  organizationTransformer?: OrganizationTransformer;
+  schedule: 'manual' | TaskRunner;
+  target: string;
+  userTransformer?: UserTransformer;
 }
 
 // @public

plugins/catalog-backend-module-msgraph/package.json

@@ -34,6 +34,7 @@
   },
   "dependencies": {
     "@azure/msal-node": "^1.1.0",
+    "@backstage/backend-tasks": "^0.2.0",
     "@backstage/catalog-model": "^0.13.0",
     "@backstage/config": "^0.1.15",
     "@backstage/plugin-catalog-backend": "^0.24.0",
@@ -42,6 +43,7 @@
     "lodash": "^4.17.21",
     "node-fetch": "^2.6.7",
     "p-limit": "^3.0.2",
+    "uuid": "^8.0.0",
     "winston": "^3.2.1",
     "qs": "^6.9.4"
   },

plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+import { TaskRunner } from '@backstage/backend-tasks';
 import {
   ANNOTATION_LOCATION,
   ANNOTATION_ORIGIN_LOCATION,
@@ -25,6 +26,7 @@ import {
   EntityProviderConnection,
 } from '@backstage/plugin-catalog-backend';
 import { merge } from 'lodash';
+import * as uuid from 'uuid';
 import { Logger } from 'winston';
 import {
   GroupTransformer,
@@ -39,6 +41,62 @@ import {
   UserTransformer,
 } from '../microsoftGraph';
 
+/**
+ * Options for {@link MicrosoftGraphOrgEntityProvider}.
+ *
+ * @public
+ */
+export interface MicrosoftGraphOrgEntityProviderOptions {
+  /**
+   * A unique, stable identifier for this provider.
+   *
+   * @example "production"
+   */
+  id: string;
+
+  /**
+   * The target that this provider should consume.
+   *
+   * Should exactly match the "target" field of one of the providers
+   * configuration entries.
+   */
+  target: string;
+
+  /**
+   * The logger to use.
+   */
+  logger: Logger;
+
+  /**
+   * The refresh schedule to use.
+   *
+   * @remarks
+   *
+   * If you pass in 'manual', you are responsible for calling the `read` method
+   * manually at some interval.
+   *
+   * But more commonly you will pass in the result of
+   * {@link @backstage/backend-tasks#PluginTaskScheduler.createScheduledTaskRunner}
+   * to enable automatic scheduling of tasks.
+   */
+  schedule: 'manual' | TaskRunner;
+
+  /**
+   * The function that transforms a user entry in msgraph to an entity.
+   */
+  userTransformer?: UserTransformer;
+
+  /**
+   * The function that transforms a group entry in msgraph to an entity.
+   */
+  groupTransformer?: GroupTransformer;
+
+  /**
+   * The function that transforms an organization entry in msgraph to an entity.
+   */
+  organizationTransformer?: OrganizationTransformer;
+}
+
 /**
  * Reads user and group entries out of Microsoft Graph, and provides them as
  * User and Group entities for the catalog.
@@ -47,25 +105,21 @@ import {
  */
 export class MicrosoftGraphOrgEntityProvider implements EntityProvider {
   private connection?: EntityProviderConnection;
+  private scheduleFn?: () => Promise<void>;
 
   static fromConfig(
-    config: Config,
-    options: {
-      id: string;
-      target: string;
-      logger: Logger;
-      userTransformer?: UserTransformer;
-      groupTransformer?: GroupTransformer;
-      organizationTransformer?: OrganizationTransformer;
-    },
+    configRoot: Config,
+    options: MicrosoftGraphOrgEntityProviderOptions,
   ) {
-    const c = config.getOptionalConfig('catalog.processors.microsoftGraphOrg');
-    const providers = c ? readMicrosoftGraphConfig(c) : [];
+    const config = configRoot.getOptionalConfig(
+      'catalog.processors.microsoftGraphOrg',
+    );
+    const providers = config ? readMicrosoftGraphConfig(config) : [];
     const provider = providers.find(p => options.target.startsWith(p.target));
 
     if (!provider) {
       throw new Error(
-        `There is no Microsoft Graph Org provider that matches ${options.target}. Please add a configuration entry for it under catalog.processors.microsoftGraphOrg.providers.`,
+        `There is no Microsoft Graph Org provider that matches "${options.target}". Please add a configuration entry for it under "catalog.processors.microsoftGraphOrg.providers".`,
       );
     }
 
@@ -73,7 +127,7 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider {
       target: options.target,
     });
 
-    return new MicrosoftGraphOrgEntityProvider({
+    const result = new MicrosoftGraphOrgEntityProvider({
       id: options.id,
       userTransformer: options.userTransformer,
       groupTransformer: options.groupTransformer,
@@ -81,6 +135,10 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider {
       logger,
       provider,
     });
+
+    result.schedule(options.schedule);
+
+    return result;
   }
 
   constructor(
@@ -102,19 +160,21 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider {
   /** {@inheritdoc @backstage/plugin-catalog-backend#EntityProvider.connect} */
   async connect(connection: EntityProviderConnection) {
     this.connection = connection;
+    await this.scheduleFn?.();
   }
 
   /**
    * Runs one complete ingestion loop. Call this method regularly at some
    * appropriate cadence.
    */
-  async read() {
+  async read(options?: { logger?: Logger }) {
     if (!this.connection) {
       throw new Error('Not initialized');
     }
 
+    const logger = options?.logger ?? this.options.logger;
     const provider = this.options.provider;
-    const { markReadComplete } = trackProgress(this.options.logger);
+    const { markReadComplete } = trackProgress(logger);
     const client = MicrosoftGraphClient.create(this.options.provider);
 
     const { users, groups } = await readMicrosoftGraphOrg(
@@ -130,7 +190,7 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider {
         groupTransformer: this.options.groupTransformer,
         userTransformer: this.options.userTransformer,
         organizationTransformer: this.options.organizationTransformer,
-        logger: this.options.logger,
+        logger: logger,
       },
     );
 
@@ -146,6 +206,34 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider {
 
     markCommitComplete();
   }
+
+  private schedule(
+    schedule: MicrosoftGraphOrgEntityProviderOptions['schedule'],
+  ) {
+    if (schedule === 'manual') {
+      return;
+    }
+
+    this.scheduleFn = async () => {
+      const id = `${this.getProviderName()}:refresh`;
+      await schedule.run({
+        id,
+        fn: async () => {
+          const logger = this.options.logger.child({
+            class: MicrosoftGraphOrgEntityProvider.prototype.constructor.name,
+            taskId: id,
+            taskInstanceId: uuid.v4(),
+          });
+
+          try {
+            await this.read({ logger });
+          } catch (error) {
+            logger.error(error);
+          }
+        },
+      });
+    };
+  }
 }
 
 // Helps wrap the timing and logging behaviors
@@ -173,12 +261,12 @@ function trackProgress(logger: Logger) {
 
 // Makes sure that emitted entities have a proper location based on their uuid
 export function withLocations(providerId: string, entity: Entity): Entity {
-  const uuid =
+  const uid =
     entity.metadata.annotations?.[MICROSOFT_GRAPH_USER_ID_ANNOTATION] ||
     entity.metadata.annotations?.[MICROSOFT_GRAPH_GROUP_ID_ANNOTATION] ||
     entity.metadata.annotations?.[MICROSOFT_GRAPH_TENANT_ID_ANNOTATION] ||
     entity.metadata.name;
-  const location = `msgraph:${providerId}/${encodeURIComponent(uuid)}`;
+  const location = `msgraph:${providerId}/${encodeURIComponent(uid)}`;
   return merge(
     {
       metadata: {

plugins/catalog-backend-module-msgraph/src/processors/index.ts

@@ -15,4 +15,5 @@
  */
 
 export { MicrosoftGraphOrgEntityProvider } from './MicrosoftGraphOrgEntityProvider';
+export type { MicrosoftGraphOrgEntityProviderOptions } from './MicrosoftGraphOrgEntityProvider';
 export { MicrosoftGraphOrgReaderProcessor } from './MicrosoftGraphOrgReaderProcessor';