diff --git a/.changeset/msgraph-incremental-initial.md b/.changeset/msgraph-incremental-initial.md index 3137c2b57f..6f41346a12 100644 --- a/.changeset/msgraph-incremental-initial.md +++ b/.changeset/msgraph-incremental-initial.md @@ -1,7 +1,5 @@ --- -'@backstage/plugin-catalog-backend-module-msgraph-incremental': patch +'@backstage/plugin-catalog-backend-module-msgraph-incremental': minor --- -New package: `@backstage/plugin-catalog-backend-module-msgraph-incremental` - Introduces a cursor-based incremental ingestion provider for Microsoft Graph that processes users and groups one page at a time. Unlike `MicrosoftGraphOrgEntityProvider`, this module never holds the full dataset in memory — each burst processes a single page (up to 999 users or 100 groups). The `@odata.nextLink` cursor is persisted so a pod restart resumes from the last completed page rather than starting over. diff --git a/docs/integrations/azure/org.md b/docs/integrations/azure/org.md index 3c82f6e2be..defe0aab41 100644 --- a/docs/integrations/azure/org.md +++ b/docs/integrations/azure/org.md @@ -50,6 +50,37 @@ backend.add(import('@backstage/plugin-catalog-backend-module-msgraph')); /* highlight-add-end */ ``` +## Incremental Ingestion for Large Tenants + +For very large Azure AD tenants where loading the full dataset into memory at once is not feasible, the `@backstage/plugin-catalog-backend-module-msgraph-incremental` package provides a memory-efficient alternative. It processes users and groups one page at a time and persists the `@odata.nextLink` cursor so ingestion resumes from the last completed page after a pod restart. + +```bash title="From your Backstage root directory" +yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-incremental-ingestion +yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-msgraph-incremental +``` + +```ts title="packages/backend/src/index.ts" +backend.add(import('@backstage/plugin-catalog-backend')); +/* highlight-add-start */ +backend.add( + import('@backstage/plugin-catalog-backend-module-incremental-ingestion'), +); +backend.add( + import('@backstage/plugin-catalog-backend-module-msgraph-incremental'), +); +/* highlight-add-end */ +``` + +It uses the same `catalog.providers.microsoftGraphOrg` configuration as the standard module. The following options are **not** supported by the incremental provider: `userGroupMember*` and `groupIncludeSubGroups`. Use `MicrosoftGraphOrgEntityProvider` if you require those. + +| | `MicrosoftGraphOrgEntityProvider` | Incremental provider | +| -------------------------- | --------------------------------- | -------------------- | +| Memory usage | Full dataset in RAM | One page at a time | +| Resume on restart | Starts from scratch | Resumes from cursor | +| `userGroupMember*` options | Supported | Not supported | +| `groupIncludeSubGroups` | Supported | Not supported | +| Suitable for large tenants | No | Yes | + ## Authenticating with Microsoft Graph ### Local Development diff --git a/plugins/catalog-backend-module-msgraph-incremental/CHANGELOG.md b/plugins/catalog-backend-module-msgraph-incremental/CHANGELOG.md deleted file mode 100644 index 0bb7f6f0f8..0000000000 --- a/plugins/catalog-backend-module-msgraph-incremental/CHANGELOG.md +++ /dev/null @@ -1,16 +0,0 @@ -# @backstage/plugin-catalog-backend-module-msgraph-incremental - -## 0.1.0 - -### Minor Changes - -- Initial release of the incremental Microsoft Graph catalog module. - - Provides `catalogModuleMicrosoftGraphIncrementalEntityProvider`, which - ingests users and groups from Microsoft Graph one page at a time using the - catalog's incremental ingestion framework. Unlike - `MicrosoftGraphOrgEntityProvider`, this module never holds the full dataset - in memory and resumes from its cursor on pod restart. - - Requires `@backstage/plugin-catalog-backend-module-incremental-ingestion` to - be installed in the backend. diff --git a/plugins/catalog-backend-module-msgraph-incremental/package.json b/plugins/catalog-backend-module-msgraph-incremental/package.json index e84a57f403..e9b025e9ad 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/package.json +++ b/plugins/catalog-backend-module-msgraph-incremental/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-catalog-backend-module-msgraph-incremental", - "version": "0.1.0", + "version": "0.0.0", "description": "A Backstage catalog backend module that incrementally ingests users and groups from Microsoft Graph", "backstage": { "role": "backend-plugin-module", diff --git a/plugins/catalog-backend-module-msgraph-incremental/report-alpha.api.md b/plugins/catalog-backend-module-msgraph-incremental/report-alpha.api.md index ed6376fb04..6cf15e34d4 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/report-alpha.api.md +++ b/plugins/catalog-backend-module-msgraph-incremental/report-alpha.api.md @@ -3,96 +3,5 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts -import { BackendFeature } from '@backstage/backend-plugin-api'; -import { Config } from '@backstage/config'; -import { EntityIteratorResult } from '@backstage/plugin-catalog-backend-module-incremental-ingestion'; -import { ExtensionPoint } from '@backstage/backend-plugin-api'; -import { GroupTransformer } from '@backstage/plugin-catalog-backend-module-msgraph'; -import { IncrementalEntityProvider } from '@backstage/plugin-catalog-backend-module-incremental-ingestion'; -import { LoggerService } from '@backstage/backend-plugin-api'; -import { MicrosoftGraphClient } from '@backstage/plugin-catalog-backend-module-msgraph'; -import { MicrosoftGraphProviderConfig } from '@backstage/plugin-catalog-backend-module-msgraph'; -import { OrganizationTransformer } from '@backstage/plugin-catalog-backend-module-msgraph'; -import { ProviderConfigTransformer } from '@backstage/plugin-catalog-backend-module-msgraph'; -import { UserTransformer } from '@backstage/plugin-catalog-backend-module-msgraph'; -// @public -const catalogModuleMicrosoftGraphIncrementalEntityProvider: BackendFeature; -export { catalogModuleMicrosoftGraphIncrementalEntityProvider }; -export default catalogModuleMicrosoftGraphIncrementalEntityProvider; - -// @public -export class MicrosoftGraphIncrementalEntityProvider - implements IncrementalEntityProvider -{ - constructor(options: { - id: string; - provider: MicrosoftGraphProviderConfig; - logger: LoggerService; - userTransformer?: UserTransformer; - groupTransformer?: GroupTransformer; - organizationTransformer?: OrganizationTransformer; - providerConfigTransformer?: ProviderConfigTransformer; - }); - around(burst: (context: MSGraphContext) => Promise): Promise; - static fromConfig( - configRoot: Config, - options: MicrosoftGraphIncrementalEntityProviderOptions, - ): MicrosoftGraphIncrementalEntityProvider[]; - getProviderName(): string; - next( - input: MSGraphContext, - cursor?: MSGraphCursor, - ): Promise>; -} - -// @public -export interface MicrosoftGraphIncrementalEntityProviderOptions { - groupTransformer?: GroupTransformer | Record; - logger: LoggerService; - organizationTransformer?: - | OrganizationTransformer - | Record; - providerConfigTransformer?: - | ProviderConfigTransformer - | Record; - userTransformer?: UserTransformer | Record; -} - -// @public -export const microsoftGraphIncrementalEntityProviderTransformExtensionPoint: ExtensionPoint; - -// @public -export interface MicrosoftGraphIncrementalEntityProviderTransformsExtensionPoint { - setGroupTransformer( - transformer: GroupTransformer | Record, - ): void; - setOrganizationTransformer( - transformer: - | OrganizationTransformer - | Record, - ): void; - setProviderConfigTransformer( - transformer: - | ProviderConfigTransformer - | Record, - ): void; - setUserTransformer( - transformer: UserTransformer | Record, - ): void; -} - -// @public -export type MSGraphContext = { - client: MicrosoftGraphClient; - provider: MicrosoftGraphProviderConfig; -}; - -// @public -export type MSGraphCursor = { - phase: 'users' | 'groups'; - nextLink?: string; -}; - -// (No @packageDocumentation comment for this package) ``` diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts index 630b246f81..faf8d1f4a7 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Backstage Authors + * Copyright 2026 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts index dedc94b13b..57cbc16c66 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Backstage Authors + * Copyright 2026 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/alpha.ts b/plugins/catalog-backend-module-msgraph-incremental/src/alpha.ts index cb99b69c6a..e512e4ba83 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/alpha.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/alpha.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Backstage Authors + * Copyright 2026 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,4 @@ * limitations under the License. */ -// Re-export everything from the main entry point while the package is in alpha. -export * from './index'; -export { default } from './index'; +export {}; diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.test.ts b/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.test.ts index 93dd290f2a..a103787f0d 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.test.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.test.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Backstage Authors + * Copyright 2026 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.ts b/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.ts index 2397ba46b9..61b78e4c64 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Backstage Authors + * Copyright 2026 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/index.ts b/plugins/catalog-backend-module-msgraph-incremental/src/index.ts index dbae9a4f2c..37ad8b96c6 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/index.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/index.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Backstage Authors + * Copyright 2026 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.test.ts b/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.test.ts index a30388bae1..c7ca22fe48 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.test.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.test.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Backstage Authors + * Copyright 2026 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.ts b/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.ts index 674ce7a009..3ab51aa9b0 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Backstage Authors + * Copyright 2026 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/module/index.ts b/plugins/catalog-backend-module-msgraph-incremental/src/module/index.ts index 5d5e3710a4..988418dd9b 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/module/index.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/module/index.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Backstage Authors + * Copyright 2026 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License.