Merge pull request #34053 from sriharsha9618/msgraph-incremental

feat(catalog): add plugin-catalog-backend-module-msgraph-incremental
This commit is contained in:
Fredrik Adelöw
2026-05-05 16:21:37 +03:00
committed by GitHub
19 changed files with 2769 additions and 2 deletions
@@ -0,0 +1,5 @@
---
'@backstage/plugin-catalog-backend-module-msgraph-incremental': minor
---
Introduces a cursor-based incremental ingestion provider for Microsoft Graph that processes users and groups one page at a time. Unlike `MicrosoftGraphOrgEntityProvider`, this module never holds the full dataset in memory — each burst processes a single page (up to 999 users or 100 groups). The `@odata.nextLink` cursor is persisted so a pod restart resumes from the last completed page rather than starting over.
+1
View File
@@ -58,6 +58,7 @@ yarn.lock @backstage/maintainers @backst
/plugins/catalog-backend-module-backstage-openapi @backstage/maintainers @backstage/openapi-tooling-maintainers
/plugins/catalog-backend-module-gitea @backstage/maintainers @backstage/catalog-maintainers
/plugins/catalog-backend-module-msgraph @backstage/maintainers @backstage/catalog-maintainers @pjungermann
/plugins/catalog-backend-module-msgraph-incremental @backstage/maintainers @backstage/catalog-maintainers
/plugins/catalog-backend-module-puppetdb @backstage/maintainers @backstage/catalog-maintainers
/plugins/catalog-graph @backstage/maintainers @backstage/catalog-maintainers @backstage/sda-se-reviewers
/plugins/devtools @backstage/maintainers @awanlin
+31
View File
@@ -50,6 +50,37 @@ backend.add(import('@backstage/plugin-catalog-backend-module-msgraph'));
/* highlight-add-end */
```
## Incremental Ingestion for Large Tenants
For very large Azure AD tenants where loading the full dataset into memory at once is not feasible, the `@backstage/plugin-catalog-backend-module-msgraph-incremental` package provides a memory-efficient alternative. It processes users and groups one page at a time and persists the `@odata.nextLink` cursor so ingestion resumes from the last completed page after a pod restart.
```bash title="From your Backstage root directory"
yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-incremental-ingestion
yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-msgraph-incremental
```
```ts title="packages/backend/src/index.ts"
backend.add(import('@backstage/plugin-catalog-backend'));
/* highlight-add-start */
backend.add(
import('@backstage/plugin-catalog-backend-module-incremental-ingestion'),
);
backend.add(
import('@backstage/plugin-catalog-backend-module-msgraph-incremental'),
);
/* highlight-add-end */
```
It uses the same `catalog.providers.microsoftGraphOrg` configuration as the standard module. The following options are **not** supported by the incremental provider: `userGroupMember*` and `groupIncludeSubGroups`. Use `MicrosoftGraphOrgEntityProvider` if you require those.
| | `MicrosoftGraphOrgEntityProvider` | Incremental provider |
| -------------------------- | --------------------------------- | -------------------- |
| Memory usage | Full dataset in RAM | One page at a time |
| Resume on restart | Starts from scratch | Resumes from cursor |
| `userGroupMember*` options | Supported | Not supported |
| `groupIncludeSubGroups` | Supported | Not supported |
| Suitable for large tenants | No | Yes |
## Authenticating with Microsoft Graph
### Local Development
@@ -0,0 +1 @@
module.exports = require('@backstage/cli/config/eslint-factory')(__dirname);
@@ -0,0 +1,74 @@
# @backstage/plugin-catalog-backend-module-msgraph-incremental
This module incrementally ingests **users** and **groups** from Microsoft Graph
into the Backstage catalog, one page at a time. It is suitable for large Azure
AD tenants where holding the full dataset in memory at once is not practical.
## Features
- **Cursor-based resumption** — the `@odata.nextLink` URL is persisted as the
cursor, so a pod restart during ingestion resumes from the last completed page
rather than starting over.
- **Memory-efficient** — each burst processes a single page (up to 999 users
or 100 groups), keeping memory usage flat regardless of tenant size.
- **Photo support** — user profile photos are fetched with a gated pre-check to
avoid unnecessary API calls for users without photos.
- **Transformer extension point** — user, group, organization, and provider
config transformers can be customised via the
`microsoftGraphIncrementalEntityProviderTransformExtensionPoint`.
## Prerequisites
This module requires the incremental ingestion framework to be installed:
```ts
backend.add(
import('@backstage/plugin-catalog-backend-module-incremental-ingestion'),
);
```
## Installation
```ts
// packages/backend/src/index.ts
backend.add(
import('@backstage/plugin-catalog-backend-module-incremental-ingestion'),
);
backend.add(
import('@backstage/plugin-catalog-backend-module-msgraph-incremental'),
);
```
## Configuration
Uses the same `catalog.providers.microsoftGraphOrg` configuration as
`@backstage/plugin-catalog-backend-module-msgraph`. See that package's
documentation for full config reference.
```yaml
catalog:
providers:
microsoftGraphOrg:
default:
tenantId: ${AZURE_TENANT_ID}
clientId: ${AZURE_CLIENT_ID}
clientSecret: ${AZURE_CLIENT_SECRET}
queryMode: advanced
user:
filter: 'accountEnabled eq true'
group:
filter: 'securityEnabled eq true'
schedule:
frequency: { hours: 12 }
timeout: { hours: 4 }
```
## Differences from `MicrosoftGraphOrgEntityProvider`
| | `MicrosoftGraphOrgEntityProvider` | This module |
| -------------------------- | --------------------------------- | ------------------- |
| Memory usage | Full dataset in RAM | One page at a time |
| Resume on restart | Starts from scratch | Resumes from cursor |
| `userGroupMember*` options | Supported | Not supported |
| `groupIncludeSubGroups` | Supported | Not supported |
| Suitable for large tenants | No | Yes |
@@ -0,0 +1,12 @@
apiVersion: backstage.io/v1alpha1
kind: Component
metadata:
name: backstage-plugin-catalog-backend-module-msgraph-incremental
title: '@backstage/plugin-catalog-backend-module-msgraph-incremental'
description: >-
A Backstage catalog backend module that incrementally ingests users and
groups from Microsoft Graph, one page at a time
spec:
lifecycle: experimental
type: backstage-backend-plugin-module
owner: catalog-maintainers
@@ -0,0 +1,67 @@
{
"name": "@backstage/plugin-catalog-backend-module-msgraph-incremental",
"version": "0.0.0",
"description": "A Backstage catalog backend module that incrementally ingests users and groups from Microsoft Graph",
"backstage": {
"role": "backend-plugin-module",
"pluginId": "catalog",
"pluginPackage": "@backstage/plugin-catalog-backend"
},
"publishConfig": {
"access": "public"
},
"keywords": [
"backstage"
],
"homepage": "https://backstage.io",
"repository": {
"type": "git",
"url": "https://github.com/backstage/backstage",
"directory": "plugins/catalog-backend-module-msgraph-incremental"
},
"license": "Apache-2.0",
"exports": {
".": "./src/index.ts",
"./alpha": "./src/alpha.ts",
"./package.json": "./package.json"
},
"main": "src/index.ts",
"types": "src/index.ts",
"typesVersions": {
"*": {
"alpha": [
"src/alpha.ts"
],
"package.json": [
"package.json"
]
}
},
"files": [
"dist"
],
"scripts": {
"build": "backstage-cli package build",
"clean": "backstage-cli package clean",
"lint": "backstage-cli package lint",
"prepack": "backstage-cli package prepack",
"postpack": "backstage-cli package postpack",
"start": "backstage-cli package start",
"test": "backstage-cli package test"
},
"dependencies": {
"@backstage/backend-plugin-api": "workspace:^",
"@backstage/catalog-model": "workspace:^",
"@backstage/config": "workspace:^",
"@backstage/plugin-catalog-backend-module-incremental-ingestion": "workspace:^",
"@backstage/plugin-catalog-backend-module-msgraph": "workspace:^",
"@backstage/plugin-catalog-node": "workspace:^",
"@backstage/types": "workspace:^",
"@microsoft/microsoft-graph-types": "^2.6.0",
"p-limit": "^3.0.2"
},
"devDependencies": {
"@backstage/backend-test-utils": "workspace:^",
"@backstage/cli": "workspace:^"
}
}
@@ -0,0 +1,7 @@
## API Report File for "@backstage/plugin-catalog-backend-module-msgraph-incremental"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
```ts
// (No @packageDocumentation comment for this package)
```
@@ -0,0 +1,96 @@
## API Report File for "@backstage/plugin-catalog-backend-module-msgraph-incremental"
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
```ts
import { BackendFeature } from '@backstage/backend-plugin-api';
import { Config } from '@backstage/config';
import { EntityIteratorResult } from '@backstage/plugin-catalog-backend-module-incremental-ingestion';
import { ExtensionPoint } from '@backstage/backend-plugin-api';
import { GroupTransformer } from '@backstage/plugin-catalog-backend-module-msgraph';
import { IncrementalEntityProvider } from '@backstage/plugin-catalog-backend-module-incremental-ingestion';
import { LoggerService } from '@backstage/backend-plugin-api';
import { MicrosoftGraphClient } from '@backstage/plugin-catalog-backend-module-msgraph';
import { MicrosoftGraphProviderConfig } from '@backstage/plugin-catalog-backend-module-msgraph';
import { OrganizationTransformer } from '@backstage/plugin-catalog-backend-module-msgraph';
import { ProviderConfigTransformer } from '@backstage/plugin-catalog-backend-module-msgraph';
import { UserTransformer } from '@backstage/plugin-catalog-backend-module-msgraph';
// @public
const catalogModuleMicrosoftGraphIncrementalEntityProvider: BackendFeature;
export { catalogModuleMicrosoftGraphIncrementalEntityProvider };
export default catalogModuleMicrosoftGraphIncrementalEntityProvider;
// @public
export class MicrosoftGraphIncrementalEntityProvider
implements IncrementalEntityProvider<MSGraphCursor, MSGraphContext>
{
constructor(options: {
id: string;
provider: MicrosoftGraphProviderConfig;
logger: LoggerService;
userTransformer?: UserTransformer;
groupTransformer?: GroupTransformer;
organizationTransformer?: OrganizationTransformer;
providerConfigTransformer?: ProviderConfigTransformer;
});
around(burst: (context: MSGraphContext) => Promise<void>): Promise<void>;
static fromConfig(
configRoot: Config,
options: MicrosoftGraphIncrementalEntityProviderOptions,
): MicrosoftGraphIncrementalEntityProvider[];
getProviderName(): string;
next(
input: MSGraphContext,
cursor?: MSGraphCursor,
): Promise<EntityIteratorResult<MSGraphCursor>>;
}
// @public
export interface MicrosoftGraphIncrementalEntityProviderOptions {
groupTransformer?: GroupTransformer | Record<string, GroupTransformer>;
logger: LoggerService;
organizationTransformer?:
| OrganizationTransformer
| Record<string, OrganizationTransformer>;
providerConfigTransformer?:
| ProviderConfigTransformer
| Record<string, ProviderConfigTransformer>;
userTransformer?: UserTransformer | Record<string, UserTransformer>;
}
// @public
export const microsoftGraphIncrementalEntityProviderTransformExtensionPoint: ExtensionPoint<MicrosoftGraphIncrementalEntityProviderTransformsExtensionPoint>;
// @public
export interface MicrosoftGraphIncrementalEntityProviderTransformsExtensionPoint {
setGroupTransformer(
transformer: GroupTransformer | Record<string, GroupTransformer>,
): void;
setOrganizationTransformer(
transformer:
| OrganizationTransformer
| Record<string, OrganizationTransformer>,
): void;
setProviderConfigTransformer(
transformer:
| ProviderConfigTransformer
| Record<string, ProviderConfigTransformer>,
): void;
setUserTransformer(
transformer: UserTransformer | Record<string, UserTransformer>,
): void;
}
// @public
export type MSGraphContext = {
client: MicrosoftGraphClient;
provider: MicrosoftGraphProviderConfig;
};
// @public
export type MSGraphCursor = {
phase: 'users' | 'groups';
nextLink?: string;
};
```
@@ -0,0 +1,551 @@
/*
* Copyright 2026 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import crypto from 'node:crypto';
import {
ANNOTATION_LOCATION,
ANNOTATION_ORIGIN_LOCATION,
Entity,
stringifyEntityRef,
} from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import {
IncrementalEntityProvider,
EntityIteratorResult,
} from '@backstage/plugin-catalog-backend-module-incremental-ingestion';
import { DeferredEntity } from '@backstage/plugin-catalog-node';
import limiterFactory from 'p-limit';
import * as MicrosoftGraph from '@microsoft/microsoft-graph-types';
import {
GroupTransformer,
MICROSOFT_GRAPH_GROUP_ID_ANNOTATION,
MICROSOFT_GRAPH_TENANT_ID_ANNOTATION,
MICROSOFT_GRAPH_USER_ID_ANNOTATION,
MicrosoftGraphClient,
MicrosoftGraphProviderConfig,
OrganizationTransformer,
ProviderConfigTransformer,
UserTransformer,
defaultGroupTransformer,
defaultOrganizationTransformer,
defaultUserTransformer,
readProviderConfigs,
} from '@backstage/plugin-catalog-backend-module-msgraph';
import { LoggerService } from '@backstage/backend-plugin-api';
import { getUserPhotoGated, requestOnePage } from './clientHelpers';
const USER_PAGE_SIZE = 999;
// Groups phase fetches members for every group on the page, so a smaller page
// size keeps each burst within its time budget.
const GROUP_PAGE_SIZE = 100;
/**
* Backstage entity names must be 63 chars ([a-zA-Z0-9] separated by [-_.]).
* When MS Graph UPNs exceed that (e.g. calendar/booking accounts), we truncate
* to 54 chars and append an 8-char SHA-1 hash to preserve uniqueness.
*/
function capEntityName(name: string): string {
if (name.length <= 63) return name;
const hash = crypto.createHash('sha1').update(name).digest('hex').slice(0, 8);
return `${name.slice(0, 54)}_${hash}`;
}
/** Stamps `annotations.backstage.io/location` on an entity using the MS Graph UID. */
function withLocations(providerId: string, entity: Entity): Entity {
const uid =
entity.metadata.annotations?.[MICROSOFT_GRAPH_USER_ID_ANNOTATION] ||
entity.metadata.annotations?.[MICROSOFT_GRAPH_GROUP_ID_ANNOTATION] ||
entity.metadata.annotations?.[MICROSOFT_GRAPH_TENANT_ID_ANNOTATION] ||
entity.metadata.name;
const location = `msgraph:${providerId}/${encodeURIComponent(uid)}`;
return {
...entity,
metadata: {
...entity.metadata,
annotations: {
...entity.metadata.annotations,
[ANNOTATION_LOCATION]: location,
[ANNOTATION_ORIGIN_LOCATION]: location,
},
},
};
}
/**
* Pagination cursor used by {@link MicrosoftGraphIncrementalEntityProvider}.
*
* The `nextLink` field holds the `@odata.nextLink` URL returned by the
* Microsoft Graph API, which encodes all state needed to resume a paged
* request. An absent value means the current phase is starting fresh.
*
* @public
*/
export type MSGraphCursor = {
phase: 'users' | 'groups';
nextLink?: string;
};
/**
* Context passed to each burst of {@link MicrosoftGraphIncrementalEntityProvider}.
*
* @public
*/
export type MSGraphContext = {
client: MicrosoftGraphClient;
provider: MicrosoftGraphProviderConfig;
};
/**
* Options for {@link MicrosoftGraphIncrementalEntityProvider}.
*
* @public
*/
export interface MicrosoftGraphIncrementalEntityProviderOptions {
/**
* The logger to use.
*/
logger: LoggerService;
/**
* The function that transforms a user entry in msgraph to an entity.
* Optionally, you can pass separate transformers per provider ID.
*/
userTransformer?: UserTransformer | Record<string, UserTransformer>;
/**
* The function that transforms a group entry in msgraph to an entity.
* Optionally, you can pass separate transformers per provider ID.
*/
groupTransformer?: GroupTransformer | Record<string, GroupTransformer>;
/**
* The function that transforms an organization entry in msgraph to an entity.
* Optionally, you can pass separate transformers per provider ID.
*/
organizationTransformer?:
| OrganizationTransformer
| Record<string, OrganizationTransformer>;
/**
* The function that transforms provider config dynamically before each sync.
* Optionally, you can pass separate transformers per provider ID.
*/
providerConfigTransformer?:
| ProviderConfigTransformer
| Record<string, ProviderConfigTransformer>;
}
/**
* Incrementally reads user and group entries out of Microsoft Graph, one page
* at a time, and provides them as User and Group entities for the catalog.
*
* Unlike `MicrosoftGraphOrgEntityProvider`, this provider never holds the full
* dataset in memory at once. Each burst processes a single page (up to 999
* users or 100 groups). This makes it suitable for very large tenants and
* avoids the memory pressure and long-running task issues of the full-scan
* provider.
*
* The Microsoft Graph `@odata.nextLink` URL is stored as the cursor, so a pod
* restart during ingestion resumes from the last completed page.
*
* Group membership (`spec.members`) is resolved inline during the groups phase
* by fetching the direct members of each group. The catalog's built-in relation
* stitching derives `spec.memberOf` on users from these group membership lists.
*
* @remarks
* `userGroupMemberFilter`, `userGroupMemberSearch`, `userGroupMemberPath`, and
* `groupIncludeSubGroups` are not supported. Use `userFilter` / `userPath` to
* restrict which users are ingested, and `groupFilter` / `groupSearch` to
* restrict which groups. Switch to `MicrosoftGraphOrgEntityProvider` if you
* require any of these options.
*
* @public
*/
export class MicrosoftGraphIncrementalEntityProvider
implements IncrementalEntityProvider<MSGraphCursor, MSGraphContext>
{
/**
* Create one provider instance per provider entry in
* `catalog.providers.microsoftGraphOrg`.
*/
static fromConfig(
configRoot: Config,
options: MicrosoftGraphIncrementalEntityProviderOptions,
): MicrosoftGraphIncrementalEntityProvider[] {
function getTransformer<T extends Function>(
id: string,
transformers?: T | Record<string, T>,
): T | undefined {
if (['undefined', 'function'].includes(typeof transformers)) {
return transformers as T;
}
return (transformers as Record<string, T>)[id];
}
return readProviderConfigs(configRoot).map(
providerConfig =>
new MicrosoftGraphIncrementalEntityProvider({
id: providerConfig.id,
provider: providerConfig,
logger: options.logger,
userTransformer: getTransformer(
providerConfig.id,
options.userTransformer,
),
groupTransformer: getTransformer(
providerConfig.id,
options.groupTransformer,
),
organizationTransformer: getTransformer(
providerConfig.id,
options.organizationTransformer,
),
providerConfigTransformer: getTransformer(
providerConfig.id,
options.providerConfigTransformer,
),
}),
);
}
private readonly options: {
id: string;
provider: MicrosoftGraphProviderConfig;
logger: LoggerService;
userTransformer?: UserTransformer;
groupTransformer?: GroupTransformer;
organizationTransformer?: OrganizationTransformer;
providerConfigTransformer?: ProviderConfigTransformer;
};
constructor(options: {
id: string;
provider: MicrosoftGraphProviderConfig;
logger: LoggerService;
userTransformer?: UserTransformer;
groupTransformer?: GroupTransformer;
organizationTransformer?: OrganizationTransformer;
providerConfigTransformer?: ProviderConfigTransformer;
}) {
this.options = options;
}
/** {@inheritdoc @backstage/plugin-catalog-backend-module-incremental-ingestion#IncrementalEntityProvider.getProviderName} */
getProviderName(): string {
return `MicrosoftGraphIncrementalEntityProvider:${this.options.id}`;
}
/**
* Sets up the Microsoft Graph client for the duration of a full ingestion
* cycle. The optional `providerConfigTransformer` is applied here so that
* dynamic config changes (e.g., rotating credentials) take effect at the
* start of each cycle rather than mid-way through.
*/
async around(
burst: (context: MSGraphContext) => Promise<void>,
): Promise<void> {
const provider = this.options.providerConfigTransformer
? await this.options.providerConfigTransformer(this.options.provider)
: this.options.provider;
if (
provider.userGroupMemberFilter ||
provider.userGroupMemberSearch ||
provider.userGroupMemberPath
) {
this.options.logger.warn(
`${this.getProviderName()}: userGroupMemberFilter/Search/Path are not supported by ` +
`MicrosoftGraphIncrementalEntityProvider. Users will be fetched via the standard ` +
`userFilter/userPath options instead. Switch to MicrosoftGraphOrgEntityProvider if ` +
`you require userGroupMember-based ingestion.`,
);
}
if (provider.groupIncludeSubGroups) {
this.options.logger.warn(
`${this.getProviderName()}: groupIncludeSubGroups is not supported by ` +
`MicrosoftGraphIncrementalEntityProvider and will be ignored. ` +
`Switch to MicrosoftGraphOrgEntityProvider if you require this option.`,
);
}
const client = MicrosoftGraphClient.create(provider);
await burst({ client, provider });
}
/** {@inheritdoc @backstage/plugin-catalog-backend-module-incremental-ingestion#IncrementalEntityProvider.next} */
async next(
{ client, provider }: MSGraphContext,
cursor?: MSGraphCursor,
): Promise<EntityIteratorResult<MSGraphCursor>> {
const phase = cursor?.phase ?? 'users';
const nextLink = cursor?.nextLink;
if (phase === 'users') {
return this.readUsersPage(client, provider, nextLink);
}
return this.readGroupsPage(client, provider, nextLink);
}
private async readUsersPage(
client: MicrosoftGraphClient,
provider: MicrosoftGraphProviderConfig,
nextLink: string | undefined,
): Promise<EntityIteratorResult<MSGraphCursor>> {
const { items: rawUsers, nextLink: newNextLink } =
await requestOnePage<MicrosoftGraph.User>(
client,
provider.userPath ?? 'users',
{
query: {
filter: provider.userFilter,
expand: provider.userExpand,
select: provider.userSelect,
top: USER_PAGE_SIZE,
},
queryMode: provider.queryMode,
nextLink,
},
);
const transformer = this.options.userTransformer ?? defaultUserTransformer;
const limiter = limiterFactory(10);
const entities: DeferredEntity[] = [];
await Promise.all(
rawUsers.map(user =>
limiter(async () => {
let userPhoto: string | undefined;
if (user.id && provider.loadUserPhotos !== false) {
try {
userPhoto = await getUserPhotoGated(client, user.id, 120);
} catch (e) {
this.options.logger.debug(
`${this.getProviderName()}: failed to load photo for user ${
user.id
}`,
{ error: e },
);
}
}
const entity = await transformer(user, userPhoto);
if (entity) {
entity.metadata.name = capEntityName(entity.metadata.name);
entities.push({
locationKey: `msgraph-org-provider:${this.options.id}`,
entity: withLocations(this.options.id, entity),
});
}
}),
),
);
this.options.logger.debug(
`${this.getProviderName()}: read ${entities.length} users`,
{ phase: 'users', hasNextPage: !!newNextLink },
);
if (newNextLink) {
return {
done: false,
entities,
cursor: { phase: 'users', nextLink: newNextLink },
};
}
return {
done: false,
entities,
cursor: { phase: 'groups' },
};
}
private async readGroupsPage(
client: MicrosoftGraphClient,
provider: MicrosoftGraphProviderConfig,
nextLink: string | undefined,
): Promise<EntityIteratorResult<MSGraphCursor>> {
const { items: rawGroups, nextLink: newNextLink } =
await requestOnePage<MicrosoftGraph.Group>(
client,
provider.groupPath ?? 'groups',
{
query: {
filter: provider.groupFilter,
search: provider.groupSearch,
expand: provider.groupExpand,
select: provider.groupSelect,
top: GROUP_PAGE_SIZE,
},
queryMode: provider.queryMode,
nextLink,
},
);
const groupTransformer =
this.options.groupTransformer ?? defaultGroupTransformer;
const userTransformer =
this.options.userTransformer ?? defaultUserTransformer;
const limiter = limiterFactory(10);
const entities: DeferredEntity[] = [];
// Emit the tenant root group on the very first groups page
if (!nextLink) {
try {
const organization = await client.getOrganization(provider.tenantId);
const orgTransformer =
this.options.organizationTransformer ??
defaultOrganizationTransformer;
const rootGroup = await orgTransformer(organization);
if (rootGroup) {
entities.push({
locationKey: `msgraph-org-provider:${this.options.id}`,
entity: withLocations(this.options.id, rootGroup),
});
}
} catch (e) {
this.options.logger.warn(
`${this.getProviderName()}: failed to read organization root group`,
{ error: e },
);
}
}
await Promise.all(
rawGroups.map(group =>
limiter(async () => {
const entity = await groupTransformer(group);
if (!entity) {
return;
}
entity.metadata.name = capEntityName(entity.metadata.name);
const userRefs: string[] = [];
const childRefs: string[] = [];
for await (const member of client.getGroupMembers(group.id!, {
top: GROUP_PAGE_SIZE,
// Request the minimum fields needed by defaultUserTransformer and
// defaultGroupTransformer so member objects are never sparse.
select: [
'id',
'displayName',
'mail',
'mailNickname',
'userPrincipalName',
'description',
'securityEnabled',
],
})) {
if (member['@odata.type'] === '#microsoft.graph.user') {
try {
const userEntity = await userTransformer(
member as MicrosoftGraph.User,
);
if (userEntity) {
userEntity.metadata.name = capEntityName(
userEntity.metadata.name,
);
userRefs.push(stringifyEntityRef(userEntity));
} else {
this.options.logger.debug(
`${this.getProviderName()}: group member user ${
member.id
} could not be transformed (sparse object?), skipping`,
);
}
} catch (e) {
this.options.logger.warn(
`${this.getProviderName()}: group member user ${
member.id
} failed to transform, skipping`,
{ error: e },
);
}
} else if (member['@odata.type'] === '#microsoft.graph.group') {
// Only emit child refs when no group filter/search is active.
// With a filter, child groups may not be ingested themselves,
// which would produce dangling spec.children references.
if (!provider.groupFilter && !provider.groupSearch) {
try {
const childEntity = await groupTransformer(
member as MicrosoftGraph.Group,
);
if (childEntity) {
childEntity.metadata.name = capEntityName(
childEntity.metadata.name,
);
childRefs.push(stringifyEntityRef(childEntity));
} else {
this.options.logger.debug(
`${this.getProviderName()}: group member child group ${
member.id
} could not be transformed (sparse object?), skipping`,
);
}
} catch (e) {
this.options.logger.warn(
`${this.getProviderName()}: group member child group ${
member.id
} failed to transform, skipping`,
{ error: e },
);
}
}
}
}
// Merge fetched membership with any members/children the transformer
// may have pre-populated, so custom transformers can augment the list.
const existingMembers = Array.isArray(entity.spec?.members)
? (entity.spec.members as string[])
: [];
const existingChildren = Array.isArray(entity.spec?.children)
? (entity.spec.children as string[])
: [];
entities.push({
locationKey: `msgraph-org-provider:${this.options.id}`,
entity: withLocations(this.options.id, {
...entity,
spec: {
...entity.spec,
members: [...new Set([...existingMembers, ...userRefs])],
children: [...new Set([...existingChildren, ...childRefs])],
},
}),
});
}),
),
);
this.options.logger.debug(
`${this.getProviderName()}: read ${rawGroups.length} groups`,
{ phase: 'groups', hasNextPage: !!newNextLink },
);
if (newNextLink) {
return {
done: false,
entities,
cursor: { phase: 'groups', nextLink: newNextLink },
};
}
return { done: true, entities };
}
}
@@ -0,0 +1,17 @@
/*
* Copyright 2026 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export {};
@@ -0,0 +1,298 @@
/*
* Copyright 2026 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { MicrosoftGraphClient } from '@backstage/plugin-catalog-backend-module-msgraph';
import { getUserPhotoGated, requestOnePage } from './clientHelpers';
function makeResponse(status: number, body: unknown): Response {
return {
status,
json: () => Promise.resolve(body),
} as unknown as Response;
}
describe('requestOnePage', () => {
const client = {
requestApi: jest.fn(),
requestRaw: jest.fn(),
} as unknown as jest.Mocked<MicrosoftGraphClient>;
afterEach(() => jest.resetAllMocks());
it('fetches a basic page using requestApi', async () => {
(client.requestApi as jest.Mock).mockResolvedValue(
makeResponse(200, {
value: [{ id: '1', displayName: 'Alice' }],
}),
);
const result = await requestOnePage<{ id: string }>(client, 'users', {
query: { top: 999 },
});
expect(result.items).toHaveLength(1);
expect(result.items[0].id).toBe('1');
expect(result.nextLink).toBeUndefined();
expect(client.requestApi).toHaveBeenCalledWith(
'users',
{ top: 999 },
{},
undefined,
);
});
it('adds ConsistencyLevel and $count for advanced mode with filter', async () => {
(client.requestApi as jest.Mock).mockResolvedValue(
makeResponse(200, { value: [] }),
);
await requestOnePage(client, 'users', {
query: {
filter: 'accountEnabled eq true and jobTitle ne null',
top: 999,
},
queryMode: 'advanced',
});
expect(client.requestApi).toHaveBeenCalledWith(
'users',
expect.objectContaining({ count: true }),
{ ConsistencyLevel: 'eventual' },
undefined,
);
});
it('adds ConsistencyLevel and $count for advanced mode without filter', async () => {
(client.requestApi as jest.Mock).mockResolvedValue(
makeResponse(200, { value: [] }),
);
await requestOnePage(client, 'groups', {
query: { top: 999 },
queryMode: 'advanced',
});
expect(client.requestApi).toHaveBeenCalledWith(
'groups',
expect.objectContaining({ count: true }),
{ ConsistencyLevel: 'eventual' },
undefined,
);
});
it('auto-promotes to advanced mode when $search is present', async () => {
(client.requestApi as jest.Mock).mockResolvedValue(
makeResponse(200, { value: [] }),
);
await requestOnePage(client, 'groups', {
query: { search: '"displayName:Sales"', top: 10 },
});
expect(client.requestApi).toHaveBeenCalledWith(
'groups',
expect.objectContaining({ count: true }),
{ ConsistencyLevel: 'eventual' },
undefined,
);
});
it('sends $count=true in advanced mode even when no query is provided', async () => {
(client.requestApi as jest.Mock).mockResolvedValue(
makeResponse(200, { value: [] }),
);
await requestOnePage(client, 'users', { queryMode: 'advanced' });
expect(client.requestApi).toHaveBeenCalledWith(
'users',
expect.objectContaining({ count: true }),
{ ConsistencyLevel: 'eventual' },
undefined,
);
});
it('does not set $count for basic mode', async () => {
(client.requestApi as jest.Mock).mockResolvedValue(
makeResponse(200, { value: [] }),
);
await requestOnePage(client, 'users', {
query: { filter: 'accountEnabled eq true', top: 999 },
queryMode: 'basic',
});
expect(client.requestApi).toHaveBeenCalledWith(
'users',
expect.not.objectContaining({ count: true }),
{},
undefined,
);
});
it('follows nextLink using requestRaw', async () => {
const nextLink =
'https://graph.microsoft.com/v1.0/users?$skiptoken=abc123&$count=true';
(client.requestRaw as jest.Mock).mockResolvedValue(
makeResponse(200, {
value: [{ id: '2' }],
'@odata.nextLink':
'https://graph.microsoft.com/v1.0/users?$skiptoken=def456',
}),
);
const result = await requestOnePage(client, 'users', { nextLink });
expect(client.requestRaw).toHaveBeenCalledWith(nextLink, {}, 2, undefined);
expect(client.requestApi).not.toHaveBeenCalled();
expect(result.items).toHaveLength(1);
expect(result.nextLink).toBe(
'https://graph.microsoft.com/v1.0/users?$skiptoken=def456',
);
});
it('passes ConsistencyLevel header when following nextLink in advanced mode', async () => {
const nextLink = 'https://graph.microsoft.com/v1.0/groups?$skiptoken=xyz';
(client.requestRaw as jest.Mock).mockResolvedValue(
makeResponse(200, { value: [] }),
);
await requestOnePage(client, 'groups', {
query: { filter: 'securityEnabled eq true', top: 999 },
queryMode: 'advanced',
nextLink,
});
expect(client.requestRaw).toHaveBeenCalledWith(
nextLink,
{ ConsistencyLevel: 'eventual' },
2,
undefined,
);
});
it('returns nextLink from response when present', async () => {
const expectedNextLink =
'https://graph.microsoft.com/v1.0/users?$skiptoken=page2';
(client.requestApi as jest.Mock).mockResolvedValue(
makeResponse(200, {
value: Array(999).fill({ id: 'x' }),
'@odata.nextLink': expectedNextLink,
}),
);
const result = await requestOnePage(client, 'users', {});
expect(result.nextLink).toBe(expectedNextLink);
expect(result.items).toHaveLength(999);
});
it('throws a descriptive error on non-200 response', async () => {
(client.requestApi as jest.Mock).mockResolvedValue(
makeResponse(403, {
error: {
code: 'Authorization_RequestDenied',
message: 'Access denied',
},
}),
);
await expect(requestOnePage(client, 'users', {})).rejects.toThrow(
'Authorization_RequestDenied - Access denied',
);
});
it('passes AbortSignal through to the client', async () => {
(client.requestApi as jest.Mock).mockResolvedValue(
makeResponse(200, { value: [] }),
);
const signal = new AbortController().signal;
await requestOnePage(client, 'users', { signal });
expect(client.requestApi).toHaveBeenCalledWith(
'users',
undefined,
{},
signal,
);
});
});
describe('getUserPhotoGated', () => {
const client = {
requestApi: jest.fn(),
getUserPhotoWithSizeLimit: jest.fn(),
} as unknown as jest.Mocked<MicrosoftGraphClient>;
afterEach(() => jest.resetAllMocks());
it('returns undefined when the photo check returns 404', async () => {
(client.requestApi as jest.Mock).mockResolvedValue({
status: 404,
} as Response);
const result = await getUserPhotoGated(client, 'user-id', 120);
expect(result).toBeUndefined();
expect(client.getUserPhotoWithSizeLimit).not.toHaveBeenCalled();
});
it('throws for non-404 error responses so callers can distinguish real errors', async () => {
(client.requestApi as jest.Mock).mockResolvedValue({
status: 403,
} as Response);
await expect(getUserPhotoGated(client, 'user-id', 120)).rejects.toThrow(
'Unexpected status 403 when checking photo for user user-id',
);
expect(client.getUserPhotoWithSizeLimit).not.toHaveBeenCalled();
});
it('returns the photo data URI when the check passes', async () => {
(client.requestApi as jest.Mock).mockResolvedValue({
status: 200,
} as Response);
(client.getUserPhotoWithSizeLimit as jest.Mock).mockResolvedValue(
'data:image/jpeg;base64,/9j/abc123',
);
const result = await getUserPhotoGated(client, 'user-id', 120);
expect(result).toBe('data:image/jpeg;base64,/9j/abc123');
expect(client.requestApi).toHaveBeenCalledWith('users/user-id/photo');
expect(client.getUserPhotoWithSizeLimit).toHaveBeenCalledWith(
'user-id',
120,
);
});
it('passes the maxSize limit to getUserPhotoWithSizeLimit', async () => {
(client.requestApi as jest.Mock).mockResolvedValue({
status: 200,
} as Response);
(client.getUserPhotoWithSizeLimit as jest.Mock).mockResolvedValue(
undefined,
);
await getUserPhotoGated(client, 'user-abc', 48);
expect(client.getUserPhotoWithSizeLimit).toHaveBeenCalledWith(
'user-abc',
48,
);
});
});
@@ -0,0 +1,103 @@
/*
* Copyright 2026 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
MicrosoftGraphClient,
ODataQuery,
} from '@backstage/plugin-catalog-backend-module-msgraph';
/**
* Fetches a single page of Graph API results.
*
* When `options.nextLink` is provided it is followed directly (all query
* parameters are already encoded in it). Otherwise the request is built from
* `options.query`.
*
* MS Graph requires `ConsistencyLevel: eventual` + `$count=true` for advanced
* queries using `ne`/`not` operators in `$filter` or using `$search`.
*/
export async function requestOnePage<T>(
client: MicrosoftGraphClient,
path: string,
options: {
query?: ODataQuery;
queryMode?: 'basic' | 'advanced';
nextLink?: string;
signal?: AbortSignal;
} = {},
): Promise<{ items: T[]; nextLink?: string }> {
const { query, queryMode, nextLink, signal } = options;
const appliedQueryMode = query?.search ? 'advanced' : queryMode ?? 'basic';
// Microsoft Graph requires $count=true whenever ConsistencyLevel: eventual is set,
// including plain listing requests with no $filter or $search.
const finalQuery =
appliedQueryMode === 'advanced' ? { ...(query ?? {}), count: true } : query;
const headers: Record<string, string> =
appliedQueryMode === 'advanced' ? { ConsistencyLevel: 'eventual' } : {};
const response = nextLink
? await client.requestRaw(nextLink, headers, 2, signal)
: await client.requestApi(path, finalQuery, headers, signal);
if (response.status !== 200) {
let message = `HTTP ${response.status}`;
try {
const body = await response.json();
const err = body?.error;
if (err?.code || err?.message) {
message = `${err.code} - ${err.message}`;
}
} catch {
// Response body is not JSON; fall back to HTTP status above
}
throw new Error(
`Error while reading ${
nextLink ?? path
} from Microsoft Graph: ${message}`,
);
}
const result = await response.json();
return {
items: result.value as T[],
nextLink: result['@odata.nextLink'],
};
}
/**
* Like `getUserPhotoWithSizeLimit` but skips the size-listing call for users
* with no photo. For users without a photo: 1 fast check call. For users with
* a photo: 1 check + the normal size-limited fetch (2 more calls).
*
* Returns `undefined` only for 404 (no photo assigned). Throws for any other
* non-200 status so callers can distinguish "no photo" from real errors.
*/
export async function getUserPhotoGated(
client: MicrosoftGraphClient,
userId: string,
maxSize: number,
): Promise<string | undefined> {
const check = await client.requestApi(`users/${userId}/photo`);
if (check.status === 404) return undefined;
if (check.status !== 200) {
throw new Error(
`Unexpected status ${check.status} when checking photo for user ${userId}`,
);
}
return await client.getUserPhotoWithSizeLimit(userId, maxSize);
}
@@ -0,0 +1,31 @@
/*
* Copyright 2026 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* A Backstage catalog backend module that incrementally ingests users and
* groups from Microsoft Graph, one page at a time.
*
* @packageDocumentation
*/
export { catalogModuleMicrosoftGraphIncrementalEntityProvider as default } from './module';
export * from './module';
export { MicrosoftGraphIncrementalEntityProvider } from './MicrosoftGraphIncrementalEntityProvider';
export type {
MSGraphCursor,
MSGraphContext,
MicrosoftGraphIncrementalEntityProviderOptions,
} from './MicrosoftGraphIncrementalEntityProvider';
@@ -0,0 +1,134 @@
/*
* Copyright 2026 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { mockServices, startTestBackend } from '@backstage/backend-test-utils';
import { incrementalIngestionProvidersExtensionPoint } from '@backstage/plugin-catalog-backend-module-incremental-ingestion';
import { catalogModuleMicrosoftGraphIncrementalEntityProvider } from './catalogModuleMicrosoftGraphIncrementalEntityProvider';
import { MicrosoftGraphIncrementalEntityProvider } from '../MicrosoftGraphIncrementalEntityProvider';
describe('catalogModuleMicrosoftGraphIncrementalEntityProvider', () => {
it('registers the provider at the incremental ingestion extension point', async () => {
const addProvider = jest.fn();
const extensionPoint = { addProvider };
await startTestBackend({
extensionPoints: [
[incrementalIngestionProvidersExtensionPoint, extensionPoint],
],
features: [
catalogModuleMicrosoftGraphIncrementalEntityProvider,
mockServices.rootConfig.factory({
data: {
catalog: {
providers: {
microsoftGraphOrg: {
default: {
tenantId: 'tenant-id',
clientId: 'client-id',
clientSecret: 'client-secret',
schedule: {
frequency: 'PT12H',
timeout: 'PT4H',
},
},
},
},
},
},
}),
],
});
expect(addProvider).toHaveBeenCalledTimes(1);
const { provider, options } = addProvider.mock.calls[0][0];
expect(provider).toBeInstanceOf(MicrosoftGraphIncrementalEntityProvider);
expect(provider.getProviderName()).toBe(
'MicrosoftGraphIncrementalEntityProvider:default',
);
expect(options.burstInterval).toEqual({ seconds: 3 });
expect(options.burstLength).toEqual({ minutes: 5 });
// restLength derived from schedule.frequency (12h)
expect(options.restLength).toEqual({ hours: 12 });
});
it('creates one provider per config entry', async () => {
const addProvider = jest.fn();
const extensionPoint = { addProvider };
await startTestBackend({
extensionPoints: [
[incrementalIngestionProvidersExtensionPoint, extensionPoint],
],
features: [
catalogModuleMicrosoftGraphIncrementalEntityProvider,
mockServices.rootConfig.factory({
data: {
catalog: {
providers: {
microsoftGraphOrg: {
tenantA: {
tenantId: 'a',
clientId: 'c',
clientSecret: 's',
},
tenantB: {
tenantId: 'b',
clientId: 'c',
clientSecret: 's',
},
},
},
},
},
}),
],
});
expect(addProvider).toHaveBeenCalledTimes(2);
});
it('defaults restLength to 8 hours when no schedule frequency is configured', async () => {
const addProvider = jest.fn();
const extensionPoint = { addProvider };
await startTestBackend({
extensionPoints: [
[incrementalIngestionProvidersExtensionPoint, extensionPoint],
],
features: [
catalogModuleMicrosoftGraphIncrementalEntityProvider,
mockServices.rootConfig.factory({
data: {
catalog: {
providers: {
microsoftGraphOrg: {
default: {
tenantId: 'tenant-id',
clientId: 'client-id',
clientSecret: 'client-secret',
},
},
},
},
},
}),
],
});
const { options } = addProvider.mock.calls[0][0];
expect(options.restLength).toEqual({ hours: 8 });
});
});
@@ -0,0 +1,252 @@
/*
* Copyright 2026 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
coreServices,
createBackendModule,
createExtensionPoint,
LoggerService,
} from '@backstage/backend-plugin-api';
import { incrementalIngestionProvidersExtensionPoint } from '@backstage/plugin-catalog-backend-module-incremental-ingestion';
import {
GroupTransformer,
OrganizationTransformer,
ProviderConfigTransformer,
UserTransformer,
readProviderConfigs,
} from '@backstage/plugin-catalog-backend-module-msgraph';
import { HumanDuration } from '@backstage/types';
import {
MicrosoftGraphIncrementalEntityProvider,
MSGraphContext,
MSGraphCursor,
} from '../MicrosoftGraphIncrementalEntityProvider';
/**
* Interface for
* {@link microsoftGraphIncrementalEntityProviderTransformExtensionPoint}.
*
* @public
*/
export interface MicrosoftGraphIncrementalEntityProviderTransformsExtensionPoint {
/**
* Set the function that transforms a user entry in msgraph to an entity.
* Optionally, you can pass separate transformers per provider ID.
*/
setUserTransformer(
transformer: UserTransformer | Record<string, UserTransformer>,
): void;
/**
* Set the function that transforms a group entry in msgraph to an entity.
* Optionally, you can pass separate transformers per provider ID.
*/
setGroupTransformer(
transformer: GroupTransformer | Record<string, GroupTransformer>,
): void;
/**
* Set the function that transforms an organization entry in msgraph to an
* entity. Optionally, you can pass separate transformers per provider ID.
*/
setOrganizationTransformer(
transformer:
| OrganizationTransformer
| Record<string, OrganizationTransformer>,
): void;
/**
* Set the function that transforms provider config dynamically.
* Optionally, you can pass separate transformers per provider ID.
*/
setProviderConfigTransformer(
transformer:
| ProviderConfigTransformer
| Record<string, ProviderConfigTransformer>,
): void;
}
/**
* Extension point used to customize the transforms applied by the incremental
* module.
*
* @public
*/
export const microsoftGraphIncrementalEntityProviderTransformExtensionPoint =
createExtensionPoint<MicrosoftGraphIncrementalEntityProviderTransformsExtensionPoint>(
{
id: 'catalog.microsoftGraphIncrementalEntityProvider.transforms',
},
);
/**
* Registers {@link MicrosoftGraphIncrementalEntityProvider} instances with the
* catalog's incremental ingestion extension point.
*
* This module requires `catalogModuleIncrementalIngestionEntityProvider` to
* also be installed in the backend.
*
* @example
* ```ts
* // packages/backend/src/index.ts
* backend.add(import('@backstage/plugin-catalog-backend-module-incremental-ingestion'));
* backend.add(import('@backstage/plugin-catalog-backend-module-msgraph-incremental'));
* ```
*
* @public
*/
export const catalogModuleMicrosoftGraphIncrementalEntityProvider =
createBackendModule({
pluginId: 'catalog',
moduleId: 'microsoftGraphIncrementalEntityProvider',
register(env) {
let userTransformer:
| UserTransformer
| Record<string, UserTransformer>
| undefined;
let groupTransformer:
| GroupTransformer
| Record<string, GroupTransformer>
| undefined;
let organizationTransformer:
| OrganizationTransformer
| Record<string, OrganizationTransformer>
| undefined;
let providerConfigTransformer:
| ProviderConfigTransformer
| Record<string, ProviderConfigTransformer>
| undefined;
env.registerExtensionPoint(
microsoftGraphIncrementalEntityProviderTransformExtensionPoint,
{
setUserTransformer(transformer) {
if (userTransformer) {
throw new Error('User transformer may only be set once');
}
userTransformer = transformer;
},
setGroupTransformer(transformer) {
if (groupTransformer) {
throw new Error('Group transformer may only be set once');
}
groupTransformer = transformer;
},
setOrganizationTransformer(transformer) {
if (organizationTransformer) {
throw new Error('Organization transformer may only be set once');
}
organizationTransformer = transformer;
},
setProviderConfigTransformer(transformer) {
if (providerConfigTransformer) {
throw new Error(
'Provider config transformer may only be set once',
);
}
providerConfigTransformer = transformer;
},
},
);
env.registerInit({
deps: {
config: coreServices.rootConfig,
logger: coreServices.logger,
incremental: incrementalIngestionProvidersExtensionPoint,
},
async init({ config, logger, incremental }) {
const providerConfigs = readProviderConfigs(config);
for (const providerConfig of providerConfigs) {
const provider = new MicrosoftGraphIncrementalEntityProvider({
id: providerConfig.id,
provider: providerConfig,
logger,
userTransformer: resolveTransformer(
providerConfig.id,
userTransformer,
),
groupTransformer: resolveTransformer(
providerConfig.id,
groupTransformer,
),
organizationTransformer: resolveTransformer(
providerConfig.id,
organizationTransformer,
),
providerConfigTransformer: resolveTransformer(
providerConfig.id,
providerConfigTransformer,
),
});
const restLength = deriveRestLength(providerConfig, logger);
incremental.addProvider<MSGraphCursor, MSGraphContext>({
provider,
options: {
burstInterval: { seconds: 3 },
burstLength: { minutes: 5 },
restLength,
backoff: [
{ seconds: 30 },
{ minutes: 3 },
{ minutes: 30 },
{ hours: 3 },
],
},
});
}
},
});
},
});
function resolveTransformer<T extends Function>(
id: string,
transformer?: T | Record<string, T>,
): T | undefined {
if (['undefined', 'function'].includes(typeof transformer)) {
return transformer as T;
}
return (transformer as Record<string, T>)[id];
}
function deriveRestLength(
providerConfig: ReturnType<typeof readProviderConfigs>[number],
logger: LoggerService,
): HumanDuration {
const freq = providerConfig.schedule?.frequency;
// Only treat plain duration objects as restLength — exclude cron expressions
// and any other non-duration schedule types (e.g. manual triggers).
if (
freq &&
typeof freq === 'object' &&
!('cron' in freq) &&
!('trigger' in freq)
) {
return freq as HumanDuration;
}
if (freq) {
logger.warn(
`MicrosoftGraphIncrementalEntityProvider:${providerConfig.id}: ` +
`schedule.frequency is not a duration-based schedule; cannot derive restLength from it. ` +
`Defaulting restLength to 8 hours.`,
);
}
return { hours: 8 };
}
@@ -0,0 +1,21 @@
/*
* Copyright 2026 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export {
catalogModuleMicrosoftGraphIncrementalEntityProvider,
microsoftGraphIncrementalEntityProviderTransformExtensionPoint,
type MicrosoftGraphIncrementalEntityProviderTransformsExtensionPoint,
} from './catalogModuleMicrosoftGraphIncrementalEntityProvider';
+20 -2
View File
@@ -5002,7 +5002,7 @@ __metadata:
languageName: unknown
linkType: soft
"@backstage/plugin-catalog-backend-module-incremental-ingestion@workspace:plugins/catalog-backend-module-incremental-ingestion":
"@backstage/plugin-catalog-backend-module-incremental-ingestion@workspace:^, @backstage/plugin-catalog-backend-module-incremental-ingestion@workspace:plugins/catalog-backend-module-incremental-ingestion":
version: 0.0.0-use.local
resolution: "@backstage/plugin-catalog-backend-module-incremental-ingestion@workspace:plugins/catalog-backend-module-incremental-ingestion"
dependencies:
@@ -5056,7 +5056,25 @@ __metadata:
languageName: unknown
linkType: soft
"@backstage/plugin-catalog-backend-module-msgraph@workspace:plugins/catalog-backend-module-msgraph":
"@backstage/plugin-catalog-backend-module-msgraph-incremental@workspace:plugins/catalog-backend-module-msgraph-incremental":
version: 0.0.0-use.local
resolution: "@backstage/plugin-catalog-backend-module-msgraph-incremental@workspace:plugins/catalog-backend-module-msgraph-incremental"
dependencies:
"@backstage/backend-plugin-api": "workspace:^"
"@backstage/backend-test-utils": "workspace:^"
"@backstage/catalog-model": "workspace:^"
"@backstage/cli": "workspace:^"
"@backstage/config": "workspace:^"
"@backstage/plugin-catalog-backend-module-incremental-ingestion": "workspace:^"
"@backstage/plugin-catalog-backend-module-msgraph": "workspace:^"
"@backstage/plugin-catalog-node": "workspace:^"
"@backstage/types": "workspace:^"
"@microsoft/microsoft-graph-types": "npm:^2.6.0"
p-limit: "npm:^3.0.2"
languageName: unknown
linkType: soft
"@backstage/plugin-catalog-backend-module-msgraph@workspace:^, @backstage/plugin-catalog-backend-module-msgraph@workspace:plugins/catalog-backend-module-msgraph":
version: 0.0.0-use.local
resolution: "@backstage/plugin-catalog-backend-module-msgraph@workspace:plugins/catalog-backend-module-msgraph"
dependencies: