From f1279ea2d69f7f292a14d58f5c06592429842f33 Mon Sep 17 00:00:00 2001 From: pillaris Date: Thu, 23 Apr 2026 15:56:43 +0530 Subject: [PATCH 01/20] feat(catalog): add catalog-backend-module-msgraph-incremental plugin Introduces a new Backstage backend module that incrementally ingests users and groups from Microsoft Graph one page at a time, using the incremental ingestion framework. Unlike MicrosoftGraphOrgEntityProvider, this module never holds the full dataset in memory. Each burst processes a single page (up to 999 items), making it suitable for large Azure AD tenants where the full-scan provider causes memory pressure or OOM failures. The @odata.nextLink cursor is persisted in the incremental ingestion marks table, so a pod restart during ingestion resumes from the last completed page rather than starting over. Signed-off-by: pillaris --- .changeset/msgraph-incremental-initial.md | 7 + .github/CODEOWNERS | 1 + packages/backend/package.json | 3 + packages/backend/src/index.ts | 6 + .../.eslintrc.js | 1 + .../CHANGELOG.md | 16 + .../README.md | 73 ++ .../catalog-info.yaml | 12 + .../package.json | 66 ++ .../report-alpha.api.md | 98 +++ .../report.api.md | 96 +++ ...softGraphIncrementalEntityProvider.test.ts | 779 ++++++++++++++++++ ...MicrosoftGraphIncrementalEntityProvider.ts | 474 +++++++++++ .../src/alpha.ts | 19 + .../src/clientHelpers.test.ts | 254 ++++++ .../src/clientHelpers.ts | 91 ++ .../src/index.ts | 31 + ...softGraphIncrementalEntityProvider.test.ts | 134 +++ ...MicrosoftGraphIncrementalEntityProvider.ts | 235 ++++++ .../src/module/index.ts | 21 + yarn.lock | 26 +- 21 files changed, 2440 insertions(+), 3 deletions(-) create mode 100644 .changeset/msgraph-incremental-initial.md create mode 100644 plugins/catalog-backend-module-msgraph-incremental/.eslintrc.js create mode 100644 plugins/catalog-backend-module-msgraph-incremental/CHANGELOG.md create mode 100644 plugins/catalog-backend-module-msgraph-incremental/README.md create mode 100644 plugins/catalog-backend-module-msgraph-incremental/catalog-info.yaml create mode 100644 plugins/catalog-backend-module-msgraph-incremental/package.json create mode 100644 plugins/catalog-backend-module-msgraph-incremental/report-alpha.api.md create mode 100644 plugins/catalog-backend-module-msgraph-incremental/report.api.md create mode 100644 plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts create mode 100644 plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts create mode 100644 plugins/catalog-backend-module-msgraph-incremental/src/alpha.ts create mode 100644 plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.test.ts create mode 100644 plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.ts create mode 100644 plugins/catalog-backend-module-msgraph-incremental/src/index.ts create mode 100644 plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.test.ts create mode 100644 plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.ts create mode 100644 plugins/catalog-backend-module-msgraph-incremental/src/module/index.ts diff --git a/.changeset/msgraph-incremental-initial.md b/.changeset/msgraph-incremental-initial.md new file mode 100644 index 0000000000..2a16af91a8 --- /dev/null +++ b/.changeset/msgraph-incremental-initial.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-catalog-backend-module-msgraph-incremental': patch +--- + +New package: `@backstage/plugin-catalog-backend-module-msgraph-incremental` + +Introduces a cursor-based incremental ingestion provider for Microsoft Graph that processes users and groups one page at a time. Unlike `MicrosoftGraphOrgEntityProvider`, this module never holds the full dataset in memory — each burst processes a single page of up to 999 items. The `@odata.nextLink` cursor is persisted so a pod restart resumes from the last completed page rather than starting over. diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 7e5cc36bdf..c91f17fb23 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -58,6 +58,7 @@ yarn.lock @backstage/maintainers @backst /plugins/catalog-backend-module-backstage-openapi @backstage/maintainers @backstage/openapi-tooling-maintainers /plugins/catalog-backend-module-gitea @backstage/maintainers @backstage/catalog-maintainers /plugins/catalog-backend-module-msgraph @backstage/maintainers @backstage/catalog-maintainers @pjungermann +/plugins/catalog-backend-module-msgraph-incremental @backstage/maintainers @backstage/catalog-maintainers /plugins/catalog-backend-module-puppetdb @backstage/maintainers @backstage/catalog-maintainers /plugins/catalog-graph @backstage/maintainers @backstage/catalog-maintainers @backstage/sda-se-reviewers /plugins/devtools @backstage/maintainers @awanlin diff --git a/packages/backend/package.json b/packages/backend/package.json index e00539b095..88ae7f3351 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -41,7 +41,10 @@ "@backstage/plugin-auth-node": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", "@backstage/plugin-catalog-backend-module-backstage-openapi": "workspace:^", + "@backstage/plugin-catalog-backend-module-incremental-ingestion": "workspace:^", "@backstage/plugin-catalog-backend-module-logs": "workspace:^", + "@backstage/plugin-catalog-backend-module-msgraph": "workspace:^", + "@backstage/plugin-catalog-backend-module-msgraph-incremental": "workspace:^", "@backstage/plugin-catalog-backend-module-openapi": "workspace:^", "@backstage/plugin-catalog-backend-module-scaffolder-entity-model": "workspace:^", "@backstage/plugin-catalog-backend-module-unprocessed": "workspace:^", diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index e77c83baac..4db72cc42d 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -46,6 +46,12 @@ backend.add(import('@backstage/plugin-auth-backend-module-guest-provider')); backend.add(import('@backstage/plugin-auth-backend-module-openshift-provider')); backend.add(import('@backstage/plugin-app-backend')); backend.add(import('@backstage/plugin-catalog-backend-module-unprocessed')); +backend.add( + import('@backstage/plugin-catalog-backend-module-incremental-ingestion'), +); +backend.add( + import('@backstage/plugin-catalog-backend-module-msgraph-incremental'), +); backend.add( import('@backstage/plugin-catalog-backend-module-scaffolder-entity-model'), ); diff --git a/plugins/catalog-backend-module-msgraph-incremental/.eslintrc.js b/plugins/catalog-backend-module-msgraph-incremental/.eslintrc.js new file mode 100644 index 0000000000..e2a53a6ad2 --- /dev/null +++ b/plugins/catalog-backend-module-msgraph-incremental/.eslintrc.js @@ -0,0 +1 @@ +module.exports = require('@backstage/cli/config/eslint-factory')(__dirname); diff --git a/plugins/catalog-backend-module-msgraph-incremental/CHANGELOG.md b/plugins/catalog-backend-module-msgraph-incremental/CHANGELOG.md new file mode 100644 index 0000000000..0bb7f6f0f8 --- /dev/null +++ b/plugins/catalog-backend-module-msgraph-incremental/CHANGELOG.md @@ -0,0 +1,16 @@ +# @backstage/plugin-catalog-backend-module-msgraph-incremental + +## 0.1.0 + +### Minor Changes + +- Initial release of the incremental Microsoft Graph catalog module. + + Provides `catalogModuleMicrosoftGraphIncrementalEntityProvider`, which + ingests users and groups from Microsoft Graph one page at a time using the + catalog's incremental ingestion framework. Unlike + `MicrosoftGraphOrgEntityProvider`, this module never holds the full dataset + in memory and resumes from its cursor on pod restart. + + Requires `@backstage/plugin-catalog-backend-module-incremental-ingestion` to + be installed in the backend. diff --git a/plugins/catalog-backend-module-msgraph-incremental/README.md b/plugins/catalog-backend-module-msgraph-incremental/README.md new file mode 100644 index 0000000000..60c971e87f --- /dev/null +++ b/plugins/catalog-backend-module-msgraph-incremental/README.md @@ -0,0 +1,73 @@ +# @backstage/plugin-catalog-backend-module-msgraph-incremental + +This module incrementally ingests **users** and **groups** from Microsoft Graph +into the Backstage catalog, one page at a time. It is suitable for large Azure +AD tenants where holding the full dataset in memory at once is not practical. + +## Features + +- **Cursor-based resumption** — the `@odata.nextLink` URL is persisted as the + cursor, so a pod restart during ingestion resumes from the last completed page + rather than starting over. +- **Memory-efficient** — each burst processes a single page (up to 999 items), + keeping memory usage flat regardless of tenant size. +- **Photo support** — user profile photos are fetched with a gated pre-check to + avoid unnecessary API calls for users without photos. +- **Transformer extension point** — user, group, organization, and provider + config transformers can be customised via the + `microsoftGraphIncrementalEntityProviderTransformExtensionPoint`. + +## Prerequisites + +This module requires the incremental ingestion framework to be installed: + +```ts +backend.add( + import('@backstage/plugin-catalog-backend-module-incremental-ingestion'), +); +``` + +## Installation + +```ts +// packages/backend/src/index.ts +backend.add( + import('@backstage/plugin-catalog-backend-module-incremental-ingestion'), +); +backend.add( + import('@backstage/plugin-catalog-backend-module-msgraph-incremental'), +); +``` + +## Configuration + +Uses the same `catalog.providers.microsoftGraphOrg` configuration as +`@backstage/plugin-catalog-backend-module-msgraph`. See that package's +documentation for full config reference. + +```yaml +catalog: + providers: + microsoftGraphOrg: + default: + tenantId: ${AZURE_TENANT_ID} + clientId: ${AZURE_CLIENT_ID} + clientSecret: ${AZURE_CLIENT_SECRET} + queryMode: advanced + user: + filter: 'accountEnabled eq true' + group: + filter: 'securityEnabled eq true' + schedule: + frequency: { hours: 12 } + timeout: { hours: 4 } +``` + +## Differences from `MicrosoftGraphOrgEntityProvider` + +| | `MicrosoftGraphOrgEntityProvider` | This module | +| -------------------------- | --------------------------------- | ------------------- | +| Memory usage | Full dataset in RAM | One page at a time | +| Resume on restart | Starts from scratch | Resumes from cursor | +| `userGroupMember*` options | Supported | Not supported | +| Suitable for large tenants | No | Yes | diff --git a/plugins/catalog-backend-module-msgraph-incremental/catalog-info.yaml b/plugins/catalog-backend-module-msgraph-incremental/catalog-info.yaml new file mode 100644 index 0000000000..a4d80f958b --- /dev/null +++ b/plugins/catalog-backend-module-msgraph-incremental/catalog-info.yaml @@ -0,0 +1,12 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: backstage-plugin-catalog-backend-module-msgraph-incremental + title: '@backstage/plugin-catalog-backend-module-msgraph-incremental' + description: >- + A Backstage catalog backend module that incrementally ingests users and + groups from Microsoft Graph, one page at a time +spec: + lifecycle: experimental + type: backstage-backend-plugin-module + owner: catalog-maintainers diff --git a/plugins/catalog-backend-module-msgraph-incremental/package.json b/plugins/catalog-backend-module-msgraph-incremental/package.json new file mode 100644 index 0000000000..e04edf0c88 --- /dev/null +++ b/plugins/catalog-backend-module-msgraph-incremental/package.json @@ -0,0 +1,66 @@ +{ + "name": "@backstage/plugin-catalog-backend-module-msgraph-incremental", + "version": "0.1.0", + "description": "A Backstage catalog backend module that incrementally ingests users and groups from Microsoft Graph", + "backstage": { + "role": "backend-plugin-module", + "pluginId": "catalog", + "pluginPackage": "@backstage/plugin-catalog-backend" + }, + "publishConfig": { + "access": "public" + }, + "keywords": [ + "backstage" + ], + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "plugins/catalog-backend-module-msgraph-incremental" + }, + "license": "Apache-2.0", + "exports": { + ".": "./src/index.ts", + "./alpha": "./src/alpha.ts", + "./package.json": "./package.json" + }, + "main": "src/index.ts", + "types": "src/index.ts", + "typesVersions": { + "*": { + "alpha": [ + "src/alpha.ts" + ], + "package.json": [ + "package.json" + ] + } + }, + "files": [ + "dist" + ], + "scripts": { + "build": "backstage-cli package build", + "clean": "backstage-cli package clean", + "lint": "backstage-cli package lint", + "prepack": "backstage-cli package prepack", + "postpack": "backstage-cli package postpack", + "start": "backstage-cli package start", + "test": "backstage-cli package test" + }, + "dependencies": { + "@backstage/backend-plugin-api": "workspace:^", + "@backstage/catalog-model": "workspace:^", + "@backstage/config": "workspace:^", + "@backstage/plugin-catalog-backend-module-incremental-ingestion": "workspace:^", + "@backstage/plugin-catalog-backend-module-msgraph": "workspace:^", + "@backstage/plugin-catalog-node": "workspace:^", + "@backstage/types": "workspace:^", + "@microsoft/microsoft-graph-types": "^2.6.0", + "p-limit": "^3.0.2" + }, + "devDependencies": { + "@backstage/cli": "workspace:^" + } +} diff --git a/plugins/catalog-backend-module-msgraph-incremental/report-alpha.api.md b/plugins/catalog-backend-module-msgraph-incremental/report-alpha.api.md new file mode 100644 index 0000000000..38d4e4278d --- /dev/null +++ b/plugins/catalog-backend-module-msgraph-incremental/report-alpha.api.md @@ -0,0 +1,98 @@ +## API Report File for "@backstage/plugin-catalog-backend-module-msgraph-incremental" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { BackendFeature } from '@backstage/backend-plugin-api'; +import { Config } from '@backstage/config'; +import { EntityIteratorResult } from '@backstage/plugin-catalog-backend-module-incremental-ingestion'; +import { ExtensionPoint } from '@backstage/backend-plugin-api'; +import { GroupTransformer } from '@backstage/plugin-catalog-backend-module-msgraph'; +import { IncrementalEntityProvider } from '@backstage/plugin-catalog-backend-module-incremental-ingestion'; +import { LoggerService } from '@backstage/backend-plugin-api'; +import { MicrosoftGraphClient } from '@backstage/plugin-catalog-backend-module-msgraph'; +import { MicrosoftGraphProviderConfig } from '@backstage/plugin-catalog-backend-module-msgraph'; +import { OrganizationTransformer } from '@backstage/plugin-catalog-backend-module-msgraph'; +import { ProviderConfigTransformer } from '@backstage/plugin-catalog-backend-module-msgraph'; +import { UserTransformer } from '@backstage/plugin-catalog-backend-module-msgraph'; + +// @public +const catalogModuleMicrosoftGraphIncrementalEntityProvider: BackendFeature; +export { catalogModuleMicrosoftGraphIncrementalEntityProvider }; +export default catalogModuleMicrosoftGraphIncrementalEntityProvider; + +// @public +export class MicrosoftGraphIncrementalEntityProvider + implements IncrementalEntityProvider +{ + constructor(options: { + id: string; + provider: MicrosoftGraphProviderConfig; + logger: LoggerService; + userTransformer?: UserTransformer; + groupTransformer?: GroupTransformer; + organizationTransformer?: OrganizationTransformer; + providerConfigTransformer?: ProviderConfigTransformer; + }); + around(burst: (context: MSGraphContext) => Promise): Promise; + static fromConfig( + configRoot: Config, + options: MicrosoftGraphIncrementalEntityProviderOptions, + ): MicrosoftGraphIncrementalEntityProvider[]; + getProviderName(): string; + next( + input: MSGraphContext, + cursor?: MSGraphCursor, + ): Promise>; +} + +// @public +export interface MicrosoftGraphIncrementalEntityProviderOptions { + groupTransformer?: GroupTransformer | Record; + logger: LoggerService; + organizationTransformer?: + | OrganizationTransformer + | Record; + providerConfigTransformer?: + | ProviderConfigTransformer + | Record; + userTransformer?: UserTransformer | Record; +} + +// @public +export const microsoftGraphIncrementalEntityProviderTransformExtensionPoint: ExtensionPoint; + +// @public +export interface MicrosoftGraphIncrementalEntityProviderTransformsExtensionPoint { + setGroupTransformer( + transformer: GroupTransformer | Record, + ): void; + setOrganizationTransformer( + transformer: + | OrganizationTransformer + | Record, + ): void; + setProviderConfigTransformer( + transformer: + | ProviderConfigTransformer + | Record, + ): void; + setUserTransformer( + transformer: UserTransformer | Record, + ): void; +} + +// @public +export type MSGraphContext = { + client: MicrosoftGraphClient; + provider: MicrosoftGraphProviderConfig; +}; + +// @public +export type MSGraphCursor = { + phase: 'users' | 'groups'; + nextLink: string | null; +}; + +// (No @packageDocumentation comment for this package) +``` diff --git a/plugins/catalog-backend-module-msgraph-incremental/report.api.md b/plugins/catalog-backend-module-msgraph-incremental/report.api.md new file mode 100644 index 0000000000..9e01dd01f8 --- /dev/null +++ b/plugins/catalog-backend-module-msgraph-incremental/report.api.md @@ -0,0 +1,96 @@ +## API Report File for "@backstage/plugin-catalog-backend-module-msgraph-incremental" + +> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). + +```ts +import { BackendFeature } from '@backstage/backend-plugin-api'; +import { Config } from '@backstage/config'; +import { EntityIteratorResult } from '@backstage/plugin-catalog-backend-module-incremental-ingestion'; +import { ExtensionPoint } from '@backstage/backend-plugin-api'; +import { GroupTransformer } from '@backstage/plugin-catalog-backend-module-msgraph'; +import { IncrementalEntityProvider } from '@backstage/plugin-catalog-backend-module-incremental-ingestion'; +import { LoggerService } from '@backstage/backend-plugin-api'; +import { MicrosoftGraphClient } from '@backstage/plugin-catalog-backend-module-msgraph'; +import { MicrosoftGraphProviderConfig } from '@backstage/plugin-catalog-backend-module-msgraph'; +import { OrganizationTransformer } from '@backstage/plugin-catalog-backend-module-msgraph'; +import { ProviderConfigTransformer } from '@backstage/plugin-catalog-backend-module-msgraph'; +import { UserTransformer } from '@backstage/plugin-catalog-backend-module-msgraph'; + +// @public +const catalogModuleMicrosoftGraphIncrementalEntityProvider: BackendFeature; +export { catalogModuleMicrosoftGraphIncrementalEntityProvider }; +export default catalogModuleMicrosoftGraphIncrementalEntityProvider; + +// @public +export class MicrosoftGraphIncrementalEntityProvider + implements IncrementalEntityProvider +{ + constructor(options: { + id: string; + provider: MicrosoftGraphProviderConfig; + logger: LoggerService; + userTransformer?: UserTransformer; + groupTransformer?: GroupTransformer; + organizationTransformer?: OrganizationTransformer; + providerConfigTransformer?: ProviderConfigTransformer; + }); + around(burst: (context: MSGraphContext) => Promise): Promise; + static fromConfig( + configRoot: Config, + options: MicrosoftGraphIncrementalEntityProviderOptions, + ): MicrosoftGraphIncrementalEntityProvider[]; + getProviderName(): string; + next( + input: MSGraphContext, + cursor?: MSGraphCursor, + ): Promise>; +} + +// @public +export interface MicrosoftGraphIncrementalEntityProviderOptions { + groupTransformer?: GroupTransformer | Record; + logger: LoggerService; + organizationTransformer?: + | OrganizationTransformer + | Record; + providerConfigTransformer?: + | ProviderConfigTransformer + | Record; + userTransformer?: UserTransformer | Record; +} + +// @public +export const microsoftGraphIncrementalEntityProviderTransformExtensionPoint: ExtensionPoint; + +// @public +export interface MicrosoftGraphIncrementalEntityProviderTransformsExtensionPoint { + setGroupTransformer( + transformer: GroupTransformer | Record, + ): void; + setOrganizationTransformer( + transformer: + | OrganizationTransformer + | Record, + ): void; + setProviderConfigTransformer( + transformer: + | ProviderConfigTransformer + | Record, + ): void; + setUserTransformer( + transformer: UserTransformer | Record, + ): void; +} + +// @public +export type MSGraphContext = { + client: MicrosoftGraphClient; + provider: MicrosoftGraphProviderConfig; +}; + +// @public +export type MSGraphCursor = { + phase: 'users' | 'groups'; + nextLink: string | null; +}; +``` diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts new file mode 100644 index 0000000000..65f998b1ad --- /dev/null +++ b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts @@ -0,0 +1,779 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + ANNOTATION_LOCATION, + ANNOTATION_ORIGIN_LOCATION, +} from '@backstage/catalog-model'; +import { ConfigReader } from '@backstage/config'; +import { + MicrosoftGraphClient, + MICROSOFT_GRAPH_GROUP_ID_ANNOTATION, + MICROSOFT_GRAPH_USER_ID_ANNOTATION, +} from '@backstage/plugin-catalog-backend-module-msgraph'; +import { mockServices } from '@backstage/backend-test-utils'; +import { + MicrosoftGraphIncrementalEntityProvider, + MSGraphContext, + MSGraphCursor, +} from './MicrosoftGraphIncrementalEntityProvider'; +import { getUserPhotoGated, requestOnePage } from './clientHelpers'; + +jest.mock('./clientHelpers', () => ({ + requestOnePage: jest.fn(), + getUserPhotoGated: jest.fn(), +})); + +const mockRequestOnePage = requestOnePage as jest.MockedFunction< + typeof requestOnePage +>; +const mockGetUserPhotoGated = getUserPhotoGated as jest.MockedFunction< + typeof getUserPhotoGated +>; + +const mockClient = { + getOrganization: jest.fn(), + getGroupMembers: jest.fn(), +} as unknown as jest.Mocked; + +const logger = mockServices.logger.mock(); + +const baseProviderConfig = { + id: 'default', + target: 'https://graph.microsoft.com/v1.0', + tenantId: 'tenant-id', + clientId: 'client-id', + clientSecret: 'client-secret', +}; + +function makeContext(overrides?: Partial): MSGraphContext { + return { + client: mockClient, + provider: baseProviderConfig as any, + ...overrides, + }; +} + +async function* asyncYield(...items: T[]): AsyncIterable { + for (const item of items) yield item; +} + +describe('MicrosoftGraphIncrementalEntityProvider', () => { + beforeEach(() => { + jest + .spyOn(MicrosoftGraphClient, 'create') + .mockReturnValue(mockClient as unknown as MicrosoftGraphClient); + }); + + afterEach(() => jest.resetAllMocks()); + + describe('getProviderName', () => { + it('returns namespaced name using the provider id', () => { + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: 'my-tenant', + provider: baseProviderConfig as any, + logger, + }); + expect(provider.getProviderName()).toBe( + 'MicrosoftGraphIncrementalEntityProvider:my-tenant', + ); + }); + }); + + describe('fromConfig', () => { + it('creates one provider per provider config entry', () => { + const config = new ConfigReader({ + catalog: { + providers: { + microsoftGraphOrg: { + tenantA: { tenantId: 'a', clientId: 'c', clientSecret: 's' }, + tenantB: { tenantId: 'b', clientId: 'c', clientSecret: 's' }, + }, + }, + }, + }); + + const providers = MicrosoftGraphIncrementalEntityProvider.fromConfig( + config, + { logger }, + ); + + expect(providers).toHaveLength(2); + expect(providers[0].getProviderName()).toBe( + 'MicrosoftGraphIncrementalEntityProvider:tenantA', + ); + expect(providers[1].getProviderName()).toBe( + 'MicrosoftGraphIncrementalEntityProvider:tenantB', + ); + }); + + it('assigns per-provider transformers when a Record is provided', () => { + const config = new ConfigReader({ + catalog: { + providers: { + microsoftGraphOrg: { + p1: { tenantId: 't', clientId: 'c', clientSecret: 's' }, + p2: { tenantId: 't', clientId: 'c', clientSecret: 's' }, + }, + }, + }, + }); + const transformerA = jest.fn(); + const transformerB = jest.fn(); + + const providers = MicrosoftGraphIncrementalEntityProvider.fromConfig( + config, + { + logger, + userTransformer: { p1: transformerA, p2: transformerB }, + }, + ); + + expect(providers).toHaveLength(2); + }); + }); + + describe('around', () => { + it('creates the client and passes it to the burst function', async () => { + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: 'default', + provider: baseProviderConfig as any, + logger, + }); + + let capturedContext: MSGraphContext | undefined; + await provider.around(async ctx => { + capturedContext = ctx; + }); + + expect(MicrosoftGraphClient.create).toHaveBeenCalledWith( + baseProviderConfig, + ); + expect(capturedContext?.client).toBe(mockClient); + }); + + it('applies providerConfigTransformer before creating the client', async () => { + const transformedConfig = { + ...baseProviderConfig, + clientSecret: 'rotated-secret', + }; + const transformer = jest.fn().mockResolvedValue(transformedConfig); + + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: 'default', + provider: baseProviderConfig as any, + logger, + providerConfigTransformer: transformer, + }); + + await provider.around(async () => {}); + + expect(transformer).toHaveBeenCalledWith(baseProviderConfig); + expect(MicrosoftGraphClient.create).toHaveBeenCalledWith( + transformedConfig, + ); + }); + + it('warns when unsupported userGroupMember options are set', async () => { + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: 'default', + provider: { + ...baseProviderConfig, + userGroupMemberFilter: 'some-filter', + } as any, + logger, + }); + + await provider.around(async () => {}); + + expect(logger.warn).toHaveBeenCalledWith( + expect.stringContaining( + 'userGroupMemberFilter/Search/Path are not supported', + ), + ); + }); + }); + + describe('next — users phase', () => { + it('starts in users phase when cursor is undefined', async () => { + mockRequestOnePage.mockResolvedValue({ + items: [], + nextLink: undefined, + }); + (mockClient.getOrganization as jest.Mock).mockResolvedValue({ + id: 'org-id', + displayName: 'My Org', + }); + (mockClient.getGroupMembers as jest.Mock).mockReturnValue(asyncYield()); + + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: 'default', + provider: baseProviderConfig as any, + logger, + }); + + const result = await provider.next(makeContext()); + + expect(mockRequestOnePage).toHaveBeenCalledWith( + mockClient, + 'users', + expect.objectContaining({ + query: expect.objectContaining({ top: 999 }), + }), + ); + // No users → advances straight to groups phase + expect(result.done).toBe(false); + expect((result.cursor as MSGraphCursor).phase).toBe('groups'); + }); + + it('emits User entities with location annotations', async () => { + mockRequestOnePage.mockResolvedValueOnce({ + items: [ + { + id: 'user-id-1', + displayName: 'Alice', + mail: 'alice@example.com', + userPrincipalName: 'alice@example.com', + }, + ], + nextLink: 'https://graph.microsoft.com/v1.0/users?$skiptoken=page2', + }); + mockGetUserPhotoGated.mockResolvedValue(undefined); + + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: 'default', + provider: baseProviderConfig as any, + logger, + }); + + const result = await provider.next(makeContext()); + + expect(result.done).toBe(false); + expect(result.entities).toHaveLength(1); + + const entity = result.entities[0].entity; + expect(entity.kind).toBe('User'); + expect( + entity.metadata.annotations?.[MICROSOFT_GRAPH_USER_ID_ANNOTATION], + ).toBe('user-id-1'); + expect(entity.metadata.annotations?.[ANNOTATION_LOCATION]).toBe( + 'msgraph:default/user-id-1', + ); + expect(entity.metadata.annotations?.[ANNOTATION_ORIGIN_LOCATION]).toBe( + 'msgraph:default/user-id-1', + ); + expect(result.entities[0].locationKey).toBe( + 'msgraph-org-provider:default', + ); + }); + + it('returns users cursor with nextLink when more pages remain', async () => { + const nextLink = + 'https://graph.microsoft.com/v1.0/users?$skiptoken=page2'; + mockRequestOnePage.mockResolvedValueOnce({ + items: [ + { id: 'u1', displayName: 'U1', userPrincipalName: 'u1@example.com' }, + ], + nextLink, + }); + mockGetUserPhotoGated.mockResolvedValue(undefined); + + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: 'default', + provider: baseProviderConfig as any, + logger, + }); + + const result = await provider.next(makeContext()); + + expect(result.done).toBe(false); + expect((result.cursor as MSGraphCursor).phase).toBe('users'); + expect((result.cursor as MSGraphCursor).nextLink).toBe(nextLink); + }); + + it('transitions to groups phase when last users page has no nextLink', async () => { + mockRequestOnePage.mockResolvedValueOnce({ + items: [], + nextLink: undefined, + }); + + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: 'default', + provider: baseProviderConfig as any, + logger, + }); + + const result = await provider.next(makeContext()); + + expect(result.done).toBe(false); + expect((result.cursor as MSGraphCursor).phase).toBe('groups'); + expect((result.cursor as MSGraphCursor).nextLink).toBeNull(); + }); + + it('skips users where the transformer returns undefined', async () => { + mockRequestOnePage.mockResolvedValueOnce({ + items: [{ id: 'u-no-name', userPrincipalName: '' }], + nextLink: undefined, + }); + mockGetUserPhotoGated.mockResolvedValue(undefined); + + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: 'default', + provider: baseProviderConfig as any, + logger, + userTransformer: async () => undefined, + }); + + const result = await provider.next(makeContext()); + + expect(result.entities).toHaveLength(0); + }); + + it('truncates entity names longer than 63 characters', async () => { + const longUPN = 'a'.repeat(64) + '@example.com'; + mockRequestOnePage.mockResolvedValueOnce({ + items: [{ id: 'u1', displayName: 'Test', userPrincipalName: longUPN }], + nextLink: undefined, + }); + mockGetUserPhotoGated.mockResolvedValue(undefined); + + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: 'default', + provider: baseProviderConfig as any, + logger, + }); + + const result = await provider.next(makeContext()); + + // Entity may be emitted or skipped depending on transformer handling of the UPN; + // what matters is that no name exceeds 63 characters. + for (const { entity } of result.entities) { + expect(entity.metadata.name.length).toBeLessThanOrEqual(63); + } + }); + + it('skips photo loading when loadUserPhotos is false', async () => { + mockRequestOnePage.mockResolvedValueOnce({ + items: [ + { + id: 'u1', + displayName: 'Alice', + userPrincipalName: 'alice@example.com', + }, + ], + nextLink: undefined, + }); + + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: 'default', + provider: { ...baseProviderConfig, loadUserPhotos: false } as any, + logger, + }); + + await provider.next( + makeContext({ + provider: { ...baseProviderConfig, loadUserPhotos: false } as any, + }), + ); + + expect(mockGetUserPhotoGated).not.toHaveBeenCalled(); + }); + + it('attempts photo loading when loadUserPhotos is not set', async () => { + mockRequestOnePage.mockResolvedValueOnce({ + items: [ + { + id: 'u1', + displayName: 'Alice', + userPrincipalName: 'alice@example.com', + }, + ], + nextLink: undefined, + }); + mockGetUserPhotoGated.mockResolvedValue(undefined); + + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: 'default', + provider: baseProviderConfig as any, + logger, + }); + + await provider.next(makeContext()); + + expect(mockGetUserPhotoGated).toHaveBeenCalledWith(mockClient, 'u1', 120); + }); + + it('continues processing remaining users when a photo load fails', async () => { + mockRequestOnePage.mockResolvedValueOnce({ + items: [ + { + id: 'u1', + displayName: 'Alice', + userPrincipalName: 'alice@example.com', + }, + { + id: 'u2', + displayName: 'Bob', + userPrincipalName: 'bob@example.com', + }, + ], + nextLink: undefined, + }); + mockGetUserPhotoGated + .mockRejectedValueOnce(new Error('Photo service unavailable')) + .mockResolvedValueOnce(undefined); + + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: 'default', + provider: baseProviderConfig as any, + logger, + }); + + const result = await provider.next(makeContext()); + + // Both users should still be emitted despite the photo failure + expect(result.entities).toHaveLength(2); + }); + }); + + describe('next — groups phase', () => { + const groupsCursor: MSGraphCursor = { phase: 'groups', nextLink: null }; + + it('emits the tenant root group on the first groups page', async () => { + (mockClient.getOrganization as jest.Mock).mockResolvedValue({ + id: 'org-id', + displayName: 'My Org', + }); + mockRequestOnePage.mockResolvedValueOnce({ + items: [], + nextLink: undefined, + }); + (mockClient.getGroupMembers as jest.Mock).mockReturnValue(asyncYield()); + + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: 'default', + provider: baseProviderConfig as any, + logger, + }); + + const result = await provider.next(makeContext(), groupsCursor); + + const rootGroup = result.entities.find( + e => + e.entity.metadata.annotations?.[ + MICROSOFT_GRAPH_GROUP_ID_ANNOTATION + ] === undefined && e.entity.spec?.type === 'root', + ); + // Root group is emitted (org entity kind=Group, type=root) + expect(result.entities.some(e => e.entity.spec?.type === 'root')).toBe( + true, + ); + }); + + it('does NOT emit the root group on subsequent pages', async () => { + const continuationCursor: MSGraphCursor = { + phase: 'groups', + nextLink: 'https://graph.microsoft.com/v1.0/groups?$skiptoken=page2', + }; + mockRequestOnePage.mockResolvedValueOnce({ + items: [], + nextLink: undefined, + }); + (mockClient.getGroupMembers as jest.Mock).mockReturnValue(asyncYield()); + + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: 'default', + provider: baseProviderConfig as any, + logger, + }); + + await provider.next(makeContext(), continuationCursor); + + expect(mockClient.getOrganization).not.toHaveBeenCalled(); + }); + + it('emits Group entities with location annotations', async () => { + (mockClient.getOrganization as jest.Mock).mockResolvedValue({ + id: 'org-id', + displayName: 'My Org', + }); + mockRequestOnePage.mockResolvedValueOnce({ + items: [ + { id: 'grp-1', displayName: 'Engineering', mail: 'eng@example.com' }, + ], + nextLink: undefined, + }); + (mockClient.getGroupMembers as jest.Mock).mockReturnValue(asyncYield()); + + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: 'default', + provider: baseProviderConfig as any, + logger, + }); + + const result = await provider.next(makeContext(), groupsCursor); + + const groupEntity = result.entities.find( + e => + e.entity.metadata.annotations?.[ + MICROSOFT_GRAPH_GROUP_ID_ANNOTATION + ] === 'grp-1', + ); + expect(groupEntity).toBeDefined(); + expect( + groupEntity!.entity.metadata.annotations?.[ANNOTATION_LOCATION], + ).toBe('msgraph:default/grp-1'); + expect( + groupEntity!.entity.metadata.annotations?.[ANNOTATION_ORIGIN_LOCATION], + ).toBe('msgraph:default/grp-1'); + expect(groupEntity!.locationKey).toBe('msgraph-org-provider:default'); + }); + + it('populates spec.members with user refs from getGroupMembers', async () => { + (mockClient.getOrganization as jest.Mock).mockResolvedValue({ + id: 'org-id', + displayName: 'My Org', + }); + mockRequestOnePage.mockResolvedValueOnce({ + items: [ + { id: 'grp-1', displayName: 'Engineering', mail: 'eng@example.com' }, + ], + nextLink: undefined, + }); + (mockClient.getGroupMembers as jest.Mock).mockReturnValue( + asyncYield( + { + '@odata.type': '#microsoft.graph.user', + id: 'u1', + displayName: 'Alice', + userPrincipalName: 'alice@example.com', + }, + { + '@odata.type': '#microsoft.graph.user', + id: 'u2', + displayName: 'Bob', + userPrincipalName: 'bob@example.com', + }, + ), + ); + + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: 'default', + provider: baseProviderConfig as any, + logger, + }); + + const result = await provider.next(makeContext(), groupsCursor); + + const groupEntity = result.entities.find( + e => + e.entity.metadata.annotations?.[ + MICROSOFT_GRAPH_GROUP_ID_ANNOTATION + ] === 'grp-1', + ); + expect(groupEntity!.entity.spec?.members).toHaveLength(2); + expect(groupEntity!.entity.spec?.members).toContain( + 'user:default/alice_example.com', + ); + }); + + it('populates spec.children with nested group refs from getGroupMembers', async () => { + (mockClient.getOrganization as jest.Mock).mockResolvedValue({ + id: 'org-id', + displayName: 'My Org', + }); + mockRequestOnePage.mockResolvedValueOnce({ + items: [ + { + id: 'grp-parent', + displayName: 'Parent', + mail: 'parent@example.com', + }, + ], + nextLink: undefined, + }); + (mockClient.getGroupMembers as jest.Mock).mockReturnValue( + asyncYield({ + '@odata.type': '#microsoft.graph.group', + id: 'grp-child', + displayName: 'Child Group', + mail: 'child@example.com', + }), + ); + + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: 'default', + provider: baseProviderConfig as any, + logger, + }); + + const result = await provider.next(makeContext(), groupsCursor); + + const parentEntity = result.entities.find( + e => + e.entity.metadata.annotations?.[ + MICROSOFT_GRAPH_GROUP_ID_ANNOTATION + ] === 'grp-parent', + ); + expect(parentEntity!.entity.spec?.children).toHaveLength(1); + }); + + it('returns done:false with groups cursor when nextLink is present', async () => { + const nextLink = + 'https://graph.microsoft.com/v1.0/groups?$skiptoken=page2'; + (mockClient.getOrganization as jest.Mock).mockResolvedValue({ + id: 'org-id', + displayName: 'My Org', + }); + mockRequestOnePage.mockResolvedValueOnce({ items: [], nextLink }); + (mockClient.getGroupMembers as jest.Mock).mockReturnValue(asyncYield()); + + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: 'default', + provider: baseProviderConfig as any, + logger, + }); + + const result = await provider.next(makeContext(), groupsCursor); + + expect(result.done).toBe(false); + expect((result.cursor as MSGraphCursor).phase).toBe('groups'); + expect((result.cursor as MSGraphCursor).nextLink).toBe(nextLink); + }); + + it('returns done:true when the last groups page has no nextLink', async () => { + (mockClient.getOrganization as jest.Mock).mockResolvedValue({ + id: 'org-id', + displayName: 'My Org', + }); + mockRequestOnePage.mockResolvedValueOnce({ + items: [], + nextLink: undefined, + }); + (mockClient.getGroupMembers as jest.Mock).mockReturnValue(asyncYield()); + + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: 'default', + provider: baseProviderConfig as any, + logger, + }); + + const result = await provider.next(makeContext(), groupsCursor); + + expect(result.done).toBe(true); + }); + + it('continues processing remaining groups when root group fetch fails', async () => { + (mockClient.getOrganization as jest.Mock).mockRejectedValue( + new Error('Organization not found'), + ); + mockRequestOnePage.mockResolvedValueOnce({ + items: [ + { id: 'grp-1', displayName: 'Engineering', mail: 'eng@example.com' }, + ], + nextLink: undefined, + }); + (mockClient.getGroupMembers as jest.Mock).mockReturnValue(asyncYield()); + + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: 'default', + provider: baseProviderConfig as any, + logger, + }); + + const result = await provider.next(makeContext(), groupsCursor); + + expect(logger.warn).toHaveBeenCalledWith( + expect.stringContaining('failed to read organization root group'), + expect.anything(), + ); + // The group itself is still emitted + expect( + result.entities.some( + e => + e.entity.metadata.annotations?.[ + MICROSOFT_GRAPH_GROUP_ID_ANNOTATION + ] === 'grp-1', + ), + ).toBe(true); + }); + + it('skips groups where the transformer returns undefined', async () => { + (mockClient.getOrganization as jest.Mock).mockResolvedValue({ + id: 'org-id', + displayName: 'My Org', + }); + mockRequestOnePage.mockResolvedValueOnce({ + items: [{ id: 'grp-1', displayName: 'Engineering' }], + nextLink: undefined, + }); + (mockClient.getGroupMembers as jest.Mock).mockReturnValue(asyncYield()); + + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: 'default', + provider: baseProviderConfig as any, + logger, + groupTransformer: async () => undefined, + }); + + const result = await provider.next(makeContext(), groupsCursor); + + // Only the root group entity remains + expect(result.entities.every(e => e.entity.spec?.type === 'root')).toBe( + true, + ); + }); + + it('passes group filter and search from provider config', async () => { + const providerWithFilter = { + ...baseProviderConfig, + groupFilter: 'securityEnabled eq true', + groupSearch: '"displayName:Engineering"', + }; + (mockClient.getOrganization as jest.Mock).mockResolvedValue({ + id: 'org-id', + displayName: 'My Org', + }); + mockRequestOnePage.mockResolvedValueOnce({ + items: [], + nextLink: undefined, + }); + (mockClient.getGroupMembers as jest.Mock).mockReturnValue(asyncYield()); + + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: 'default', + provider: providerWithFilter as any, + logger, + }); + + await provider.next( + { client: mockClient, provider: providerWithFilter as any }, + groupsCursor, + ); + + expect(mockRequestOnePage).toHaveBeenCalledWith( + mockClient, + 'groups', + expect.objectContaining({ + query: expect.objectContaining({ + filter: 'securityEnabled eq true', + search: '"displayName:Engineering"', + }), + }), + ); + }); + }); +}); diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts new file mode 100644 index 0000000000..1ad578b5e4 --- /dev/null +++ b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts @@ -0,0 +1,474 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import crypto from 'crypto'; +import { + ANNOTATION_LOCATION, + ANNOTATION_ORIGIN_LOCATION, + Entity, + stringifyEntityRef, +} from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { + IncrementalEntityProvider, + EntityIteratorResult, +} from '@backstage/plugin-catalog-backend-module-incremental-ingestion'; +import { DeferredEntity } from '@backstage/plugin-catalog-node'; +import limiterFactory from 'p-limit'; +import * as MicrosoftGraph from '@microsoft/microsoft-graph-types'; +import { + GroupTransformer, + MICROSOFT_GRAPH_GROUP_ID_ANNOTATION, + MICROSOFT_GRAPH_TENANT_ID_ANNOTATION, + MICROSOFT_GRAPH_USER_ID_ANNOTATION, + MicrosoftGraphClient, + MicrosoftGraphProviderConfig, + OrganizationTransformer, + ProviderConfigTransformer, + UserTransformer, + defaultGroupTransformer, + defaultOrganizationTransformer, + defaultUserTransformer, + readProviderConfigs, +} from '@backstage/plugin-catalog-backend-module-msgraph'; +import { LoggerService } from '@backstage/backend-plugin-api'; +import { getUserPhotoGated, requestOnePage } from './clientHelpers'; + +const PAGE_SIZE = 999; + +/** + * Backstage entity names must be ≤63 chars ([a-zA-Z0-9] separated by [-_.]). + * When MS Graph UPNs exceed that (e.g. calendar/booking accounts), we truncate + * to 54 chars and append an 8-char SHA-1 hash to preserve uniqueness. + */ +function capEntityName(name: string): string { + if (name.length <= 63) return name; + const hash = crypto.createHash('sha1').update(name).digest('hex').slice(0, 8); + return `${name.slice(0, 54)}_${hash}`; +} + +/** Stamps `annotations.backstage.io/location` on an entity using the MS Graph UID. */ +function withLocations(providerId: string, entity: Entity): Entity { + const uid = + entity.metadata.annotations?.[MICROSOFT_GRAPH_USER_ID_ANNOTATION] || + entity.metadata.annotations?.[MICROSOFT_GRAPH_GROUP_ID_ANNOTATION] || + entity.metadata.annotations?.[MICROSOFT_GRAPH_TENANT_ID_ANNOTATION] || + entity.metadata.name; + const location = `msgraph:${providerId}/${encodeURIComponent(uid)}`; + return { + ...entity, + metadata: { + ...entity.metadata, + annotations: { + ...entity.metadata.annotations, + [ANNOTATION_LOCATION]: location, + [ANNOTATION_ORIGIN_LOCATION]: location, + }, + }, + }; +} + +/** + * Pagination cursor used by {@link MicrosoftGraphIncrementalEntityProvider}. + * + * The `nextLink` field holds the `@odata.nextLink` URL returned by the + * Microsoft Graph API, which encodes all state needed to resume a paged + * request. A `null` value means the current phase is starting fresh. + * + * @public + */ +export type MSGraphCursor = { + phase: 'users' | 'groups'; + nextLink: string | null; +}; + +/** + * Context passed to each burst of {@link MicrosoftGraphIncrementalEntityProvider}. + * + * @public + */ +export type MSGraphContext = { + client: MicrosoftGraphClient; + provider: MicrosoftGraphProviderConfig; +}; + +/** + * Options for {@link MicrosoftGraphIncrementalEntityProvider}. + * + * @public + */ +export interface MicrosoftGraphIncrementalEntityProviderOptions { + /** + * The logger to use. + */ + logger: LoggerService; + + /** + * The function that transforms a user entry in msgraph to an entity. + * Optionally, you can pass separate transformers per provider ID. + */ + userTransformer?: UserTransformer | Record; + + /** + * The function that transforms a group entry in msgraph to an entity. + * Optionally, you can pass separate transformers per provider ID. + */ + groupTransformer?: GroupTransformer | Record; + + /** + * The function that transforms an organization entry in msgraph to an entity. + * Optionally, you can pass separate transformers per provider ID. + */ + organizationTransformer?: + | OrganizationTransformer + | Record; + + /** + * The function that transforms provider config dynamically before each sync. + * Optionally, you can pass separate transformers per provider ID. + */ + providerConfigTransformer?: + | ProviderConfigTransformer + | Record; +} + +/** + * Incrementally reads user and group entries out of Microsoft Graph, one page + * at a time, and provides them as User and Group entities for the catalog. + * + * Unlike `MicrosoftGraphOrgEntityProvider`, this provider never holds the full + * dataset in memory at once. Each burst processes a single page of users or + * groups (up to 999 items). This makes it suitable for very large tenants and + * avoids the memory pressure and long-running task issues of the full-scan + * provider. + * + * The Microsoft Graph `@odata.nextLink` URL is stored as the cursor, so a pod + * restart during ingestion resumes from the last completed page. + * + * Group membership (`spec.members`) is resolved inline during the groups phase + * by fetching the direct members of each group. The catalog's built-in relation + * stitching derives `spec.memberOf` on users from these group membership lists. + * + * @remarks + * `userGroupMemberFilter`, `userGroupMemberSearch`, and `userGroupMemberPath` + * are not supported. Use `userFilter` / `userPath` to restrict which users are + * ingested, and `groupFilter` / `groupSearch` to restrict which groups. + * + * @public + */ +export class MicrosoftGraphIncrementalEntityProvider + implements IncrementalEntityProvider +{ + /** + * Create one provider instance per provider entry in + * `catalog.providers.microsoftGraphOrg`. + */ + static fromConfig( + configRoot: Config, + options: MicrosoftGraphIncrementalEntityProviderOptions, + ): MicrosoftGraphIncrementalEntityProvider[] { + function getTransformer( + id: string, + transformers?: T | Record, + ): T | undefined { + if (['undefined', 'function'].includes(typeof transformers)) { + return transformers as T; + } + return (transformers as Record)[id]; + } + + return readProviderConfigs(configRoot).map( + providerConfig => + new MicrosoftGraphIncrementalEntityProvider({ + id: providerConfig.id, + provider: providerConfig, + logger: options.logger, + userTransformer: getTransformer( + providerConfig.id, + options.userTransformer, + ), + groupTransformer: getTransformer( + providerConfig.id, + options.groupTransformer, + ), + organizationTransformer: getTransformer( + providerConfig.id, + options.organizationTransformer, + ), + providerConfigTransformer: getTransformer( + providerConfig.id, + options.providerConfigTransformer, + ), + }), + ); + } + + private readonly options: { + id: string; + provider: MicrosoftGraphProviderConfig; + logger: LoggerService; + userTransformer?: UserTransformer; + groupTransformer?: GroupTransformer; + organizationTransformer?: OrganizationTransformer; + providerConfigTransformer?: ProviderConfigTransformer; + }; + + constructor(options: { + id: string; + provider: MicrosoftGraphProviderConfig; + logger: LoggerService; + userTransformer?: UserTransformer; + groupTransformer?: GroupTransformer; + organizationTransformer?: OrganizationTransformer; + providerConfigTransformer?: ProviderConfigTransformer; + }) { + this.options = options; + } + + /** {@inheritdoc @backstage/plugin-catalog-backend-module-incremental-ingestion#IncrementalEntityProvider.getProviderName} */ + getProviderName(): string { + return `MicrosoftGraphIncrementalEntityProvider:${this.options.id}`; + } + + /** + * Sets up the Microsoft Graph client for the duration of a full ingestion + * cycle. The optional `providerConfigTransformer` is applied here so that + * dynamic config changes (e.g., rotating credentials) take effect at the + * start of each cycle rather than mid-way through. + */ + async around( + burst: (context: MSGraphContext) => Promise, + ): Promise { + const provider = this.options.providerConfigTransformer + ? await this.options.providerConfigTransformer(this.options.provider) + : this.options.provider; + + if ( + provider.userGroupMemberFilter || + provider.userGroupMemberSearch || + provider.userGroupMemberPath + ) { + this.options.logger.warn( + `${this.getProviderName()}: userGroupMemberFilter/Search/Path are not supported by ` + + `MicrosoftGraphIncrementalEntityProvider. Users will be fetched via the standard ` + + `userFilter/userPath options instead. Switch to MicrosoftGraphOrgEntityProvider if ` + + `you require userGroupMember-based ingestion.`, + ); + } + + const client = MicrosoftGraphClient.create(provider); + await burst({ client, provider }); + } + + /** {@inheritdoc @backstage/plugin-catalog-backend-module-incremental-ingestion#IncrementalEntityProvider.next} */ + async next( + { client, provider }: MSGraphContext, + cursor?: MSGraphCursor, + ): Promise> { + const phase = cursor?.phase ?? 'users'; + const nextLink = cursor?.nextLink ?? null; + + if (phase === 'users') { + return this.readUsersPage(client, provider, nextLink); + } + return this.readGroupsPage(client, provider, nextLink); + } + + private async readUsersPage( + client: MicrosoftGraphClient, + provider: MicrosoftGraphProviderConfig, + nextLink: string | null, + ): Promise> { + const { items: rawUsers, nextLink: newNextLink } = + await requestOnePage( + client, + provider.userPath ?? 'users', + { + query: { + filter: provider.userFilter, + expand: provider.userExpand, + select: provider.userSelect, + top: PAGE_SIZE, + }, + queryMode: provider.queryMode, + nextLink: nextLink ?? undefined, + }, + ); + + const transformer = this.options.userTransformer ?? defaultUserTransformer; + const limiter = limiterFactory(10); + const entities: DeferredEntity[] = []; + + await Promise.all( + rawUsers.map(user => + limiter(async () => { + let userPhoto: string | undefined; + if (provider.loadUserPhotos !== false) { + try { + userPhoto = await getUserPhotoGated(client, user.id!, 120); + } catch { + // Photo load failures are non-fatal + } + } + + const entity = await transformer(user, userPhoto); + if (entity) { + entity.metadata.name = capEntityName(entity.metadata.name); + entities.push({ + locationKey: `msgraph-org-provider:${this.options.id}`, + entity: withLocations(this.options.id, entity), + }); + } + }), + ), + ); + + this.options.logger.debug( + `${this.getProviderName()}: read ${entities.length} users`, + { phase: 'users', hasNextPage: !!newNextLink }, + ); + + if (newNextLink) { + return { + done: false, + entities, + cursor: { phase: 'users', nextLink: newNextLink }, + }; + } + + return { + done: false, + entities, + cursor: { phase: 'groups', nextLink: null }, + }; + } + + private async readGroupsPage( + client: MicrosoftGraphClient, + provider: MicrosoftGraphProviderConfig, + nextLink: string | null, + ): Promise> { + const { items: rawGroups, nextLink: newNextLink } = + await requestOnePage( + client, + provider.groupPath ?? 'groups', + { + query: { + filter: provider.groupFilter, + search: provider.groupSearch, + expand: provider.groupExpand, + select: provider.groupSelect, + top: PAGE_SIZE, + }, + queryMode: provider.queryMode, + nextLink: nextLink ?? undefined, + }, + ); + + const groupTransformer = + this.options.groupTransformer ?? defaultGroupTransformer; + const userTransformer = + this.options.userTransformer ?? defaultUserTransformer; + const limiter = limiterFactory(10); + const entities: DeferredEntity[] = []; + + // Emit the tenant root group on the very first groups page + if (!nextLink) { + try { + const organization = await client.getOrganization(provider.tenantId); + const orgTransformer = + this.options.organizationTransformer ?? + defaultOrganizationTransformer; + const rootGroup = await orgTransformer(organization); + if (rootGroup) { + entities.push({ + locationKey: `msgraph-org-provider:${this.options.id}`, + entity: withLocations(this.options.id, rootGroup), + }); + } + } catch (e) { + this.options.logger.warn( + `${this.getProviderName()}: failed to read organization root group`, + { error: e }, + ); + } + } + + await Promise.all( + rawGroups.map(group => + limiter(async () => { + const entity = await groupTransformer(group); + if (!entity) { + return; + } + entity.metadata.name = capEntityName(entity.metadata.name); + + const userRefs: string[] = []; + const childRefs: string[] = []; + + for await (const member of client.getGroupMembers(group.id!, { + top: PAGE_SIZE, + })) { + if (member['@odata.type'] === '#microsoft.graph.user') { + const userEntity = await userTransformer( + member as MicrosoftGraph.User, + ); + if (userEntity) { + userEntity.metadata.name = capEntityName( + userEntity.metadata.name, + ); + userRefs.push(stringifyEntityRef(userEntity)); + } + } else if (member['@odata.type'] === '#microsoft.graph.group') { + const childEntity = await groupTransformer( + member as MicrosoftGraph.Group, + ); + if (childEntity) { + childEntity.metadata.name = capEntityName( + childEntity.metadata.name, + ); + childRefs.push(stringifyEntityRef(childEntity)); + } + } + } + + entity.spec.members = userRefs; + entity.spec.children = childRefs; + + entities.push({ + locationKey: `msgraph-org-provider:${this.options.id}`, + entity: withLocations(this.options.id, entity), + }); + }), + ), + ); + + this.options.logger.debug( + `${this.getProviderName()}: read ${rawGroups.length} groups`, + { phase: 'groups', hasNextPage: !!newNextLink }, + ); + + if (newNextLink) { + return { + done: false, + entities, + cursor: { phase: 'groups', nextLink: newNextLink }, + }; + } + + return { done: true, entities }; + } +} diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/alpha.ts b/plugins/catalog-backend-module-msgraph-incremental/src/alpha.ts new file mode 100644 index 0000000000..cb99b69c6a --- /dev/null +++ b/plugins/catalog-backend-module-msgraph-incremental/src/alpha.ts @@ -0,0 +1,19 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Re-export everything from the main entry point while the package is in alpha. +export * from './index'; +export { default } from './index'; diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.test.ts b/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.test.ts new file mode 100644 index 0000000000..3b81b24099 --- /dev/null +++ b/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.test.ts @@ -0,0 +1,254 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { MicrosoftGraphClient } from '@backstage/plugin-catalog-backend-module-msgraph'; +import { getUserPhotoGated, requestOnePage } from './clientHelpers'; + +function makeResponse(status: number, body: unknown): Response { + return { + status, + json: () => Promise.resolve(body), + } as unknown as Response; +} + +describe('requestOnePage', () => { + const client = { + requestApi: jest.fn(), + requestRaw: jest.fn(), + } as unknown as jest.Mocked; + + afterEach(() => jest.resetAllMocks()); + + it('fetches a basic page using requestApi', async () => { + (client.requestApi as jest.Mock).mockResolvedValue( + makeResponse(200, { + value: [{ id: '1', displayName: 'Alice' }], + }), + ); + + const result = await requestOnePage<{ id: string }>(client, 'users', { + query: { top: 999 }, + }); + + expect(result.items).toHaveLength(1); + expect(result.items[0].id).toBe('1'); + expect(result.nextLink).toBeUndefined(); + expect(client.requestApi).toHaveBeenCalledWith( + 'users', + { top: 999 }, + {}, + undefined, + ); + }); + + it('adds ConsistencyLevel and $count for advanced mode with filter', async () => { + (client.requestApi as jest.Mock).mockResolvedValue( + makeResponse(200, { value: [] }), + ); + + await requestOnePage(client, 'users', { + query: { + filter: 'accountEnabled eq true and jobTitle ne null', + top: 999, + }, + queryMode: 'advanced', + }); + + expect(client.requestApi).toHaveBeenCalledWith( + 'users', + expect.objectContaining({ count: true }), + { ConsistencyLevel: 'eventual' }, + undefined, + ); + }); + + it('auto-promotes to advanced mode when $search is present', async () => { + (client.requestApi as jest.Mock).mockResolvedValue( + makeResponse(200, { value: [] }), + ); + + await requestOnePage(client, 'groups', { + query: { search: '"displayName:Sales"', top: 10 }, + }); + + expect(client.requestApi).toHaveBeenCalledWith( + 'groups', + expect.objectContaining({ count: true }), + { ConsistencyLevel: 'eventual' }, + undefined, + ); + }); + + it('does not set $count for basic mode', async () => { + (client.requestApi as jest.Mock).mockResolvedValue( + makeResponse(200, { value: [] }), + ); + + await requestOnePage(client, 'users', { + query: { filter: 'accountEnabled eq true', top: 999 }, + queryMode: 'basic', + }); + + expect(client.requestApi).toHaveBeenCalledWith( + 'users', + expect.not.objectContaining({ count: true }), + {}, + undefined, + ); + }); + + it('follows nextLink using requestRaw', async () => { + const nextLink = + 'https://graph.microsoft.com/v1.0/users?$skiptoken=abc123&$count=true'; + (client.requestRaw as jest.Mock).mockResolvedValue( + makeResponse(200, { + value: [{ id: '2' }], + '@odata.nextLink': + 'https://graph.microsoft.com/v1.0/users?$skiptoken=def456', + }), + ); + + const result = await requestOnePage(client, 'users', { nextLink }); + + expect(client.requestRaw).toHaveBeenCalledWith(nextLink, {}, 2, undefined); + expect(client.requestApi).not.toHaveBeenCalled(); + expect(result.items).toHaveLength(1); + expect(result.nextLink).toBe( + 'https://graph.microsoft.com/v1.0/users?$skiptoken=def456', + ); + }); + + it('passes ConsistencyLevel header when following nextLink in advanced mode', async () => { + const nextLink = 'https://graph.microsoft.com/v1.0/groups?$skiptoken=xyz'; + (client.requestRaw as jest.Mock).mockResolvedValue( + makeResponse(200, { value: [] }), + ); + + await requestOnePage(client, 'groups', { + query: { filter: 'securityEnabled eq true', top: 999 }, + queryMode: 'advanced', + nextLink, + }); + + expect(client.requestRaw).toHaveBeenCalledWith( + nextLink, + { ConsistencyLevel: 'eventual' }, + 2, + undefined, + ); + }); + + it('returns nextLink from response when present', async () => { + const expectedNextLink = + 'https://graph.microsoft.com/v1.0/users?$skiptoken=page2'; + (client.requestApi as jest.Mock).mockResolvedValue( + makeResponse(200, { + value: Array(999).fill({ id: 'x' }), + '@odata.nextLink': expectedNextLink, + }), + ); + + const result = await requestOnePage(client, 'users', {}); + + expect(result.nextLink).toBe(expectedNextLink); + expect(result.items).toHaveLength(999); + }); + + it('throws a descriptive error on non-200 response', async () => { + (client.requestApi as jest.Mock).mockResolvedValue( + makeResponse(403, { + error: { + code: 'Authorization_RequestDenied', + message: 'Access denied', + }, + }), + ); + + await expect(requestOnePage(client, 'users', {})).rejects.toThrow( + 'Authorization_RequestDenied - Access denied', + ); + }); + + it('passes AbortSignal through to the client', async () => { + (client.requestApi as jest.Mock).mockResolvedValue( + makeResponse(200, { value: [] }), + ); + const signal = new AbortController().signal; + + await requestOnePage(client, 'users', { signal }); + + expect(client.requestApi).toHaveBeenCalledWith( + 'users', + undefined, + {}, + signal, + ); + }); +}); + +describe('getUserPhotoGated', () => { + const client = { + requestApi: jest.fn(), + getUserPhotoWithSizeLimit: jest.fn(), + } as unknown as jest.Mocked; + + afterEach(() => jest.resetAllMocks()); + + it('returns undefined when the photo check returns non-200', async () => { + (client.requestApi as jest.Mock).mockResolvedValue({ + status: 404, + } as Response); + + const result = await getUserPhotoGated(client, 'user-id', 120); + + expect(result).toBeUndefined(); + expect(client.getUserPhotoWithSizeLimit).not.toHaveBeenCalled(); + }); + + it('returns the photo data URI when the check passes', async () => { + (client.requestApi as jest.Mock).mockResolvedValue({ + status: 200, + } as Response); + (client.getUserPhotoWithSizeLimit as jest.Mock).mockResolvedValue( + 'data:image/jpeg;base64,/9j/abc123', + ); + + const result = await getUserPhotoGated(client, 'user-id', 120); + + expect(result).toBe('data:image/jpeg;base64,/9j/abc123'); + expect(client.requestApi).toHaveBeenCalledWith('users/user-id/photo'); + expect(client.getUserPhotoWithSizeLimit).toHaveBeenCalledWith( + 'user-id', + 120, + ); + }); + + it('passes the maxSize limit to getUserPhotoWithSizeLimit', async () => { + (client.requestApi as jest.Mock).mockResolvedValue({ + status: 200, + } as Response); + (client.getUserPhotoWithSizeLimit as jest.Mock).mockResolvedValue( + undefined, + ); + + await getUserPhotoGated(client, 'user-abc', 48); + + expect(client.getUserPhotoWithSizeLimit).toHaveBeenCalledWith( + 'user-abc', + 48, + ); + }); +}); diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.ts b/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.ts new file mode 100644 index 0000000000..751a75059b --- /dev/null +++ b/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.ts @@ -0,0 +1,91 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + MicrosoftGraphClient, + ODataQuery, +} from '@backstage/plugin-catalog-backend-module-msgraph'; + +/** + * Fetches a single page of Graph API results. + * + * When `options.nextLink` is provided it is followed directly (all query + * parameters are already encoded in it). Otherwise the request is built from + * `options.query`. + * + * MS Graph requires `ConsistencyLevel: eventual` + `$count=true` for advanced + * queries using `ne`/`not` operators in `$filter` or using `$search`. + */ +export async function requestOnePage( + client: MicrosoftGraphClient, + path: string, + options: { + query?: ODataQuery; + queryMode?: 'basic' | 'advanced'; + nextLink?: string; + signal?: AbortSignal; + } = {}, +): Promise<{ items: T[]; nextLink?: string }> { + const { query, queryMode, nextLink, signal } = options; + const appliedQueryMode = query?.search ? 'advanced' : queryMode ?? 'basic'; + + // $count=true is required for advanced queries using ne/not in $filter or $search + if ( + appliedQueryMode === 'advanced' && + query && + (query.filter || query.search) + ) { + query.count = true; + } + + const headers: Record = + appliedQueryMode === 'advanced' ? { ConsistencyLevel: 'eventual' } : {}; + + const response = nextLink + ? await client.requestRaw(nextLink, headers, 2, signal) + : await client.requestApi(path, query, headers, signal); + + if (response.status !== 200) { + const body = await response.json(); + const err = body?.error; + throw new Error( + `Error while reading ${nextLink ?? path} from Microsoft Graph: ${ + err?.code + } - ${err?.message}`, + ); + } + + const result = await response.json(); + return { + items: result.value as T[], + nextLink: result['@odata.nextLink'], + }; +} + +/** + * Like `getUserPhotoWithSizeLimit` but skips the size-listing call for users + * with no photo. For users without a photo: 1 fast check call. For users with + * a photo: 1 check + the normal size-limited fetch (2 more calls). + */ +export async function getUserPhotoGated( + client: MicrosoftGraphClient, + userId: string, + maxSize: number, +): Promise { + const check = await client.requestApi(`users/${userId}/photo`); + if (check.status !== 200) return undefined; + return await client.getUserPhotoWithSizeLimit(userId, maxSize); +} diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/index.ts b/plugins/catalog-backend-module-msgraph-incremental/src/index.ts new file mode 100644 index 0000000000..dbae9a4f2c --- /dev/null +++ b/plugins/catalog-backend-module-msgraph-incremental/src/index.ts @@ -0,0 +1,31 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * A Backstage catalog backend module that incrementally ingests users and + * groups from Microsoft Graph, one page at a time. + * + * @packageDocumentation + */ + +export { catalogModuleMicrosoftGraphIncrementalEntityProvider as default } from './module'; +export * from './module'; +export { MicrosoftGraphIncrementalEntityProvider } from './MicrosoftGraphIncrementalEntityProvider'; +export type { + MSGraphCursor, + MSGraphContext, + MicrosoftGraphIncrementalEntityProviderOptions, +} from './MicrosoftGraphIncrementalEntityProvider'; diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.test.ts b/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.test.ts new file mode 100644 index 0000000000..a30388bae1 --- /dev/null +++ b/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.test.ts @@ -0,0 +1,134 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { mockServices, startTestBackend } from '@backstage/backend-test-utils'; +import { incrementalIngestionProvidersExtensionPoint } from '@backstage/plugin-catalog-backend-module-incremental-ingestion'; +import { catalogModuleMicrosoftGraphIncrementalEntityProvider } from './catalogModuleMicrosoftGraphIncrementalEntityProvider'; +import { MicrosoftGraphIncrementalEntityProvider } from '../MicrosoftGraphIncrementalEntityProvider'; + +describe('catalogModuleMicrosoftGraphIncrementalEntityProvider', () => { + it('registers the provider at the incremental ingestion extension point', async () => { + const addProvider = jest.fn(); + const extensionPoint = { addProvider }; + + await startTestBackend({ + extensionPoints: [ + [incrementalIngestionProvidersExtensionPoint, extensionPoint], + ], + features: [ + catalogModuleMicrosoftGraphIncrementalEntityProvider, + mockServices.rootConfig.factory({ + data: { + catalog: { + providers: { + microsoftGraphOrg: { + default: { + tenantId: 'tenant-id', + clientId: 'client-id', + clientSecret: 'client-secret', + schedule: { + frequency: 'PT12H', + timeout: 'PT4H', + }, + }, + }, + }, + }, + }, + }), + ], + }); + + expect(addProvider).toHaveBeenCalledTimes(1); + const { provider, options } = addProvider.mock.calls[0][0]; + expect(provider).toBeInstanceOf(MicrosoftGraphIncrementalEntityProvider); + expect(provider.getProviderName()).toBe( + 'MicrosoftGraphIncrementalEntityProvider:default', + ); + expect(options.burstInterval).toEqual({ seconds: 3 }); + expect(options.burstLength).toEqual({ minutes: 5 }); + // restLength derived from schedule.frequency (12h) + expect(options.restLength).toEqual({ hours: 12 }); + }); + + it('creates one provider per config entry', async () => { + const addProvider = jest.fn(); + const extensionPoint = { addProvider }; + + await startTestBackend({ + extensionPoints: [ + [incrementalIngestionProvidersExtensionPoint, extensionPoint], + ], + features: [ + catalogModuleMicrosoftGraphIncrementalEntityProvider, + mockServices.rootConfig.factory({ + data: { + catalog: { + providers: { + microsoftGraphOrg: { + tenantA: { + tenantId: 'a', + clientId: 'c', + clientSecret: 's', + }, + tenantB: { + tenantId: 'b', + clientId: 'c', + clientSecret: 's', + }, + }, + }, + }, + }, + }), + ], + }); + + expect(addProvider).toHaveBeenCalledTimes(2); + }); + + it('defaults restLength to 8 hours when no schedule frequency is configured', async () => { + const addProvider = jest.fn(); + const extensionPoint = { addProvider }; + + await startTestBackend({ + extensionPoints: [ + [incrementalIngestionProvidersExtensionPoint, extensionPoint], + ], + features: [ + catalogModuleMicrosoftGraphIncrementalEntityProvider, + mockServices.rootConfig.factory({ + data: { + catalog: { + providers: { + microsoftGraphOrg: { + default: { + tenantId: 'tenant-id', + clientId: 'client-id', + clientSecret: 'client-secret', + }, + }, + }, + }, + }, + }), + ], + }); + + const { options } = addProvider.mock.calls[0][0]; + expect(options.restLength).toEqual({ hours: 8 }); + }); +}); diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.ts b/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.ts new file mode 100644 index 0000000000..fd30762d5f --- /dev/null +++ b/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.ts @@ -0,0 +1,235 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + coreServices, + createBackendModule, + createExtensionPoint, +} from '@backstage/backend-plugin-api'; +import { incrementalIngestionProvidersExtensionPoint } from '@backstage/plugin-catalog-backend-module-incremental-ingestion'; +import { + GroupTransformer, + OrganizationTransformer, + ProviderConfigTransformer, + UserTransformer, + readProviderConfigs, +} from '@backstage/plugin-catalog-backend-module-msgraph'; +import { HumanDuration } from '@backstage/types'; +import { + MicrosoftGraphIncrementalEntityProvider, + MSGraphCursor, +} from '../MicrosoftGraphIncrementalEntityProvider'; + +/** + * Interface for + * {@link microsoftGraphIncrementalEntityProviderTransformExtensionPoint}. + * + * @public + */ +export interface MicrosoftGraphIncrementalEntityProviderTransformsExtensionPoint { + /** + * Set the function that transforms a user entry in msgraph to an entity. + * Optionally, you can pass separate transformers per provider ID. + */ + setUserTransformer( + transformer: UserTransformer | Record, + ): void; + + /** + * Set the function that transforms a group entry in msgraph to an entity. + * Optionally, you can pass separate transformers per provider ID. + */ + setGroupTransformer( + transformer: GroupTransformer | Record, + ): void; + + /** + * Set the function that transforms an organization entry in msgraph to an + * entity. Optionally, you can pass separate transformers per provider ID. + */ + setOrganizationTransformer( + transformer: + | OrganizationTransformer + | Record, + ): void; + + /** + * Set the function that transforms provider config dynamically. + * Optionally, you can pass separate transformers per provider ID. + */ + setProviderConfigTransformer( + transformer: + | ProviderConfigTransformer + | Record, + ): void; +} + +/** + * Extension point used to customize the transforms applied by the incremental + * module. + * + * @public + */ +export const microsoftGraphIncrementalEntityProviderTransformExtensionPoint = + createExtensionPoint( + { + id: 'catalog.microsoftGraphIncrementalEntityProvider.transforms', + }, + ); + +/** + * Registers {@link MicrosoftGraphIncrementalEntityProvider} instances with the + * catalog's incremental ingestion extension point. + * + * This module requires `catalogModuleIncrementalIngestionEntityProvider` to + * also be installed in the backend. + * + * @example + * ```ts + * // packages/backend/src/index.ts + * backend.add(import('@backstage/plugin-catalog-backend-module-incremental-ingestion')); + * backend.add(import('@backstage/plugin-catalog-backend-module-msgraph-incremental')); + * ``` + * + * @public + */ +export const catalogModuleMicrosoftGraphIncrementalEntityProvider = + createBackendModule({ + pluginId: 'catalog', + moduleId: 'microsoftGraphIncrementalEntityProvider', + register(env) { + let userTransformer: + | UserTransformer + | Record + | undefined; + let groupTransformer: + | GroupTransformer + | Record + | undefined; + let organizationTransformer: + | OrganizationTransformer + | Record + | undefined; + let providerConfigTransformer: + | ProviderConfigTransformer + | Record + | undefined; + + env.registerExtensionPoint( + microsoftGraphIncrementalEntityProviderTransformExtensionPoint, + { + setUserTransformer(transformer) { + if (userTransformer) { + throw new Error('User transformer may only be set once'); + } + userTransformer = transformer; + }, + setGroupTransformer(transformer) { + if (groupTransformer) { + throw new Error('Group transformer may only be set once'); + } + groupTransformer = transformer; + }, + setOrganizationTransformer(transformer) { + if (organizationTransformer) { + throw new Error('Organization transformer may only be set once'); + } + organizationTransformer = transformer; + }, + setProviderConfigTransformer(transformer) { + if (providerConfigTransformer) { + throw new Error( + 'Provider config transformer may only be set once', + ); + } + providerConfigTransformer = transformer; + }, + }, + ); + + env.registerInit({ + deps: { + config: coreServices.rootConfig, + logger: coreServices.logger, + incremental: incrementalIngestionProvidersExtensionPoint, + }, + async init({ config, logger, incremental }) { + const providerConfigs = readProviderConfigs(config); + + for (const providerConfig of providerConfigs) { + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: providerConfig.id, + provider: providerConfig, + logger, + userTransformer: resolveTransformer( + providerConfig.id, + userTransformer, + ), + groupTransformer: resolveTransformer( + providerConfig.id, + groupTransformer, + ), + organizationTransformer: resolveTransformer( + providerConfig.id, + organizationTransformer, + ), + providerConfigTransformer: resolveTransformer( + providerConfig.id, + providerConfigTransformer, + ), + }); + + const restLength = deriveRestLength(providerConfig); + + incremental.addProvider({ + provider, + options: { + burstInterval: { seconds: 3 }, + burstLength: { minutes: 5 }, + restLength, + backoff: [ + { seconds: 30 }, + { minutes: 3 }, + { minutes: 30 }, + { hours: 3 }, + ], + }, + }); + } + }, + }); + }, + }); + +function resolveTransformer( + id: string, + transformer?: T | Record, +): T | undefined { + if (['undefined', 'function'].includes(typeof transformer)) { + return transformer as T; + } + return (transformer as Record)[id]; +} + +function deriveRestLength( + providerConfig: ReturnType[number], +): HumanDuration { + const freq = providerConfig.schedule?.frequency; + if (freq && typeof freq === 'object' && !('cron' in freq)) { + return freq as HumanDuration; + } + return { hours: 8 }; +} diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/module/index.ts b/plugins/catalog-backend-module-msgraph-incremental/src/module/index.ts new file mode 100644 index 0000000000..5d5e3710a4 --- /dev/null +++ b/plugins/catalog-backend-module-msgraph-incremental/src/module/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { + catalogModuleMicrosoftGraphIncrementalEntityProvider, + microsoftGraphIncrementalEntityProviderTransformExtensionPoint, + type MicrosoftGraphIncrementalEntityProviderTransformsExtensionPoint, +} from './catalogModuleMicrosoftGraphIncrementalEntityProvider'; diff --git a/yarn.lock b/yarn.lock index 68ba5bc315..1423c77651 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5016,7 +5016,7 @@ __metadata: languageName: unknown linkType: soft -"@backstage/plugin-catalog-backend-module-incremental-ingestion@workspace:plugins/catalog-backend-module-incremental-ingestion": +"@backstage/plugin-catalog-backend-module-incremental-ingestion@workspace:^, @backstage/plugin-catalog-backend-module-incremental-ingestion@workspace:plugins/catalog-backend-module-incremental-ingestion": version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-backend-module-incremental-ingestion@workspace:plugins/catalog-backend-module-incremental-ingestion" dependencies: @@ -5072,7 +5072,24 @@ __metadata: languageName: unknown linkType: soft -"@backstage/plugin-catalog-backend-module-msgraph@workspace:plugins/catalog-backend-module-msgraph": +"@backstage/plugin-catalog-backend-module-msgraph-incremental@workspace:plugins/catalog-backend-module-msgraph-incremental": + version: 0.0.0-use.local + resolution: "@backstage/plugin-catalog-backend-module-msgraph-incremental@workspace:plugins/catalog-backend-module-msgraph-incremental" + dependencies: + "@backstage/backend-plugin-api": "workspace:^" + "@backstage/catalog-model": "workspace:^" + "@backstage/cli": "workspace:^" + "@backstage/config": "workspace:^" + "@backstage/plugin-catalog-backend-module-incremental-ingestion": "workspace:^" + "@backstage/plugin-catalog-backend-module-msgraph": "workspace:^" + "@backstage/plugin-catalog-node": "workspace:^" + "@backstage/types": "workspace:^" + "@microsoft/microsoft-graph-types": "npm:^2.6.0" + p-limit: "npm:^3.0.2" + languageName: unknown + linkType: soft + +"@backstage/plugin-catalog-backend-module-msgraph@workspace:^, @backstage/plugin-catalog-backend-module-msgraph@workspace:plugins/catalog-backend-module-msgraph": version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-backend-module-msgraph@workspace:plugins/catalog-backend-module-msgraph" dependencies: @@ -29085,7 +29102,9 @@ __metadata: "@backstage/plugin-auth-node": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" "@backstage/plugin-catalog-backend-module-backstage-openapi": "workspace:^" + "@backstage/plugin-catalog-backend-module-incremental-ingestion": "workspace:^" "@backstage/plugin-catalog-backend-module-logs": "workspace:^" + "@backstage/plugin-catalog-backend-module-msgraph": "workspace:^" "@backstage/plugin-catalog-backend-module-openapi": "workspace:^" "@backstage/plugin-catalog-backend-module-scaffolder-entity-model": "workspace:^" "@backstage/plugin-catalog-backend-module-unprocessed": "workspace:^" @@ -29115,6 +29134,7 @@ __metadata: "@opentelemetry/exporter-prometheus": "npm:^0.213.0" "@opentelemetry/sdk-node": "npm:^0.213.0" example-app: "link:../app" + pg: "npm:^8.11.0" languageName: unknown linkType: soft @@ -39927,7 +39947,7 @@ __metadata: languageName: node linkType: hard -"pg@npm:^8.11.3, pg@npm:^8.7.3, pg@npm:^8.9.0": +"pg@npm:^8.11.0, pg@npm:^8.11.3, pg@npm:^8.7.3, pg@npm:^8.9.0": version: 8.20.0 resolution: "pg@npm:8.20.0" dependencies: From 70adee104e2de3bd866d235e3ef04efce53ef12a Mon Sep 17 00:00:00 2001 From: pillaris Date: Fri, 24 Apr 2026 12:21:00 +0530 Subject: [PATCH 02/20] fix(catalog): always pair \$count=true with ConsistencyLevel: eventual in advanced mode Microsoft Graph requires \$count=true whenever the ConsistencyLevel: eventual header is present, including plain listing requests with no \$filter or \$search. The previous condition only added \$count when a filter or search was present, causing the /groups endpoint to silently return an empty value array when queryMode is set to advanced without a group filter. Signed-off-by: pillaris Co-Authored-By: Claude Sonnet 4.6 --- .../src/clientHelpers.test.ts | 18 ++++++++++++++++++ .../src/clientHelpers.ts | 9 +++------ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.test.ts b/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.test.ts index 3b81b24099..1819492be8 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.test.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.test.ts @@ -75,6 +75,24 @@ describe('requestOnePage', () => { ); }); + it('adds ConsistencyLevel and $count for advanced mode without filter', async () => { + (client.requestApi as jest.Mock).mockResolvedValue( + makeResponse(200, { value: [] }), + ); + + await requestOnePage(client, 'groups', { + query: { top: 999 }, + queryMode: 'advanced', + }); + + expect(client.requestApi).toHaveBeenCalledWith( + 'groups', + expect.objectContaining({ count: true }), + { ConsistencyLevel: 'eventual' }, + undefined, + ); + }); + it('auto-promotes to advanced mode when $search is present', async () => { (client.requestApi as jest.Mock).mockResolvedValue( makeResponse(200, { value: [] }), diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.ts b/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.ts index 751a75059b..0c3f005f0e 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.ts @@ -42,12 +42,9 @@ export async function requestOnePage( const { query, queryMode, nextLink, signal } = options; const appliedQueryMode = query?.search ? 'advanced' : queryMode ?? 'basic'; - // $count=true is required for advanced queries using ne/not in $filter or $search - if ( - appliedQueryMode === 'advanced' && - query && - (query.filter || query.search) - ) { + // Microsoft Graph requires $count=true whenever ConsistencyLevel: eventual is set, + // including plain listing requests with no $filter or $search. + if (appliedQueryMode === 'advanced' && query) { query.count = true; } From 5126adb17ae480753709235579d085ad9c13c526 Mon Sep 17 00:00:00 2001 From: pillaris Date: Sat, 25 Apr 2026 09:56:42 +0530 Subject: [PATCH 03/20] chore: update yarn.lock for msgraph-incremental workspace entry Add workspace:^ alias for the new plugin so the lockfile is consistent with the packages/backend dependency reference. Remove stale pg@^8.11.0 range no longer referenced by any workspace package. Signed-off-by: pillaris --- yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn.lock b/yarn.lock index 1423c77651..99a37480bb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5072,7 +5072,7 @@ __metadata: languageName: unknown linkType: soft -"@backstage/plugin-catalog-backend-module-msgraph-incremental@workspace:plugins/catalog-backend-module-msgraph-incremental": +"@backstage/plugin-catalog-backend-module-msgraph-incremental@workspace:^, @backstage/plugin-catalog-backend-module-msgraph-incremental@workspace:plugins/catalog-backend-module-msgraph-incremental": version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-backend-module-msgraph-incremental@workspace:plugins/catalog-backend-module-msgraph-incremental" dependencies: @@ -29105,6 +29105,7 @@ __metadata: "@backstage/plugin-catalog-backend-module-incremental-ingestion": "workspace:^" "@backstage/plugin-catalog-backend-module-logs": "workspace:^" "@backstage/plugin-catalog-backend-module-msgraph": "workspace:^" + "@backstage/plugin-catalog-backend-module-msgraph-incremental": "workspace:^" "@backstage/plugin-catalog-backend-module-openapi": "workspace:^" "@backstage/plugin-catalog-backend-module-scaffolder-entity-model": "workspace:^" "@backstage/plugin-catalog-backend-module-unprocessed": "workspace:^" @@ -29134,7 +29135,6 @@ __metadata: "@opentelemetry/exporter-prometheus": "npm:^0.213.0" "@opentelemetry/sdk-node": "npm:^0.213.0" example-app: "link:../app" - pg: "npm:^8.11.0" languageName: unknown linkType: soft @@ -39947,7 +39947,7 @@ __metadata: languageName: node linkType: hard -"pg@npm:^8.11.0, pg@npm:^8.11.3, pg@npm:^8.7.3, pg@npm:^8.9.0": +"pg@npm:^8.11.3, pg@npm:^8.7.3, pg@npm:^8.9.0": version: 8.20.0 resolution: "pg@npm:8.20.0" dependencies: From 02859a1db9143dcee8b416ea889617a227509c46 Mon Sep 17 00:00:00 2001 From: pillaris Date: Sat, 25 Apr 2026 10:13:23 +0530 Subject: [PATCH 04/20] fix(catalog): fix lint and type errors in msgraph-incremental plugin - Use node:crypto import protocol - Add @backstage/backend-test-utils to devDependencies - Use template literal instead of string concatenation - Add non-null assertions on result.entities in tests - Remove unused rootGroup variable Signed-off-by: pillaris --- .../package.json | 1 + ...softGraphIncrementalEntityProvider.test.ts | 32 ++++++++----------- ...MicrosoftGraphIncrementalEntityProvider.ts | 2 +- yarn.lock | 1 + 4 files changed, 16 insertions(+), 20 deletions(-) diff --git a/plugins/catalog-backend-module-msgraph-incremental/package.json b/plugins/catalog-backend-module-msgraph-incremental/package.json index e04edf0c88..e84a57f403 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/package.json +++ b/plugins/catalog-backend-module-msgraph-incremental/package.json @@ -61,6 +61,7 @@ "p-limit": "^3.0.2" }, "devDependencies": { + "@backstage/backend-test-utils": "workspace:^", "@backstage/cli": "workspace:^" } } diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts index 65f998b1ad..afee8bc0ec 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts @@ -262,9 +262,9 @@ describe('MicrosoftGraphIncrementalEntityProvider', () => { const result = await provider.next(makeContext()); expect(result.done).toBe(false); - expect(result.entities).toHaveLength(1); + expect(result.entities!).toHaveLength(1); - const entity = result.entities[0].entity; + const entity = result.entities![0].entity; expect(entity.kind).toBe('User'); expect( entity.metadata.annotations?.[MICROSOFT_GRAPH_USER_ID_ANNOTATION], @@ -275,7 +275,7 @@ describe('MicrosoftGraphIncrementalEntityProvider', () => { expect(entity.metadata.annotations?.[ANNOTATION_ORIGIN_LOCATION]).toBe( 'msgraph:default/user-id-1', ); - expect(result.entities[0].locationKey).toBe( + expect(result.entities![0].locationKey).toBe( 'msgraph-org-provider:default', ); }); @@ -339,11 +339,11 @@ describe('MicrosoftGraphIncrementalEntityProvider', () => { const result = await provider.next(makeContext()); - expect(result.entities).toHaveLength(0); + expect(result.entities!).toHaveLength(0); }); it('truncates entity names longer than 63 characters', async () => { - const longUPN = 'a'.repeat(64) + '@example.com'; + const longUPN = `${'a'.repeat(64)}@example.com`; mockRequestOnePage.mockResolvedValueOnce({ items: [{ id: 'u1', displayName: 'Test', userPrincipalName: longUPN }], nextLink: undefined, @@ -360,7 +360,7 @@ describe('MicrosoftGraphIncrementalEntityProvider', () => { // Entity may be emitted or skipped depending on transformer handling of the UPN; // what matters is that no name exceeds 63 characters. - for (const { entity } of result.entities) { + for (const { entity } of result.entities!) { expect(entity.metadata.name.length).toBeLessThanOrEqual(63); } }); @@ -445,7 +445,7 @@ describe('MicrosoftGraphIncrementalEntityProvider', () => { const result = await provider.next(makeContext()); // Both users should still be emitted despite the photo failure - expect(result.entities).toHaveLength(2); + expect(result.entities!).toHaveLength(2); }); }); @@ -471,14 +471,8 @@ describe('MicrosoftGraphIncrementalEntityProvider', () => { const result = await provider.next(makeContext(), groupsCursor); - const rootGroup = result.entities.find( - e => - e.entity.metadata.annotations?.[ - MICROSOFT_GRAPH_GROUP_ID_ANNOTATION - ] === undefined && e.entity.spec?.type === 'root', - ); // Root group is emitted (org entity kind=Group, type=root) - expect(result.entities.some(e => e.entity.spec?.type === 'root')).toBe( + expect(result.entities!.some(e => e.entity.spec?.type === 'root')).toBe( true, ); }); @@ -526,7 +520,7 @@ describe('MicrosoftGraphIncrementalEntityProvider', () => { const result = await provider.next(makeContext(), groupsCursor); - const groupEntity = result.entities.find( + const groupEntity = result.entities!.find( e => e.entity.metadata.annotations?.[ MICROSOFT_GRAPH_GROUP_ID_ANNOTATION @@ -578,7 +572,7 @@ describe('MicrosoftGraphIncrementalEntityProvider', () => { const result = await provider.next(makeContext(), groupsCursor); - const groupEntity = result.entities.find( + const groupEntity = result.entities!.find( e => e.entity.metadata.annotations?.[ MICROSOFT_GRAPH_GROUP_ID_ANNOTATION @@ -622,7 +616,7 @@ describe('MicrosoftGraphIncrementalEntityProvider', () => { const result = await provider.next(makeContext(), groupsCursor); - const parentEntity = result.entities.find( + const parentEntity = result.entities!.find( e => e.entity.metadata.annotations?.[ MICROSOFT_GRAPH_GROUP_ID_ANNOTATION @@ -702,7 +696,7 @@ describe('MicrosoftGraphIncrementalEntityProvider', () => { ); // The group itself is still emitted expect( - result.entities.some( + result.entities!.some( e => e.entity.metadata.annotations?.[ MICROSOFT_GRAPH_GROUP_ID_ANNOTATION @@ -732,7 +726,7 @@ describe('MicrosoftGraphIncrementalEntityProvider', () => { const result = await provider.next(makeContext(), groupsCursor); // Only the root group entity remains - expect(result.entities.every(e => e.entity.spec?.type === 'root')).toBe( + expect(result.entities!.every(e => e.entity.spec?.type === 'root')).toBe( true, ); }); diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts index 1ad578b5e4..246f53ab86 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import crypto from 'crypto'; +import crypto from 'node:crypto'; import { ANNOTATION_LOCATION, ANNOTATION_ORIGIN_LOCATION, diff --git a/yarn.lock b/yarn.lock index 99a37480bb..c9f8c05aac 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5077,6 +5077,7 @@ __metadata: resolution: "@backstage/plugin-catalog-backend-module-msgraph-incremental@workspace:plugins/catalog-backend-module-msgraph-incremental" dependencies: "@backstage/backend-plugin-api": "workspace:^" + "@backstage/backend-test-utils": "workspace:^" "@backstage/catalog-model": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" From 758bfaa3563a663acc5f1a442cbcf7e45b11516c Mon Sep 17 00:00:00 2001 From: pillaris Date: Sat, 25 Apr 2026 12:40:42 +0530 Subject: [PATCH 05/20] fix: address Copilot review comments on msgraph-incremental provider - Use undefined instead of null for MSGraphCursor.nextLink (repo style) - Avoid mutating caller-provided query object in requestOnePage - Improve error handling for non-JSON Graph API error responses - Build entity spec immutably instead of mutating entity.spec directly - Log debug message when a sparse group member cannot be transformed - Warn when schedule.frequency is a cron expression (restLength falls back to 8h) - Use proper MSGraphContext generic instead of unknown in addProvider Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: pillaris --- .../report.api.md | 2 +- ...softGraphIncrementalEntityProvider.test.ts | 4 +-- ...MicrosoftGraphIncrementalEntityProvider.ts | 30 +++++++++++-------- .../src/clientHelpers.ts | 27 +++++++++++------ ...MicrosoftGraphIncrementalEntityProvider.ts | 14 +++++++-- 5 files changed, 51 insertions(+), 26 deletions(-) diff --git a/plugins/catalog-backend-module-msgraph-incremental/report.api.md b/plugins/catalog-backend-module-msgraph-incremental/report.api.md index 9e01dd01f8..2f1e32717c 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/report.api.md +++ b/plugins/catalog-backend-module-msgraph-incremental/report.api.md @@ -91,6 +91,6 @@ export type MSGraphContext = { // @public export type MSGraphCursor = { phase: 'users' | 'groups'; - nextLink: string | null; + nextLink?: string; }; ``` diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts index afee8bc0ec..ffb7793dae 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts @@ -320,7 +320,7 @@ describe('MicrosoftGraphIncrementalEntityProvider', () => { expect(result.done).toBe(false); expect((result.cursor as MSGraphCursor).phase).toBe('groups'); - expect((result.cursor as MSGraphCursor).nextLink).toBeNull(); + expect((result.cursor as MSGraphCursor).nextLink).toBeUndefined(); }); it('skips users where the transformer returns undefined', async () => { @@ -450,7 +450,7 @@ describe('MicrosoftGraphIncrementalEntityProvider', () => { }); describe('next — groups phase', () => { - const groupsCursor: MSGraphCursor = { phase: 'groups', nextLink: null }; + const groupsCursor: MSGraphCursor = { phase: 'groups' }; it('emits the tenant root group on the first groups page', async () => { (mockClient.getOrganization as jest.Mock).mockResolvedValue({ diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts index 246f53ab86..de7f57d1f5 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts @@ -86,13 +86,13 @@ function withLocations(providerId: string, entity: Entity): Entity { * * The `nextLink` field holds the `@odata.nextLink` URL returned by the * Microsoft Graph API, which encodes all state needed to resume a paged - * request. A `null` value means the current phase is starting fresh. + * request. An absent value means the current phase is starting fresh. * * @public */ export type MSGraphCursor = { phase: 'users' | 'groups'; - nextLink: string | null; + nextLink?: string; }; /** @@ -279,7 +279,7 @@ export class MicrosoftGraphIncrementalEntityProvider cursor?: MSGraphCursor, ): Promise> { const phase = cursor?.phase ?? 'users'; - const nextLink = cursor?.nextLink ?? null; + const nextLink = cursor?.nextLink; if (phase === 'users') { return this.readUsersPage(client, provider, nextLink); @@ -290,7 +290,7 @@ export class MicrosoftGraphIncrementalEntityProvider private async readUsersPage( client: MicrosoftGraphClient, provider: MicrosoftGraphProviderConfig, - nextLink: string | null, + nextLink: string | undefined, ): Promise> { const { items: rawUsers, nextLink: newNextLink } = await requestOnePage( @@ -304,7 +304,7 @@ export class MicrosoftGraphIncrementalEntityProvider top: PAGE_SIZE, }, queryMode: provider.queryMode, - nextLink: nextLink ?? undefined, + nextLink, }, ); @@ -352,14 +352,14 @@ export class MicrosoftGraphIncrementalEntityProvider return { done: false, entities, - cursor: { phase: 'groups', nextLink: null }, + cursor: { phase: 'groups' }, }; } private async readGroupsPage( client: MicrosoftGraphClient, provider: MicrosoftGraphProviderConfig, - nextLink: string | null, + nextLink: string | undefined, ): Promise> { const { items: rawGroups, nextLink: newNextLink } = await requestOnePage( @@ -374,7 +374,7 @@ export class MicrosoftGraphIncrementalEntityProvider top: PAGE_SIZE, }, queryMode: provider.queryMode, - nextLink: nextLink ?? undefined, + nextLink, }, ); @@ -431,6 +431,12 @@ export class MicrosoftGraphIncrementalEntityProvider userEntity.metadata.name, ); userRefs.push(stringifyEntityRef(userEntity)); + } else { + this.options.logger.debug( + `${this.getProviderName()}: group member user ${ + member.id + } could not be transformed (sparse object?), skipping`, + ); } } else if (member['@odata.type'] === '#microsoft.graph.group') { const childEntity = await groupTransformer( @@ -445,12 +451,12 @@ export class MicrosoftGraphIncrementalEntityProvider } } - entity.spec.members = userRefs; - entity.spec.children = childRefs; - entities.push({ locationKey: `msgraph-org-provider:${this.options.id}`, - entity: withLocations(this.options.id, entity), + entity: withLocations(this.options.id, { + ...entity, + spec: { ...entity.spec, members: userRefs, children: childRefs }, + }), }); }), ), diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.ts b/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.ts index 0c3f005f0e..79fd4d3487 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.ts @@ -44,24 +44,33 @@ export async function requestOnePage( // Microsoft Graph requires $count=true whenever ConsistencyLevel: eventual is set, // including plain listing requests with no $filter or $search. - if (appliedQueryMode === 'advanced' && query) { - query.count = true; - } + const finalQuery = + appliedQueryMode === 'advanced' && query + ? { ...query, count: true } + : query; const headers: Record = appliedQueryMode === 'advanced' ? { ConsistencyLevel: 'eventual' } : {}; const response = nextLink ? await client.requestRaw(nextLink, headers, 2, signal) - : await client.requestApi(path, query, headers, signal); + : await client.requestApi(path, finalQuery, headers, signal); if (response.status !== 200) { - const body = await response.json(); - const err = body?.error; + let message = `HTTP ${response.status}`; + try { + const body = await response.json(); + const err = body?.error; + if (err?.code || err?.message) { + message = `${err.code} - ${err.message}`; + } + } catch { + // Response body is not JSON; fall back to HTTP status above + } throw new Error( - `Error while reading ${nextLink ?? path} from Microsoft Graph: ${ - err?.code - } - ${err?.message}`, + `Error while reading ${ + nextLink ?? path + } from Microsoft Graph: ${message}`, ); } diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.ts b/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.ts index fd30762d5f..dc96fde43a 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.ts @@ -18,6 +18,7 @@ import { coreServices, createBackendModule, createExtensionPoint, + LoggerService, } from '@backstage/backend-plugin-api'; import { incrementalIngestionProvidersExtensionPoint } from '@backstage/plugin-catalog-backend-module-incremental-ingestion'; import { @@ -30,6 +31,7 @@ import { import { HumanDuration } from '@backstage/types'; import { MicrosoftGraphIncrementalEntityProvider, + MSGraphContext, MSGraphCursor, } from '../MicrosoftGraphIncrementalEntityProvider'; @@ -192,9 +194,9 @@ export const catalogModuleMicrosoftGraphIncrementalEntityProvider = ), }); - const restLength = deriveRestLength(providerConfig); + const restLength = deriveRestLength(providerConfig, logger); - incremental.addProvider({ + incremental.addProvider({ provider, options: { burstInterval: { seconds: 3 }, @@ -226,10 +228,18 @@ function resolveTransformer( function deriveRestLength( providerConfig: ReturnType[number], + logger: LoggerService, ): HumanDuration { const freq = providerConfig.schedule?.frequency; if (freq && typeof freq === 'object' && !('cron' in freq)) { return freq as HumanDuration; } + if (freq) { + logger.warn( + `MicrosoftGraphIncrementalEntityProvider:${providerConfig.id}: ` + + `schedule.frequency is a cron expression; cannot derive restLength from it. ` + + `Defaulting restLength to 8 hours.`, + ); + } return { hours: 8 }; } From e1cb610c7bed1d49372adaee658bacd47e1fc443 Mon Sep 17 00:00:00 2001 From: pillaris Date: Mon, 27 Apr 2026 10:55:37 +0530 Subject: [PATCH 06/20] fix: update alpha API report for MSGraphCursor.nextLink type change Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: pillaris --- .../report-alpha.api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-msgraph-incremental/report-alpha.api.md b/plugins/catalog-backend-module-msgraph-incremental/report-alpha.api.md index 38d4e4278d..ed6376fb04 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/report-alpha.api.md +++ b/plugins/catalog-backend-module-msgraph-incremental/report-alpha.api.md @@ -91,7 +91,7 @@ export type MSGraphContext = { // @public export type MSGraphCursor = { phase: 'users' | 'groups'; - nextLink: string | null; + nextLink?: string; }; // (No @packageDocumentation comment for this package) From 159a862ec340a0cdff06f5ab560008a9119aee50 Mon Sep 17 00:00:00 2001 From: pillaris Date: Mon, 27 Apr 2026 12:15:34 +0530 Subject: [PATCH 07/20] fix: address second round of Copilot review comments - getUserPhotoGated now throws for non-404 errors (401/403/5xx) so callers can distinguish a missing photo from a real API failure - Log a debug message when photo loading fails instead of swallowing the error silently - Merge transformer-pre-populated spec.members/spec.children with fetched Graph membership rather than overwriting them - deriveRestLength now also excludes manual-trigger schedule objects (not just cron expressions) from being cast as HumanDuration Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: pillaris --- ...softGraphIncrementalEntityProvider.test.ts | 58 ++++++++++++++++++- ...MicrosoftGraphIncrementalEntityProvider.ts | 24 +++++++- .../src/clientHelpers.test.ts | 13 ++++- .../src/clientHelpers.ts | 10 +++- ...MicrosoftGraphIncrementalEntityProvider.ts | 11 +++- 5 files changed, 108 insertions(+), 8 deletions(-) diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts index ffb7793dae..c384debb71 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts @@ -416,7 +416,7 @@ describe('MicrosoftGraphIncrementalEntityProvider', () => { expect(mockGetUserPhotoGated).toHaveBeenCalledWith(mockClient, 'u1', 120); }); - it('continues processing remaining users when a photo load fails', async () => { + it('continues processing remaining users when a photo load fails and logs the error', async () => { mockRequestOnePage.mockResolvedValueOnce({ items: [ { @@ -446,6 +446,10 @@ describe('MicrosoftGraphIncrementalEntityProvider', () => { // Both users should still be emitted despite the photo failure expect(result.entities!).toHaveLength(2); + expect(logger.debug).toHaveBeenCalledWith( + expect.stringContaining('failed to load photo for user u1'), + expect.anything(), + ); }); }); @@ -731,6 +735,58 @@ describe('MicrosoftGraphIncrementalEntityProvider', () => { ); }); + it('merges transformer-pre-populated members with fetched membership', async () => { + (mockClient.getOrganization as jest.Mock).mockResolvedValue({ + id: 'org-id', + displayName: 'My Org', + }); + mockRequestOnePage.mockResolvedValueOnce({ + items: [ + { id: 'grp-1', displayName: 'Engineering', mail: 'eng@example.com' }, + ], + nextLink: undefined, + }); + (mockClient.getGroupMembers as jest.Mock).mockReturnValue( + asyncYield({ + '@odata.type': '#microsoft.graph.user', + id: 'u2', + displayName: 'Bob', + userPrincipalName: 'bob@example.com', + }), + ); + + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: 'default', + provider: baseProviderConfig as any, + logger, + groupTransformer: async group => { + const base = await import( + '@backstage/plugin-catalog-backend-module-msgraph' + ).then(m => m.defaultGroupTransformer(group)); + if (!base) return undefined; + // Transformer pre-populates an extra member + base.spec = { ...base.spec, members: ['user:default/extra-user'] }; + return base; + }, + }); + + const result = await provider.next(makeContext(), groupsCursor); + + const groupEntity = result.entities!.find( + e => + e.entity.metadata.annotations?.[ + MICROSOFT_GRAPH_GROUP_ID_ANNOTATION + ] === 'grp-1', + ); + // Should contain both the transformer-set member and the fetched one + expect(groupEntity!.entity.spec?.members).toContain( + 'user:default/extra-user', + ); + expect(groupEntity!.entity.spec?.members).toContain( + 'user:default/bob_example.com', + ); + }); + it('passes group filter and search from provider config', async () => { const providerWithFilter = { ...baseProviderConfig, diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts index de7f57d1f5..8b0835e08d 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts @@ -319,8 +319,13 @@ export class MicrosoftGraphIncrementalEntityProvider if (provider.loadUserPhotos !== false) { try { userPhoto = await getUserPhotoGated(client, user.id!, 120); - } catch { - // Photo load failures are non-fatal + } catch (e) { + this.options.logger.debug( + `${this.getProviderName()}: failed to load photo for user ${ + user.id + }`, + { error: e }, + ); } } @@ -451,11 +456,24 @@ export class MicrosoftGraphIncrementalEntityProvider } } + // Merge fetched membership with any members/children the transformer + // may have pre-populated, so custom transformers can augment the list. + const existingMembers = Array.isArray(entity.spec?.members) + ? (entity.spec.members as string[]) + : []; + const existingChildren = Array.isArray(entity.spec?.children) + ? (entity.spec.children as string[]) + : []; + entities.push({ locationKey: `msgraph-org-provider:${this.options.id}`, entity: withLocations(this.options.id, { ...entity, - spec: { ...entity.spec, members: userRefs, children: childRefs }, + spec: { + ...entity.spec, + members: [...new Set([...existingMembers, ...userRefs])], + children: [...new Set([...existingChildren, ...childRefs])], + }, }), }); }), diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.test.ts b/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.test.ts index 1819492be8..fb8b28d8ec 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.test.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.test.ts @@ -225,7 +225,7 @@ describe('getUserPhotoGated', () => { afterEach(() => jest.resetAllMocks()); - it('returns undefined when the photo check returns non-200', async () => { + it('returns undefined when the photo check returns 404', async () => { (client.requestApi as jest.Mock).mockResolvedValue({ status: 404, } as Response); @@ -236,6 +236,17 @@ describe('getUserPhotoGated', () => { expect(client.getUserPhotoWithSizeLimit).not.toHaveBeenCalled(); }); + it('throws for non-404 error responses so callers can distinguish real errors', async () => { + (client.requestApi as jest.Mock).mockResolvedValue({ + status: 403, + } as Response); + + await expect(getUserPhotoGated(client, 'user-id', 120)).rejects.toThrow( + 'Unexpected status 403 when checking photo for user user-id', + ); + expect(client.getUserPhotoWithSizeLimit).not.toHaveBeenCalled(); + }); + it('returns the photo data URI when the check passes', async () => { (client.requestApi as jest.Mock).mockResolvedValue({ status: 200, diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.ts b/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.ts index 79fd4d3487..7a0cce38c8 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.ts @@ -85,6 +85,9 @@ export async function requestOnePage( * Like `getUserPhotoWithSizeLimit` but skips the size-listing call for users * with no photo. For users without a photo: 1 fast check call. For users with * a photo: 1 check + the normal size-limited fetch (2 more calls). + * + * Returns `undefined` only for 404 (no photo assigned). Throws for any other + * non-200 status so callers can distinguish "no photo" from real errors. */ export async function getUserPhotoGated( client: MicrosoftGraphClient, @@ -92,6 +95,11 @@ export async function getUserPhotoGated( maxSize: number, ): Promise { const check = await client.requestApi(`users/${userId}/photo`); - if (check.status !== 200) return undefined; + if (check.status === 404) return undefined; + if (check.status !== 200) { + throw new Error( + `Unexpected status ${check.status} when checking photo for user ${userId}`, + ); + } return await client.getUserPhotoWithSizeLimit(userId, maxSize); } diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.ts b/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.ts index dc96fde43a..674ce7a009 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.ts @@ -231,13 +231,20 @@ function deriveRestLength( logger: LoggerService, ): HumanDuration { const freq = providerConfig.schedule?.frequency; - if (freq && typeof freq === 'object' && !('cron' in freq)) { + // Only treat plain duration objects as restLength — exclude cron expressions + // and any other non-duration schedule types (e.g. manual triggers). + if ( + freq && + typeof freq === 'object' && + !('cron' in freq) && + !('trigger' in freq) + ) { return freq as HumanDuration; } if (freq) { logger.warn( `MicrosoftGraphIncrementalEntityProvider:${providerConfig.id}: ` + - `schedule.frequency is a cron expression; cannot derive restLength from it. ` + + `schedule.frequency is not a duration-based schedule; cannot derive restLength from it. ` + `Defaulting restLength to 8 hours.`, ); } From 99ddcc39571d72cb682a10048858fc9f9e43ff2a Mon Sep 17 00:00:00 2001 From: pillaris Date: Mon, 27 Apr 2026 12:44:50 +0530 Subject: [PATCH 08/20] fix: address third round of Copilot review comments - Wrap userTransformer call for group members in try/catch so a sparse object that causes the transformer to throw logs a warning and skips that member rather than failing the entire groups page - Split PAGE_SIZE into USER_PAGE_SIZE (999) and GROUP_PAGE_SIZE (100) so each burst stays within its time budget despite per-group member fetching in the groups phase - Always send $count=true in advanced mode even when no query object is provided (use empty object spread instead of short-circuit on query) Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: pillaris --- ...softGraphIncrementalEntityProvider.test.ts | 58 ++++++++++++++++++- ...MicrosoftGraphIncrementalEntityProvider.ts | 40 ++++++++----- .../src/clientHelpers.test.ts | 15 +++++ .../src/clientHelpers.ts | 4 +- 4 files changed, 99 insertions(+), 18 deletions(-) diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts index c384debb71..cc10e03b68 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts @@ -231,7 +231,7 @@ describe('MicrosoftGraphIncrementalEntityProvider', () => { mockClient, 'users', expect.objectContaining({ - query: expect.objectContaining({ top: 999 }), + query: expect.objectContaining({ top: 999 }), // USER_PAGE_SIZE }), ); // No users → advances straight to groups phase @@ -735,6 +735,62 @@ describe('MicrosoftGraphIncrementalEntityProvider', () => { ); }); + it('logs a warning and skips a group member when the transformer throws', async () => { + (mockClient.getOrganization as jest.Mock).mockResolvedValue({ + id: 'org-id', + displayName: 'My Org', + }); + mockRequestOnePage.mockResolvedValueOnce({ + items: [ + { id: 'grp-1', displayName: 'Engineering', mail: 'eng@example.com' }, + ], + nextLink: undefined, + }); + (mockClient.getGroupMembers as jest.Mock).mockReturnValue( + asyncYield( + { + '@odata.type': '#microsoft.graph.user', + id: 'u-bad', + // sparse — no userPrincipalName, transformer will throw + }, + { + '@odata.type': '#microsoft.graph.user', + id: 'u-good', + displayName: 'Alice', + userPrincipalName: 'alice@example.com', + }, + ), + ); + + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: 'default', + provider: baseProviderConfig as any, + logger, + userTransformer: async user => { + if (!user.userPrincipalName) throw new Error('Missing UPN'); + const { defaultUserTransformer } = await import( + '@backstage/plugin-catalog-backend-module-msgraph' + ); + return defaultUserTransformer(user); + }, + }); + + const result = await provider.next(makeContext(), groupsCursor); + + expect(logger.warn).toHaveBeenCalledWith( + expect.stringContaining('group member user u-bad failed to transform'), + expect.anything(), + ); + const groupEntity = result.entities!.find( + e => + e.entity.metadata.annotations?.[ + MICROSOFT_GRAPH_GROUP_ID_ANNOTATION + ] === 'grp-1', + ); + // Only the good member should appear + expect(groupEntity!.entity.spec?.members).toHaveLength(1); + }); + it('merges transformer-pre-populated members with fetched membership', async () => { (mockClient.getOrganization as jest.Mock).mockResolvedValue({ id: 'org-id', diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts index 8b0835e08d..06e125789c 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts @@ -47,7 +47,10 @@ import { import { LoggerService } from '@backstage/backend-plugin-api'; import { getUserPhotoGated, requestOnePage } from './clientHelpers'; -const PAGE_SIZE = 999; +const USER_PAGE_SIZE = 999; +// Groups phase fetches members for every group on the page, so a smaller page +// size keeps each burst within its time budget. +const GROUP_PAGE_SIZE = 100; /** * Backstage entity names must be ≤63 chars ([a-zA-Z0-9] separated by [-_.]). @@ -301,7 +304,7 @@ export class MicrosoftGraphIncrementalEntityProvider filter: provider.userFilter, expand: provider.userExpand, select: provider.userSelect, - top: PAGE_SIZE, + top: USER_PAGE_SIZE, }, queryMode: provider.queryMode, nextLink, @@ -376,7 +379,7 @@ export class MicrosoftGraphIncrementalEntityProvider search: provider.groupSearch, expand: provider.groupExpand, select: provider.groupSelect, - top: PAGE_SIZE, + top: GROUP_PAGE_SIZE, }, queryMode: provider.queryMode, nextLink, @@ -425,22 +428,31 @@ export class MicrosoftGraphIncrementalEntityProvider const childRefs: string[] = []; for await (const member of client.getGroupMembers(group.id!, { - top: PAGE_SIZE, + top: GROUP_PAGE_SIZE, })) { if (member['@odata.type'] === '#microsoft.graph.user') { - const userEntity = await userTransformer( - member as MicrosoftGraph.User, - ); - if (userEntity) { - userEntity.metadata.name = capEntityName( - userEntity.metadata.name, + try { + const userEntity = await userTransformer( + member as MicrosoftGraph.User, ); - userRefs.push(stringifyEntityRef(userEntity)); - } else { - this.options.logger.debug( + if (userEntity) { + userEntity.metadata.name = capEntityName( + userEntity.metadata.name, + ); + userRefs.push(stringifyEntityRef(userEntity)); + } else { + this.options.logger.debug( + `${this.getProviderName()}: group member user ${ + member.id + } could not be transformed (sparse object?), skipping`, + ); + } + } catch (e) { + this.options.logger.warn( `${this.getProviderName()}: group member user ${ member.id - } could not be transformed (sparse object?), skipping`, + } failed to transform, skipping`, + { error: e }, ); } } else if (member['@odata.type'] === '#microsoft.graph.group') { diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.test.ts b/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.test.ts index fb8b28d8ec..93dd290f2a 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.test.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.test.ts @@ -110,6 +110,21 @@ describe('requestOnePage', () => { ); }); + it('sends $count=true in advanced mode even when no query is provided', async () => { + (client.requestApi as jest.Mock).mockResolvedValue( + makeResponse(200, { value: [] }), + ); + + await requestOnePage(client, 'users', { queryMode: 'advanced' }); + + expect(client.requestApi).toHaveBeenCalledWith( + 'users', + expect.objectContaining({ count: true }), + { ConsistencyLevel: 'eventual' }, + undefined, + ); + }); + it('does not set $count for basic mode', async () => { (client.requestApi as jest.Mock).mockResolvedValue( makeResponse(200, { value: [] }), diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.ts b/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.ts index 7a0cce38c8..2397ba46b9 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.ts @@ -45,9 +45,7 @@ export async function requestOnePage( // Microsoft Graph requires $count=true whenever ConsistencyLevel: eventual is set, // including plain listing requests with no $filter or $search. const finalQuery = - appliedQueryMode === 'advanced' && query - ? { ...query, count: true } - : query; + appliedQueryMode === 'advanced' ? { ...(query ?? {}), count: true } : query; const headers: Record = appliedQueryMode === 'advanced' ? { ConsistencyLevel: 'eventual' } : {}; From 6fa66e7907c28efb025612505759b45382bcb19d Mon Sep 17 00:00:00 2001 From: pillaris Date: Mon, 27 Apr 2026 14:26:52 +0530 Subject: [PATCH 09/20] fix: remove unnecessary direct msgraph dep from example backend @backstage/plugin-catalog-backend-module-msgraph is only used transitively via msgraph-incremental; no direct import exists in packages/backend/src/index.ts. Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: pillaris --- packages/backend/package.json | 1 - yarn.lock | 1 - 2 files changed, 2 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 88ae7f3351..010ab4c75e 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -43,7 +43,6 @@ "@backstage/plugin-catalog-backend-module-backstage-openapi": "workspace:^", "@backstage/plugin-catalog-backend-module-incremental-ingestion": "workspace:^", "@backstage/plugin-catalog-backend-module-logs": "workspace:^", - "@backstage/plugin-catalog-backend-module-msgraph": "workspace:^", "@backstage/plugin-catalog-backend-module-msgraph-incremental": "workspace:^", "@backstage/plugin-catalog-backend-module-openapi": "workspace:^", "@backstage/plugin-catalog-backend-module-scaffolder-entity-model": "workspace:^", diff --git a/yarn.lock b/yarn.lock index c9f8c05aac..53a8f4b8cb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -29105,7 +29105,6 @@ __metadata: "@backstage/plugin-catalog-backend-module-backstage-openapi": "workspace:^" "@backstage/plugin-catalog-backend-module-incremental-ingestion": "workspace:^" "@backstage/plugin-catalog-backend-module-logs": "workspace:^" - "@backstage/plugin-catalog-backend-module-msgraph": "workspace:^" "@backstage/plugin-catalog-backend-module-msgraph-incremental": "workspace:^" "@backstage/plugin-catalog-backend-module-openapi": "workspace:^" "@backstage/plugin-catalog-backend-module-scaffolder-entity-model": "workspace:^" From ac04bb696dae2f7e00345833cc197189c38a3d4b Mon Sep 17 00:00:00 2001 From: pillaris Date: Mon, 27 Apr 2026 14:29:41 +0530 Subject: [PATCH 10/20] fix: pass $select when fetching group members to prevent sparse objects Without $select, Microsoft Graph returns only id and @odata.type for group members, which causes defaultUserTransformer/defaultGroupTransformer to return undefined and silently drop membership refs. Requesting the minimum fields needed by both transformers ensures member objects always have enough data to produce stable entity refs. Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: pillaris --- .../MicrosoftGraphIncrementalEntityProvider.test.ts | 11 +++++++++++ .../src/MicrosoftGraphIncrementalEntityProvider.ts | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts index cc10e03b68..7e65d8a728 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts @@ -586,6 +586,17 @@ describe('MicrosoftGraphIncrementalEntityProvider', () => { expect(groupEntity!.entity.spec?.members).toContain( 'user:default/alice_example.com', ); + // Verify $select is passed so member objects are never sparse + expect(mockClient.getGroupMembers).toHaveBeenCalledWith( + 'grp-1', + expect.objectContaining({ + select: expect.arrayContaining([ + 'id', + 'displayName', + 'userPrincipalName', + ]), + }), + ); }); it('populates spec.children with nested group refs from getGroupMembers', async () => { diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts index 06e125789c..6c290f17ea 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts @@ -429,6 +429,17 @@ export class MicrosoftGraphIncrementalEntityProvider for await (const member of client.getGroupMembers(group.id!, { top: GROUP_PAGE_SIZE, + // Request the minimum fields needed by defaultUserTransformer and + // defaultGroupTransformer so member objects are never sparse. + select: [ + 'id', + 'displayName', + 'mail', + 'mailNickname', + 'userPrincipalName', + 'description', + 'securityEnabled', + ], })) { if (member['@odata.type'] === '#microsoft.graph.user') { try { From 0f36fcd879ed7bb01c7c5d32bd67aa338d11c315 Mon Sep 17 00:00:00 2001 From: pillaris Date: Mon, 27 Apr 2026 14:35:14 +0530 Subject: [PATCH 11/20] chore: remove incremental ingestion modules from example backend These modules are not needed by all Backstage users, so registering them unconditionally in the example backend adds unnecessary routes, services, and runtime dependencies. The backend.add(...) lines are documented in the plugin README instead. Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: pillaris --- packages/backend/package.json | 2 -- packages/backend/src/index.ts | 6 ------ yarn.lock | 4 +--- 3 files changed, 1 insertion(+), 11 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 010ab4c75e..e00539b095 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -41,9 +41,7 @@ "@backstage/plugin-auth-node": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", "@backstage/plugin-catalog-backend-module-backstage-openapi": "workspace:^", - "@backstage/plugin-catalog-backend-module-incremental-ingestion": "workspace:^", "@backstage/plugin-catalog-backend-module-logs": "workspace:^", - "@backstage/plugin-catalog-backend-module-msgraph-incremental": "workspace:^", "@backstage/plugin-catalog-backend-module-openapi": "workspace:^", "@backstage/plugin-catalog-backend-module-scaffolder-entity-model": "workspace:^", "@backstage/plugin-catalog-backend-module-unprocessed": "workspace:^", diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 4db72cc42d..e77c83baac 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -46,12 +46,6 @@ backend.add(import('@backstage/plugin-auth-backend-module-guest-provider')); backend.add(import('@backstage/plugin-auth-backend-module-openshift-provider')); backend.add(import('@backstage/plugin-app-backend')); backend.add(import('@backstage/plugin-catalog-backend-module-unprocessed')); -backend.add( - import('@backstage/plugin-catalog-backend-module-incremental-ingestion'), -); -backend.add( - import('@backstage/plugin-catalog-backend-module-msgraph-incremental'), -); backend.add( import('@backstage/plugin-catalog-backend-module-scaffolder-entity-model'), ); diff --git a/yarn.lock b/yarn.lock index 53a8f4b8cb..a5e51ce81b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5072,7 +5072,7 @@ __metadata: languageName: unknown linkType: soft -"@backstage/plugin-catalog-backend-module-msgraph-incremental@workspace:^, @backstage/plugin-catalog-backend-module-msgraph-incremental@workspace:plugins/catalog-backend-module-msgraph-incremental": +"@backstage/plugin-catalog-backend-module-msgraph-incremental@workspace:plugins/catalog-backend-module-msgraph-incremental": version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-backend-module-msgraph-incremental@workspace:plugins/catalog-backend-module-msgraph-incremental" dependencies: @@ -29103,9 +29103,7 @@ __metadata: "@backstage/plugin-auth-node": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" "@backstage/plugin-catalog-backend-module-backstage-openapi": "workspace:^" - "@backstage/plugin-catalog-backend-module-incremental-ingestion": "workspace:^" "@backstage/plugin-catalog-backend-module-logs": "workspace:^" - "@backstage/plugin-catalog-backend-module-msgraph-incremental": "workspace:^" "@backstage/plugin-catalog-backend-module-openapi": "workspace:^" "@backstage/plugin-catalog-backend-module-scaffolder-entity-model": "workspace:^" "@backstage/plugin-catalog-backend-module-unprocessed": "workspace:^" From 9e9eca49c74a349993a7060adb14fe75eb886d1d Mon Sep 17 00:00:00 2001 From: pillaris Date: Mon, 27 Apr 2026 14:44:07 +0530 Subject: [PATCH 12/20] docs: fix page-size docs and note groupIncludeSubGroups unsupported Update all references to "up to 999 items" to accurately reflect that users are fetched in pages of 999 while groups use a smaller page of 100. Also document groupIncludeSubGroups as unsupported in the JSDoc @remarks, README, and comparison table. Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: pillaris --- .changeset/msgraph-incremental-initial.md | 2 +- .../README.md | 5 +++-- .../src/MicrosoftGraphIncrementalEntityProvider.ts | 12 +++++++----- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.changeset/msgraph-incremental-initial.md b/.changeset/msgraph-incremental-initial.md index 2a16af91a8..3137c2b57f 100644 --- a/.changeset/msgraph-incremental-initial.md +++ b/.changeset/msgraph-incremental-initial.md @@ -4,4 +4,4 @@ New package: `@backstage/plugin-catalog-backend-module-msgraph-incremental` -Introduces a cursor-based incremental ingestion provider for Microsoft Graph that processes users and groups one page at a time. Unlike `MicrosoftGraphOrgEntityProvider`, this module never holds the full dataset in memory — each burst processes a single page of up to 999 items. The `@odata.nextLink` cursor is persisted so a pod restart resumes from the last completed page rather than starting over. +Introduces a cursor-based incremental ingestion provider for Microsoft Graph that processes users and groups one page at a time. Unlike `MicrosoftGraphOrgEntityProvider`, this module never holds the full dataset in memory — each burst processes a single page (up to 999 users or 100 groups). The `@odata.nextLink` cursor is persisted so a pod restart resumes from the last completed page rather than starting over. diff --git a/plugins/catalog-backend-module-msgraph-incremental/README.md b/plugins/catalog-backend-module-msgraph-incremental/README.md index 60c971e87f..342148f049 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/README.md +++ b/plugins/catalog-backend-module-msgraph-incremental/README.md @@ -9,8 +9,8 @@ AD tenants where holding the full dataset in memory at once is not practical. - **Cursor-based resumption** — the `@odata.nextLink` URL is persisted as the cursor, so a pod restart during ingestion resumes from the last completed page rather than starting over. -- **Memory-efficient** — each burst processes a single page (up to 999 items), - keeping memory usage flat regardless of tenant size. +- **Memory-efficient** — each burst processes a single page (up to 999 users + or 100 groups), keeping memory usage flat regardless of tenant size. - **Photo support** — user profile photos are fetched with a gated pre-check to avoid unnecessary API calls for users without photos. - **Transformer extension point** — user, group, organization, and provider @@ -70,4 +70,5 @@ catalog: | Memory usage | Full dataset in RAM | One page at a time | | Resume on restart | Starts from scratch | Resumes from cursor | | `userGroupMember*` options | Supported | Not supported | +| `groupIncludeSubGroups` | Supported | Not supported | | Suitable for large tenants | No | Yes | diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts index 6c290f17ea..642cf88f44 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts @@ -153,8 +153,8 @@ export interface MicrosoftGraphIncrementalEntityProviderOptions { * at a time, and provides them as User and Group entities for the catalog. * * Unlike `MicrosoftGraphOrgEntityProvider`, this provider never holds the full - * dataset in memory at once. Each burst processes a single page of users or - * groups (up to 999 items). This makes it suitable for very large tenants and + * dataset in memory at once. Each burst processes a single page (up to 999 + * users or 100 groups). This makes it suitable for very large tenants and * avoids the memory pressure and long-running task issues of the full-scan * provider. * @@ -166,9 +166,11 @@ export interface MicrosoftGraphIncrementalEntityProviderOptions { * stitching derives `spec.memberOf` on users from these group membership lists. * * @remarks - * `userGroupMemberFilter`, `userGroupMemberSearch`, and `userGroupMemberPath` - * are not supported. Use `userFilter` / `userPath` to restrict which users are - * ingested, and `groupFilter` / `groupSearch` to restrict which groups. + * `userGroupMemberFilter`, `userGroupMemberSearch`, `userGroupMemberPath`, and + * `groupIncludeSubGroups` are not supported. Use `userFilter` / `userPath` to + * restrict which users are ingested, and `groupFilter` / `groupSearch` to + * restrict which groups. Switch to `MicrosoftGraphOrgEntityProvider` if you + * require any of these options. * * @public */ From e1714a861c798d673540bf9c1e179e64a0720ab6 Mon Sep 17 00:00:00 2001 From: pillaris Date: Mon, 27 Apr 2026 15:10:25 +0530 Subject: [PATCH 13/20] fix: guard user.id before photo fetch, warn on groupIncludeSubGroups, skip dangling child refs - Guard getUserPhotoGated with user.id presence to prevent requesting users/undefined/photo when Graph omits the id field - Log a warning when groupIncludeSubGroups is configured, matching the existing warning for unsupported userGroupMember* options - Skip spec.children population when groupFilter/groupSearch is active, since child groups may not pass the filter and would create dangling references in the catalog Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: pillaris --- ...softGraphIncrementalEntityProvider.test.ts | 86 +++++++++++++++++++ ...MicrosoftGraphIncrementalEntityProvider.ts | 31 +++++-- 2 files changed, 108 insertions(+), 9 deletions(-) diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts index 7e65d8a728..954c8d43c0 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts @@ -205,6 +205,25 @@ describe('MicrosoftGraphIncrementalEntityProvider', () => { ), ); }); + + it('warns when groupIncludeSubGroups is set', async () => { + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: 'default', + provider: { + ...baseProviderConfig, + groupIncludeSubGroups: true, + } as any, + logger, + }); + + await provider.around(async () => {}); + + expect(logger.warn).toHaveBeenCalledWith( + expect.stringContaining( + 'groupIncludeSubGroups is not supported', + ), + ); + }); }); describe('next — users phase', () => { @@ -416,6 +435,29 @@ describe('MicrosoftGraphIncrementalEntityProvider', () => { expect(mockGetUserPhotoGated).toHaveBeenCalledWith(mockClient, 'u1', 120); }); + it('skips photo fetch when user has no id to avoid requesting users/undefined/photo', async () => { + mockRequestOnePage.mockResolvedValueOnce({ + items: [ + { + // No id field — Graph can theoretically omit it + displayName: 'No-ID User', + userPrincipalName: 'noid@example.com', + }, + ], + nextLink: undefined, + }); + + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: 'default', + provider: baseProviderConfig as any, + logger, + }); + + await provider.next(makeContext()); + + expect(mockGetUserPhotoGated).not.toHaveBeenCalled(); + }); + it('continues processing remaining users when a photo load fails and logs the error', async () => { mockRequestOnePage.mockResolvedValueOnce({ items: [ @@ -640,6 +682,50 @@ describe('MicrosoftGraphIncrementalEntityProvider', () => { expect(parentEntity!.entity.spec?.children).toHaveLength(1); }); + it('omits child group refs when groupFilter is active to avoid dangling references', async () => { + const providerWithFilter = { + ...baseProviderConfig, + groupFilter: 'securityEnabled eq true', + }; + (mockClient.getOrganization as jest.Mock).mockResolvedValue({ + id: 'org-id', + displayName: 'My Org', + }); + mockRequestOnePage.mockResolvedValueOnce({ + items: [ + { id: 'grp-parent', displayName: 'Parent', mail: 'parent@example.com' }, + ], + nextLink: undefined, + }); + (mockClient.getGroupMembers as jest.Mock).mockReturnValue( + asyncYield({ + '@odata.type': '#microsoft.graph.group', + id: 'grp-child', + displayName: 'Child Group', + mail: 'child@example.com', + }), + ); + + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: 'default', + provider: providerWithFilter as any, + logger, + }); + + const result = await provider.next( + { client: mockClient, provider: providerWithFilter as any }, + groupsCursor, + ); + + const parentEntity = result.entities!.find( + e => + e.entity.metadata.annotations?.[ + MICROSOFT_GRAPH_GROUP_ID_ANNOTATION + ] === 'grp-parent', + ); + expect(parentEntity!.entity.spec?.children).toHaveLength(0); + }); + it('returns done:false with groups cursor when nextLink is present', async () => { const nextLink = 'https://graph.microsoft.com/v1.0/groups?$skiptoken=page2'; diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts index 642cf88f44..dc333a924f 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts @@ -274,6 +274,14 @@ export class MicrosoftGraphIncrementalEntityProvider ); } + if (provider.groupIncludeSubGroups) { + this.options.logger.warn( + `${this.getProviderName()}: groupIncludeSubGroups is not supported by ` + + `MicrosoftGraphIncrementalEntityProvider and will be ignored. ` + + `Switch to MicrosoftGraphOrgEntityProvider if you require this option.`, + ); + } + const client = MicrosoftGraphClient.create(provider); await burst({ client, provider }); } @@ -321,9 +329,9 @@ export class MicrosoftGraphIncrementalEntityProvider rawUsers.map(user => limiter(async () => { let userPhoto: string | undefined; - if (provider.loadUserPhotos !== false) { + if (user.id && provider.loadUserPhotos !== false) { try { - userPhoto = await getUserPhotoGated(client, user.id!, 120); + userPhoto = await getUserPhotoGated(client, user.id, 120); } catch (e) { this.options.logger.debug( `${this.getProviderName()}: failed to load photo for user ${ @@ -469,14 +477,19 @@ export class MicrosoftGraphIncrementalEntityProvider ); } } else if (member['@odata.type'] === '#microsoft.graph.group') { - const childEntity = await groupTransformer( - member as MicrosoftGraph.Group, - ); - if (childEntity) { - childEntity.metadata.name = capEntityName( - childEntity.metadata.name, + // Only emit child refs when no group filter/search is active. + // With a filter, child groups may not be ingested themselves, + // which would produce dangling spec.children references. + if (!provider.groupFilter && !provider.groupSearch) { + const childEntity = await groupTransformer( + member as MicrosoftGraph.Group, ); - childRefs.push(stringifyEntityRef(childEntity)); + if (childEntity) { + childEntity.metadata.name = capEntityName( + childEntity.metadata.name, + ); + childRefs.push(stringifyEntityRef(childEntity)); + } } } } From 69cf79ea0016714b73021af5aa11f8c5f8ae3aaa Mon Sep 17 00:00:00 2001 From: pillaris Date: Mon, 27 Apr 2026 15:28:39 +0530 Subject: [PATCH 14/20] chore: fix prettier formatting in provider test file Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: pillaris --- .../MicrosoftGraphIncrementalEntityProvider.test.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts index 954c8d43c0..2fbc4bccf5 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts @@ -219,9 +219,7 @@ describe('MicrosoftGraphIncrementalEntityProvider', () => { await provider.around(async () => {}); expect(logger.warn).toHaveBeenCalledWith( - expect.stringContaining( - 'groupIncludeSubGroups is not supported', - ), + expect.stringContaining('groupIncludeSubGroups is not supported'), ); }); }); @@ -693,7 +691,11 @@ describe('MicrosoftGraphIncrementalEntityProvider', () => { }); mockRequestOnePage.mockResolvedValueOnce({ items: [ - { id: 'grp-parent', displayName: 'Parent', mail: 'parent@example.com' }, + { + id: 'grp-parent', + displayName: 'Parent', + mail: 'parent@example.com', + }, ], nextLink: undefined, }); From 1b5e83401ff586caeb65c38b2ea1965af62f5c31 Mon Sep 17 00:00:00 2001 From: pillaris Date: Mon, 27 Apr 2026 15:43:33 +0530 Subject: [PATCH 15/20] fix: wrap child group transformer in try/catch to keep burst resilient A throwing custom groupTransformer on a child group member would fail the entire groups-phase burst. Apply the same try/catch + debug/warn logging pattern already used for user member transformation. Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: pillaris --- ...softGraphIncrementalEntityProvider.test.ts | 64 +++++++++++++++++++ ...MicrosoftGraphIncrementalEntityProvider.ts | 29 +++++++-- 2 files changed, 86 insertions(+), 7 deletions(-) diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts index 2fbc4bccf5..630b246f81 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts @@ -728,6 +728,70 @@ describe('MicrosoftGraphIncrementalEntityProvider', () => { expect(parentEntity!.entity.spec?.children).toHaveLength(0); }); + it('logs a warning and skips a child group member when the transformer throws', async () => { + const throwingGroupTransformer = jest + .fn() + .mockResolvedValueOnce({ + // first call: parent group transform succeeds + apiVersion: 'backstage.io/v1alpha1', + kind: 'Group', + metadata: { + name: 'parent-group', + annotations: { 'graph.microsoft.com/group-id': 'grp-parent' }, + }, + spec: { type: 'team', children: [], members: [] }, + }) + .mockRejectedValueOnce(new Error('Transformer error')); + + (mockClient.getOrganization as jest.Mock).mockResolvedValue({ + id: 'org-id', + displayName: 'My Org', + }); + mockRequestOnePage.mockResolvedValueOnce({ + items: [ + { + id: 'grp-parent', + displayName: 'Parent', + mail: 'parent@example.com', + }, + ], + nextLink: undefined, + }); + (mockClient.getGroupMembers as jest.Mock).mockReturnValue( + asyncYield({ + '@odata.type': '#microsoft.graph.group', + id: 'grp-child', + displayName: 'Child Group', + mail: 'child@example.com', + }), + ); + + const provider = new MicrosoftGraphIncrementalEntityProvider({ + id: 'default', + provider: baseProviderConfig as any, + logger, + groupTransformer: throwingGroupTransformer, + }); + + const result = await provider.next(makeContext(), groupsCursor); + + // Parent group still emitted despite child transformer throwing + const parentEntity = result.entities!.find( + e => + e.entity.metadata.annotations?.[ + MICROSOFT_GRAPH_GROUP_ID_ANNOTATION + ] === 'grp-parent', + ); + expect(parentEntity).toBeDefined(); + expect(parentEntity!.entity.spec?.children).toHaveLength(0); + expect(logger.warn).toHaveBeenCalledWith( + expect.stringContaining( + 'group member child group grp-child failed to transform, skipping', + ), + expect.anything(), + ); + }); + it('returns done:false with groups cursor when nextLink is present', async () => { const nextLink = 'https://graph.microsoft.com/v1.0/groups?$skiptoken=page2'; diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts index dc333a924f..dedc94b13b 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts @@ -481,14 +481,29 @@ export class MicrosoftGraphIncrementalEntityProvider // With a filter, child groups may not be ingested themselves, // which would produce dangling spec.children references. if (!provider.groupFilter && !provider.groupSearch) { - const childEntity = await groupTransformer( - member as MicrosoftGraph.Group, - ); - if (childEntity) { - childEntity.metadata.name = capEntityName( - childEntity.metadata.name, + try { + const childEntity = await groupTransformer( + member as MicrosoftGraph.Group, + ); + if (childEntity) { + childEntity.metadata.name = capEntityName( + childEntity.metadata.name, + ); + childRefs.push(stringifyEntityRef(childEntity)); + } else { + this.options.logger.debug( + `${this.getProviderName()}: group member child group ${ + member.id + } could not be transformed (sparse object?), skipping`, + ); + } + } catch (e) { + this.options.logger.warn( + `${this.getProviderName()}: group member child group ${ + member.id + } failed to transform, skipping`, + { error: e }, ); - childRefs.push(stringifyEntityRef(childEntity)); } } } From ff199fef07008e212d5050f14bd549910d9e814c Mon Sep 17 00:00:00 2001 From: pillaris Date: Tue, 28 Apr 2026 09:54:40 +0530 Subject: [PATCH 16/20] fix: address Andre's review comments - Update copyright year from 2024 to 2026 across all new source files - Clear alpha entry point (no alpha exports needed for a net-new package) and regenerate report-alpha.api.md accordingly - Delete CHANGELOG.md (auto-generated by the release process) - Change changeset bump from patch to minor so the first release is 0.1.0, and remove the redundant "New package:" title line - Reset package version to 0.0.0 (release process sets the real version) - Add incremental ingestion section to docs/integrations/azure/org.md Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: pillaris --- .changeset/msgraph-incremental-initial.md | 4 +- docs/integrations/azure/org.md | 31 +++++++ .../CHANGELOG.md | 16 ---- .../package.json | 2 +- .../report-alpha.api.md | 91 ------------------- ...softGraphIncrementalEntityProvider.test.ts | 2 +- ...MicrosoftGraphIncrementalEntityProvider.ts | 2 +- .../src/alpha.ts | 6 +- .../src/clientHelpers.test.ts | 2 +- .../src/clientHelpers.ts | 2 +- .../src/index.ts | 2 +- ...softGraphIncrementalEntityProvider.test.ts | 2 +- ...MicrosoftGraphIncrementalEntityProvider.ts | 2 +- .../src/module/index.ts | 2 +- 14 files changed, 43 insertions(+), 123 deletions(-) delete mode 100644 plugins/catalog-backend-module-msgraph-incremental/CHANGELOG.md diff --git a/.changeset/msgraph-incremental-initial.md b/.changeset/msgraph-incremental-initial.md index 3137c2b57f..6f41346a12 100644 --- a/.changeset/msgraph-incremental-initial.md +++ b/.changeset/msgraph-incremental-initial.md @@ -1,7 +1,5 @@ --- -'@backstage/plugin-catalog-backend-module-msgraph-incremental': patch +'@backstage/plugin-catalog-backend-module-msgraph-incremental': minor --- -New package: `@backstage/plugin-catalog-backend-module-msgraph-incremental` - Introduces a cursor-based incremental ingestion provider for Microsoft Graph that processes users and groups one page at a time. Unlike `MicrosoftGraphOrgEntityProvider`, this module never holds the full dataset in memory — each burst processes a single page (up to 999 users or 100 groups). The `@odata.nextLink` cursor is persisted so a pod restart resumes from the last completed page rather than starting over. diff --git a/docs/integrations/azure/org.md b/docs/integrations/azure/org.md index 3c82f6e2be..defe0aab41 100644 --- a/docs/integrations/azure/org.md +++ b/docs/integrations/azure/org.md @@ -50,6 +50,37 @@ backend.add(import('@backstage/plugin-catalog-backend-module-msgraph')); /* highlight-add-end */ ``` +## Incremental Ingestion for Large Tenants + +For very large Azure AD tenants where loading the full dataset into memory at once is not feasible, the `@backstage/plugin-catalog-backend-module-msgraph-incremental` package provides a memory-efficient alternative. It processes users and groups one page at a time and persists the `@odata.nextLink` cursor so ingestion resumes from the last completed page after a pod restart. + +```bash title="From your Backstage root directory" +yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-incremental-ingestion +yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-msgraph-incremental +``` + +```ts title="packages/backend/src/index.ts" +backend.add(import('@backstage/plugin-catalog-backend')); +/* highlight-add-start */ +backend.add( + import('@backstage/plugin-catalog-backend-module-incremental-ingestion'), +); +backend.add( + import('@backstage/plugin-catalog-backend-module-msgraph-incremental'), +); +/* highlight-add-end */ +``` + +It uses the same `catalog.providers.microsoftGraphOrg` configuration as the standard module. The following options are **not** supported by the incremental provider: `userGroupMember*` and `groupIncludeSubGroups`. Use `MicrosoftGraphOrgEntityProvider` if you require those. + +| | `MicrosoftGraphOrgEntityProvider` | Incremental provider | +| -------------------------- | --------------------------------- | -------------------- | +| Memory usage | Full dataset in RAM | One page at a time | +| Resume on restart | Starts from scratch | Resumes from cursor | +| `userGroupMember*` options | Supported | Not supported | +| `groupIncludeSubGroups` | Supported | Not supported | +| Suitable for large tenants | No | Yes | + ## Authenticating with Microsoft Graph ### Local Development diff --git a/plugins/catalog-backend-module-msgraph-incremental/CHANGELOG.md b/plugins/catalog-backend-module-msgraph-incremental/CHANGELOG.md deleted file mode 100644 index 0bb7f6f0f8..0000000000 --- a/plugins/catalog-backend-module-msgraph-incremental/CHANGELOG.md +++ /dev/null @@ -1,16 +0,0 @@ -# @backstage/plugin-catalog-backend-module-msgraph-incremental - -## 0.1.0 - -### Minor Changes - -- Initial release of the incremental Microsoft Graph catalog module. - - Provides `catalogModuleMicrosoftGraphIncrementalEntityProvider`, which - ingests users and groups from Microsoft Graph one page at a time using the - catalog's incremental ingestion framework. Unlike - `MicrosoftGraphOrgEntityProvider`, this module never holds the full dataset - in memory and resumes from its cursor on pod restart. - - Requires `@backstage/plugin-catalog-backend-module-incremental-ingestion` to - be installed in the backend. diff --git a/plugins/catalog-backend-module-msgraph-incremental/package.json b/plugins/catalog-backend-module-msgraph-incremental/package.json index e84a57f403..e9b025e9ad 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/package.json +++ b/plugins/catalog-backend-module-msgraph-incremental/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/plugin-catalog-backend-module-msgraph-incremental", - "version": "0.1.0", + "version": "0.0.0", "description": "A Backstage catalog backend module that incrementally ingests users and groups from Microsoft Graph", "backstage": { "role": "backend-plugin-module", diff --git a/plugins/catalog-backend-module-msgraph-incremental/report-alpha.api.md b/plugins/catalog-backend-module-msgraph-incremental/report-alpha.api.md index ed6376fb04..6cf15e34d4 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/report-alpha.api.md +++ b/plugins/catalog-backend-module-msgraph-incremental/report-alpha.api.md @@ -3,96 +3,5 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts -import { BackendFeature } from '@backstage/backend-plugin-api'; -import { Config } from '@backstage/config'; -import { EntityIteratorResult } from '@backstage/plugin-catalog-backend-module-incremental-ingestion'; -import { ExtensionPoint } from '@backstage/backend-plugin-api'; -import { GroupTransformer } from '@backstage/plugin-catalog-backend-module-msgraph'; -import { IncrementalEntityProvider } from '@backstage/plugin-catalog-backend-module-incremental-ingestion'; -import { LoggerService } from '@backstage/backend-plugin-api'; -import { MicrosoftGraphClient } from '@backstage/plugin-catalog-backend-module-msgraph'; -import { MicrosoftGraphProviderConfig } from '@backstage/plugin-catalog-backend-module-msgraph'; -import { OrganizationTransformer } from '@backstage/plugin-catalog-backend-module-msgraph'; -import { ProviderConfigTransformer } from '@backstage/plugin-catalog-backend-module-msgraph'; -import { UserTransformer } from '@backstage/plugin-catalog-backend-module-msgraph'; -// @public -const catalogModuleMicrosoftGraphIncrementalEntityProvider: BackendFeature; -export { catalogModuleMicrosoftGraphIncrementalEntityProvider }; -export default catalogModuleMicrosoftGraphIncrementalEntityProvider; - -// @public -export class MicrosoftGraphIncrementalEntityProvider - implements IncrementalEntityProvider -{ - constructor(options: { - id: string; - provider: MicrosoftGraphProviderConfig; - logger: LoggerService; - userTransformer?: UserTransformer; - groupTransformer?: GroupTransformer; - organizationTransformer?: OrganizationTransformer; - providerConfigTransformer?: ProviderConfigTransformer; - }); - around(burst: (context: MSGraphContext) => Promise): Promise; - static fromConfig( - configRoot: Config, - options: MicrosoftGraphIncrementalEntityProviderOptions, - ): MicrosoftGraphIncrementalEntityProvider[]; - getProviderName(): string; - next( - input: MSGraphContext, - cursor?: MSGraphCursor, - ): Promise>; -} - -// @public -export interface MicrosoftGraphIncrementalEntityProviderOptions { - groupTransformer?: GroupTransformer | Record; - logger: LoggerService; - organizationTransformer?: - | OrganizationTransformer - | Record; - providerConfigTransformer?: - | ProviderConfigTransformer - | Record; - userTransformer?: UserTransformer | Record; -} - -// @public -export const microsoftGraphIncrementalEntityProviderTransformExtensionPoint: ExtensionPoint; - -// @public -export interface MicrosoftGraphIncrementalEntityProviderTransformsExtensionPoint { - setGroupTransformer( - transformer: GroupTransformer | Record, - ): void; - setOrganizationTransformer( - transformer: - | OrganizationTransformer - | Record, - ): void; - setProviderConfigTransformer( - transformer: - | ProviderConfigTransformer - | Record, - ): void; - setUserTransformer( - transformer: UserTransformer | Record, - ): void; -} - -// @public -export type MSGraphContext = { - client: MicrosoftGraphClient; - provider: MicrosoftGraphProviderConfig; -}; - -// @public -export type MSGraphCursor = { - phase: 'users' | 'groups'; - nextLink?: string; -}; - -// (No @packageDocumentation comment for this package) ``` diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts index 630b246f81..faf8d1f4a7 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.test.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Backstage Authors + * Copyright 2026 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts index dedc94b13b..57cbc16c66 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/MicrosoftGraphIncrementalEntityProvider.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Backstage Authors + * Copyright 2026 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/alpha.ts b/plugins/catalog-backend-module-msgraph-incremental/src/alpha.ts index cb99b69c6a..e512e4ba83 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/alpha.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/alpha.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Backstage Authors + * Copyright 2026 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,6 +14,4 @@ * limitations under the License. */ -// Re-export everything from the main entry point while the package is in alpha. -export * from './index'; -export { default } from './index'; +export {}; diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.test.ts b/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.test.ts index 93dd290f2a..a103787f0d 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.test.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.test.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Backstage Authors + * Copyright 2026 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.ts b/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.ts index 2397ba46b9..61b78e4c64 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/clientHelpers.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Backstage Authors + * Copyright 2026 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/index.ts b/plugins/catalog-backend-module-msgraph-incremental/src/index.ts index dbae9a4f2c..37ad8b96c6 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/index.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/index.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Backstage Authors + * Copyright 2026 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.test.ts b/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.test.ts index a30388bae1..c7ca22fe48 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.test.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.test.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Backstage Authors + * Copyright 2026 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.ts b/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.ts index 674ce7a009..3ab51aa9b0 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/module/catalogModuleMicrosoftGraphIncrementalEntityProvider.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Backstage Authors + * Copyright 2026 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/module/index.ts b/plugins/catalog-backend-module-msgraph-incremental/src/module/index.ts index 5d5e3710a4..988418dd9b 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/module/index.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/module/index.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 The Backstage Authors + * Copyright 2026 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From fbc2c76d3976e2c76f7268eeebf54266f4a02f9a Mon Sep 17 00:00:00 2001 From: pillaris Date: Tue, 28 Apr 2026 10:14:28 +0530 Subject: [PATCH 17/20] fix: update alpha API report to match API Extractor output The manually-written empty report-alpha.api.md didn't match what API Extractor generates for an empty alpha entry point. Use the correct generated output to pass the CI api-reports check. Co-Authored-By: Claude Sonnet 4.6 Signed-off-by: pillaris --- .../report-alpha.api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-msgraph-incremental/report-alpha.api.md b/plugins/catalog-backend-module-msgraph-incremental/report-alpha.api.md index 6cf15e34d4..88b085c8c5 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/report-alpha.api.md +++ b/plugins/catalog-backend-module-msgraph-incremental/report-alpha.api.md @@ -3,5 +3,5 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts - +// (No @packageDocumentation comment for this package) ``` From 3a0dfd0031e6302ac3e2d1095e374391cf8f8529 Mon Sep 17 00:00:00 2001 From: pillaris Date: Tue, 28 Apr 2026 11:07:45 +0530 Subject: [PATCH 18/20] fix: export BackendFeature from alpha entrypoint and regenerate API report Re-export catalogModuleMicrosoftGraphIncrementalEntityProvider as @alpha default from src/alpha.ts, matching the pattern of catalog-backend-module-msgraph. Add default export alias in module/index.ts and regenerate report-alpha.api.md. Signed-off-by: pillaris Co-Authored-By: Claude Sonnet 4.6 --- .../report-alpha.api.md | 6 ++++++ .../catalog-backend-module-msgraph-incremental/src/alpha.ts | 6 +++++- .../src/module/index.ts | 1 + 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/plugins/catalog-backend-module-msgraph-incremental/report-alpha.api.md b/plugins/catalog-backend-module-msgraph-incremental/report-alpha.api.md index 88b085c8c5..26e5104aff 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/report-alpha.api.md +++ b/plugins/catalog-backend-module-msgraph-incremental/report-alpha.api.md @@ -3,5 +3,11 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { BackendFeature } from '@backstage/backend-plugin-api'; + +// @alpha (undocumented) +const _feature: BackendFeature; +export default _feature; + // (No @packageDocumentation comment for this package) ``` diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/alpha.ts b/plugins/catalog-backend-module-msgraph-incremental/src/alpha.ts index e512e4ba83..51ebc3f1f3 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/alpha.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/alpha.ts @@ -14,4 +14,8 @@ * limitations under the License. */ -export {}; +import { default as feature } from './module'; + +/** @alpha */ +const _feature = feature; +export default _feature; diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/module/index.ts b/plugins/catalog-backend-module-msgraph-incremental/src/module/index.ts index 988418dd9b..5a96e88c3a 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/module/index.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/module/index.ts @@ -15,6 +15,7 @@ */ export { + catalogModuleMicrosoftGraphIncrementalEntityProvider as default, catalogModuleMicrosoftGraphIncrementalEntityProvider, microsoftGraphIncrementalEntityProviderTransformExtensionPoint, type MicrosoftGraphIncrementalEntityProviderTransformsExtensionPoint, From fec21115a3b0415fb0b48b8e38f3ec8cfda42ed6 Mon Sep 17 00:00:00 2001 From: pillaris Date: Tue, 28 Apr 2026 15:09:26 +0530 Subject: [PATCH 19/20] revert: keep alpha entrypoint empty per maintainer guidance Net-new package does not need alpha exports per awanlin's review. Revert alpha.ts to export {}, remove default alias from module/index.ts, and restore empty report-alpha.api.md. Signed-off-by: pillaris Co-Authored-By: Claude Sonnet 4.6 --- .../report-alpha.api.md | 6 ------ .../catalog-backend-module-msgraph-incremental/src/alpha.ts | 6 +----- .../src/module/index.ts | 1 - 3 files changed, 1 insertion(+), 12 deletions(-) diff --git a/plugins/catalog-backend-module-msgraph-incremental/report-alpha.api.md b/plugins/catalog-backend-module-msgraph-incremental/report-alpha.api.md index 26e5104aff..88b085c8c5 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/report-alpha.api.md +++ b/plugins/catalog-backend-module-msgraph-incremental/report-alpha.api.md @@ -3,11 +3,5 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts -import { BackendFeature } from '@backstage/backend-plugin-api'; - -// @alpha (undocumented) -const _feature: BackendFeature; -export default _feature; - // (No @packageDocumentation comment for this package) ``` diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/alpha.ts b/plugins/catalog-backend-module-msgraph-incremental/src/alpha.ts index 51ebc3f1f3..e512e4ba83 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/alpha.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/alpha.ts @@ -14,8 +14,4 @@ * limitations under the License. */ -import { default as feature } from './module'; - -/** @alpha */ -const _feature = feature; -export default _feature; +export {}; diff --git a/plugins/catalog-backend-module-msgraph-incremental/src/module/index.ts b/plugins/catalog-backend-module-msgraph-incremental/src/module/index.ts index 5a96e88c3a..988418dd9b 100644 --- a/plugins/catalog-backend-module-msgraph-incremental/src/module/index.ts +++ b/plugins/catalog-backend-module-msgraph-incremental/src/module/index.ts @@ -15,7 +15,6 @@ */ export { - catalogModuleMicrosoftGraphIncrementalEntityProvider as default, catalogModuleMicrosoftGraphIncrementalEntityProvider, microsoftGraphIncrementalEntityProviderTransformExtensionPoint, type MicrosoftGraphIncrementalEntityProviderTransformsExtensionPoint, From e14f514b2df33195d3df5e6a02ec3dc11b65e658 Mon Sep 17 00:00:00 2001 From: pillaris Date: Wed, 29 Apr 2026 13:13:46 +0530 Subject: [PATCH 20/20] chore: re-trigger CI Signed-off-by: pillaris Co-Authored-By: Claude Sonnet 4.6