paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(catalog/msgraph): Add option to configure schedule via app-config.yaml

Browse Source 
Relates-to: PR #13859
Relates-to: PR #14034
Signed-off-by: Patrick Jungermann <Patrick.Jungermann@gmail.com>
This commit is contained in:
Patrick Jungermann
2022-10-07 15:23:48 +02:00
parent 6890d78303
commit 8d1a5e08ca
10 changed files with 312 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/dirty-birds-burn.md
+8
View File
@@ -0,0 +1,8 @@
---
'@backstage/plugin-catalog-backend-module-msgraph': patch
---
`MicrosoftGraphOrgEntityProvider`: Add option to configure schedule via `app-config.yaml` instead of in code.
Please find how to configure the schedule at the config at
https://github.com/backstage/backstage/tree/master/plugins/catalog-backend-module-msgraph#readme
plugins/catalog-backend-module-msgraph/README.md
+19 -7
View File
@@ -85,6 +85,13 @@ catalog:
# in order to add extra information to your groups that can be used on your custom groupTransformers
# See https://docs.microsoft.com/en-us/graph/api/resources/schemaextension?view=graph-rest-1.0
select: ['id', 'displayName', 'description']
schedule: # optional; same options as in TaskScheduleDefinition
# supports cron, ISO duration, "human duration" as used in code
frequency: { hours: 1 }
# supports ISO duration, "human duration" as used in code
timeout: { minutes: 50 }
# supports ISO duration, "human duration" as used in code
initialDelay: { seconds: 15},
```
`user.filter` and `userGroupMember.filter` are mutually exclusive, only one can be provided. If both are provided, an error will be thrown.
@@ -116,13 +123,21 @@ yarn add --cwd packages/backend @backstage/plugin-catalog-backend-module-msgraph
+ builder.addEntityProvider(
+ MicrosoftGraphOrgEntityProvider.fromConfig(env.config, {
+ logger: env.logger,
+ scheduler,
+ }),
+ );
```
Instead of configuring the refresh schedule inside the config (per provider instance),
you can define it in code (for all of them):
```diff
- scheduler,
+ schedule: env.scheduler.createScheduledTaskRunner({
+ frequency: { hours: 1 },
+ timeout: { minutes: 50 },
+ initialDelay: { seconds: 15}
+ initialDelay: { seconds: 15},
+ }),
+ }),
+ );
```
## Customize the Processor or Entity Provider
@@ -161,10 +176,7 @@ export async function myGroupTransformer(
builder.addEntityProvider(
MicrosoftGraphOrgEntityProvider.fromConfig(env.config, {
logger: env.logger,
schedule: env.scheduler.createScheduledTaskRunner({
frequency: { minutes: 5 },
timeout: { minutes: 3 },
}),
scheduler,
+ groupTransformer: myGroupTransformer,
}),
);
plugins/catalog-backend-module-msgraph/api-report.md
+5 -1
View File
@@ -12,8 +12,10 @@ import { GroupEntity } from '@backstage/catalog-model';
import { LocationSpec } from '@backstage/plugin-catalog-backend';
import { Logger } from 'winston';
import * as MicrosoftGraph from '@microsoft/microsoft-graph-types';
import { PluginTaskScheduler } from '@backstage/backend-tasks';
import { Response as Response_2 } from 'node-fetch';
import { TaskRunner } from '@backstage/backend-tasks';
import { TaskScheduleDefinition } from '@backstage/backend-tasks';
import { TokenCredential } from '@azure/identity';
import { UserEntity } from '@backstage/catalog-model';
@@ -147,7 +149,8 @@ export type MicrosoftGraphOrgEntityProviderOptions =
| MicrosoftGraphOrgEntityProviderLegacyOptions
| {
logger: Logger;
schedule: 'manual' | TaskRunner;
schedule?: 'manual' | TaskRunner;
scheduler?: PluginTaskScheduler;
userTransformer?: UserTransformer | Record<string, UserTransformer>;
groupTransformer?: GroupTransformer | Record<string, GroupTransformer>;
organizationTransformer?:
@@ -202,6 +205,7 @@ export type MicrosoftGraphProviderConfig = {
groupSearch?: string;
groupSelect?: string[];
queryMode?: 'basic' | 'advanced';
schedule?: TaskScheduleDefinition;
};
// @public
plugins/catalog-backend-module-msgraph/config.d.ts
Vendored
+13 -9
View File
@@ -14,14 +14,10 @@
* limitations under the License.
*/
import { TaskScheduleDefinitionConfig } from '@backstage/backend-tasks';
export interface Config {
/**
* Configuration options for the catalog plugin.
*/
catalog?: {
/**
* List of processor-specific options and attributes
*/
processors?: {
/**
* MicrosoftGraphOrgReaderProcessor configuration
@@ -109,9 +105,7 @@ export interface Config {
}>;
};
};
/**
* List of provider-specific options and attributes
*/
providers?: {
/**
* MicrosoftGraphOrgEntityProvider configuration.
@@ -209,6 +203,11 @@ export interface Config {
*/
search?: string;
};
/**
* (Optional) TaskScheduleDefinition for the refresh.
*/
schedule?: TaskScheduleDefinitionConfig;
}
| Record<
string,
@@ -292,6 +291,11 @@ export interface Config {
*/
search?: string;
};
/**
* (Optional) TaskScheduleDefinition for the refresh.
*/
schedule?: TaskScheduleDefinitionConfig;
}
>;
};
plugins/catalog-backend-module-msgraph/package.json
+1
View File
@@ -51,6 +51,7 @@
"@backstage/backend-test-utils": "workspace:^",
"@backstage/cli": "workspace:^",
"@types/lodash": "^4.14.151",
"luxon": "^3.0.0",
"msw": "^0.47.0"
},
"files": [
plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.test.ts
+13
View File
@@ -15,6 +15,7 @@
*/
import { ConfigReader } from '@backstage/config';
import { Duration } from 'luxon';
import { readMicrosoftGraphConfig, readProviderConfigs } from './config';
describe('readMicrosoftGraphConfig', () => {
@@ -172,6 +173,12 @@ describe('readProviderConfigs', () => {
filter: 'securityEnabled eq false',
select: ['id', 'displayName', 'description'],
},
schedule: {
frequency: 'PT30M',
timeout: {
minutes: 3,
},
},
},
},
},
@@ -192,6 +199,12 @@ describe('readProviderConfigs', () => {
groupExpand: 'member',
groupSelect: ['id', 'displayName', 'description'],
groupFilter: 'securityEnabled eq false',
schedule: {
frequency: Duration.fromISO('PT30M'),
timeout: {
minutes: 3,
},
},
},
];
expect(actual).toEqual(expected);
plugins/catalog-backend-module-msgraph/src/microsoftGraph/config.ts
+14
View File
@@ -14,6 +14,10 @@
* limitations under the License.
*/
import {
readTaskScheduleDefinitionFromConfig,
TaskScheduleDefinition,
} from '@backstage/backend-tasks';
import { Config } from '@backstage/config';
import { trimEnd } from 'lodash';
@@ -121,6 +125,11 @@ export type MicrosoftGraphProviderConfig = {
* Some features like `$expand` are not available for advanced queries, though.
*/
queryMode?: 'basic' | 'advanced';
/**
* Schedule configuration for refresh tasks.
*/
schedule?: TaskScheduleDefinition;
};
/**
@@ -296,6 +305,10 @@ export function readProviderConfig(
throw new Error(`clientId must be provided when clientSecret is defined.`);
}
const schedule = config.has('schedule')
? readTaskScheduleDefinitionFromConfig(config.getConfig('schedule'))
: undefined;
return {
id,
target,
@@ -312,5 +325,6 @@ export function readProviderConfig(
queryMode,
userGroupMemberFilter,
userGroupMemberSearch,
schedule,
};
}
plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.test.ts
+207 -58
View File
@@ -14,6 +14,11 @@
* limitations under the License.
*/
import { getVoidLogger } from '@backstage/backend-common';
import {
PluginTaskScheduler,
TaskInvocationDefinition,
TaskRunner,
} from '@backstage/backend-tasks';
import { ConfigReader } from '@backstage/config';
import {
ANNOTATION_LOCATION,
@@ -43,10 +48,21 @@ const readMicrosoftGraphOrgMocked = readMicrosoftGraphOrg as jest.Mock<
Promise<{ users: UserEntity[]; groups: GroupEntity[] }>
>;
describe('MicrosoftGraphOrgEntityProvider', () => {
afterEach(() => jest.resetAllMocks());
class PersistingTaskRunner implements TaskRunner {
private tasks: TaskInvocationDefinition[] = [];
it('should apply mutation', async () => {
getTasks() {
return this.tasks;
}
run(task: TaskInvocationDefinition): Promise<void> {
this.tasks.push(task);
return Promise.resolve(undefined);
}
}
describe('MicrosoftGraphOrgEntityProvider', () => {
beforeEach(() => {
jest
.spyOn(MicrosoftGraphClient, 'create')
.mockReturnValue({} as unknown as MicrosoftGraphClient);
@@ -78,8 +94,65 @@ describe('MicrosoftGraphOrgEntityProvider', () => {
},
],
});
});
const config = {
afterEach(() => jest.resetAllMocks());
const logger = getVoidLogger();
const taskRunner = new PersistingTaskRunner();
const scheduler = {
createScheduledTaskRunner: (_: any) => taskRunner,
} as unknown as PluginTaskScheduler;
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
const expectedMutation = {
entities: [
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'User',
metadata: {
annotations: {
'backstage.io/managed-by-location': 'msgraph:customProviderId/u1',
'backstage.io/managed-by-origin-location':
'msgraph:customProviderId/u1',
},
name: 'u1',
},
spec: {
memberOf: [],
},
},
locationKey: 'msgraph-org-provider:customProviderId',
},
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Group',
metadata: {
annotations: {
'backstage.io/managed-by-location': 'msgraph:customProviderId/g1',
'backstage.io/managed-by-origin-location':
'msgraph:customProviderId/g1',
},
name: 'g1',
},
spec: {
children: [],
type: 'team',
},
},
locationKey: 'msgraph-org-provider:customProviderId',
},
],
type: 'full',
};
it('should apply mutation - manual', async () => {
const config = new ConfigReader({
catalog: {
providers: {
microsoftGraphOrg: {
@@ -92,67 +165,143 @@ describe('MicrosoftGraphOrgEntityProvider', () => {
},
},
},
};
const entityProviderConnection: EntityProviderConnection = {
applyMutation: jest.fn(),
refresh: jest.fn(),
};
const provider = MicrosoftGraphOrgEntityProvider.fromConfig(
new ConfigReader(config),
{
logger: getVoidLogger(),
schedule: 'manual',
},
)[0];
provider.connect(entityProviderConnection);
});
const provider = MicrosoftGraphOrgEntityProvider.fromConfig(config, {
logger,
schedule: 'manual',
})[0];
await provider.connect(entityProviderConnection);
await provider.read();
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({
entities: [
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'User',
metadata: {
annotations: {
'backstage.io/managed-by-location':
'msgraph:customProviderId/u1',
'backstage.io/managed-by-origin-location':
'msgraph:customProviderId/u1',
},
name: 'u1',
},
spec: {
memberOf: [],
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith(
expectedMutation,
);
});
it('should apply mutation - schedule', async () => {
const config = new ConfigReader({
catalog: {
providers: {
microsoftGraphOrg: {
customProviderId: {
target: 'target',
tenantId: 'tenantId',
clientId: 'clientId',
clientSecret: 'clientSecret',
},
},
locationKey: 'msgraph-org-provider:customProviderId',
},
{
entity: {
apiVersion: 'backstage.io/v1alpha1',
kind: 'Group',
metadata: {
annotations: {
'backstage.io/managed-by-location':
'msgraph:customProviderId/g1',
'backstage.io/managed-by-origin-location':
'msgraph:customProviderId/g1',
},
name: 'g1',
},
spec: {
children: [],
type: 'team',
},
},
locationKey: 'msgraph-org-provider:customProviderId',
},
],
type: 'full',
},
});
const provider = MicrosoftGraphOrgEntityProvider.fromConfig(config, {
logger,
schedule: taskRunner,
})[0];
expect(provider.getProviderName()).toEqual(
'MicrosoftGraphOrgEntityProvider:customProviderId',
);
await provider.connect(entityProviderConnection);
const taskDef = taskRunner.getTasks()[0];
expect(taskDef.id).toEqual(
'MicrosoftGraphOrgEntityProvider:customProviderId:refresh',
);
await (taskDef.fn as () => Promise<void>)();
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith(
expectedMutation,
);
});
it('should apply mutation - scheduler', async () => {
const config = new ConfigReader({
catalog: {
providers: {
microsoftGraphOrg: {
customProviderId: {
target: 'target',
tenantId: 'tenantId',
clientId: 'clientId',
clientSecret: 'clientSecret',
schedule: {
frequency: 'PT30M',
timeout: 'PT3M',
},
},
},
},
},
});
const provider = MicrosoftGraphOrgEntityProvider.fromConfig(config, {
logger,
scheduler,
})[0];
expect(provider.getProviderName()).toEqual(
'MicrosoftGraphOrgEntityProvider:customProviderId',
);
await provider.connect(entityProviderConnection);
const taskDef = taskRunner.getTasks()[0];
expect(taskDef.id).toEqual(
'MicrosoftGraphOrgEntityProvider:customProviderId:refresh',
);
await (taskDef.fn as () => Promise<void>)();
expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith(
expectedMutation,
);
});
it('fail without schedule and scheduler', () => {
const config = new ConfigReader({
catalog: {
providers: {
microsoftGraphOrg: {
customProviderId: {
target: 'target',
tenantId: 'tenantId',
clientId: 'clientId',
clientSecret: 'clientSecret',
},
},
},
},
});
expect(() =>
MicrosoftGraphOrgEntityProvider.fromConfig(config, {
logger,
}),
).toThrow('Either schedule or scheduler must be provided');
});
it('fail with scheduler but no schedule config', () => {
const config = new ConfigReader({
catalog: {
providers: {
microsoftGraphOrg: {
customProviderId: {
target: 'target',
tenantId: 'tenantId',
clientId: 'clientId',
clientSecret: 'clientSecret',
},
},
},
},
});
expect(() =>
MicrosoftGraphOrgEntityProvider.fromConfig(config, {
logger,
scheduler,
}),
).toThrow(
'No schedule provided neither via code nor config for MicrosoftGraphOrgEntityProvider:customProviderId',
);
});
});
plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgEntityProvider.ts
+31 -12
View File
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { TaskRunner } from '@backstage/backend-tasks';
import { PluginTaskScheduler, TaskRunner } from '@backstage/backend-tasks';
import {
ANNOTATION_LOCATION,
ANNOTATION_ORIGIN_LOCATION,
@@ -67,7 +67,13 @@ export type MicrosoftGraphOrgEntityProviderOptions =
* {@link @backstage/backend-tasks#PluginTaskScheduler.createScheduledTaskRunner}
* to enable automatic scheduling of tasks.
*/
schedule: 'manual' | TaskRunner;
schedule?: 'manual' | TaskRunner;
/**
* Scheduler used to schedule refreshes based on
* the schedule config.
*/
scheduler?: PluginTaskScheduler;
/**
* The function that transforms a user entry in msgraph to an entity.
@@ -168,6 +174,10 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider {
];
}
if (!options.schedule && !options.scheduler) {
throw new Error('Either schedule or scheduler must be provided.');
}
function getTransformer<T extends Function>(
id: string,
transformers?: T | Record<string, T>,
@@ -180,6 +190,16 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider {
}
return readProviderConfigs(configRoot).map(providerConfig => {
if (!options.schedule && !providerConfig.schedule) {
throw new Error(
`No schedule provided neither via code nor config for MicrosoftGraphOrgEntityProvider:${providerConfig.id}.`,
);
}
const taskRunner =
options.schedule ??
options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!);
const provider = new MicrosoftGraphOrgEntityProvider({
id: providerConfig.id,
provider: providerConfig,
@@ -197,7 +217,10 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider {
options.organizationTransformer,
),
});
provider.schedule(options.schedule);
if (taskRunner !== 'manual') {
provider.schedule(taskRunner);
}
return provider;
});
@@ -238,7 +261,9 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider {
provider,
});
result.schedule(options.schedule);
if (options.schedule !== 'manual') {
result.schedule(options.schedule);
}
return result;
}
@@ -311,16 +336,10 @@ export class MicrosoftGraphOrgEntityProvider implements EntityProvider {
markCommitComplete();
}
private schedule(
schedule: MicrosoftGraphOrgEntityProviderOptions['schedule'],
) {
if (schedule === 'manual') {
return;
}
private schedule(taskRunner: TaskRunner) {
this.scheduleFn = async () => {
const id = `${this.getProviderName()}:refresh`;
await schedule.run({
await taskRunner.run({
id,
fn: async () => {
const logger = this.options.logger.child({
yarn.lock
+1
View File
@@ -4652,6 +4652,7 @@ __metadata:
"@types/lodash": ^4.14.151
"@types/node-fetch": ^2.5.12
lodash: ^4.17.21
luxon: ^3.0.0
msw: ^0.47.0
node-fetch: ^2.6.7
p-limit: ^3.0.2