paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 53 Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'refs/remotes/origin/master' into checkpoints-doc

Browse Source
This commit is contained in:
bnechyporenko
2024-07-02 19:15:21 +02:00
parent b29b39302c 0f5c956bde
commit 45aac69646
285 changed files with 7216 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/faq/technical.md
+1 -1
View File
@@ -116,7 +116,7 @@ This makes it easier to create, find, and update documentation.
[TechDocs is now open source.](https://backstage.io/docs/features/techdocs/)
(See also:
"[Will Spotify's internal plugins be open sourced, too?](https://backstage.io/docs/faq/product#will-spotifys-internal-plugins-be-open-sourced-too)"
above)
above).
### Are you planning to have plugins baked into the repo? Or should they be developed in separate repos?
docs/features/software-catalog/external-integrations.md
+121 -15
View File
@@ -84,6 +84,7 @@ import {
EntityProvider,
EntityProviderConnection,
} from '@backstage/plugin-catalog-node';
import { SchedulerServiceTaskRunner } from '@backstage/backend-plugin-api';
/**
* Provides entities from fictional frobs service.
@@ -92,11 +93,17 @@ export class FrobsProvider implements EntityProvider {
private readonly env: string;
private readonly reader: UrlReader;
private connection?: EntityProviderConnection;
private taskRunner: SchedulerServiceTaskRunner;
/** [1] */
constructor(env: string, reader: UrlReader) {
constructor(
env: string,
reader: UrlReader,
taskRunner: SchedulerServiceTaskRunner,
) {
this.env = env;
this.reader = reader;
this.taskRunner = taskRunner;
}
/** [2] */
@@ -107,6 +114,12 @@ export class FrobsProvider implements EntityProvider {
/** [3] */
async connect(connection: EntityProviderConnection): Promise<void> {
this.connection = connection;
this.taskRunner.run({
id: this.getProviderName(),
fn: async () => {
await this.run();
},
});
}
/** [4] */
@@ -248,24 +261,17 @@ export default async function createPlugin(
): Promise<Router> {
const builder = CatalogBuilder.create(env);
/* highlight-add-start */
const frobs = new FrobsProvider('production', env.reader);
const taskRunner = env.scheduler.createScheduledTaskRunner({
frequency: { minutes: 30 },
timeout: { minutes: 10 },
});
const frobs = new FrobsProvider('production', env.reader, taskRunner);
builder.addEntityProvider(frobs);
/* highlight-add-end */
const { processingEngine, router } = await builder.build();
await processingEngine.start();
/* highlight-add-start */
await env.scheduler.scheduleTask({
id: 'run_frobs_refresh',
fn: async () => {
await frobs.run();
},
frequency: { minutes: 30 },
timeout: { minutes: 10 },
});
/* highlight-add-end */
// ..
}
```
@@ -300,9 +306,17 @@ export const catalogModuleFrobsProvider = createBackendModule({
deps: {
catalog: catalogProcessingExtensionPoint,
reader: coreServices.urlReader,
/* highlight-add-start */
scheduler: coreServices.scheduler,
/* highlight-add-end */
},
async init({ catalog, reader }) {
catalog.addEntityProvider(new FrobsProvider('dev', reader));
async init({ catalog, reader, scheduler }) {
const taskRunner = scheduler.createScheduledTaskRunner({
frequency: { minutes: 30 },
timeout: { minutes: 10 },
});
const frobs = new FrobsProvider('dev', reader, taskRunner);
catalog.addEntityProvider(frobs);
},
});
},
@@ -318,6 +332,98 @@ backend.add(catalogModuleFrobsProvider);
backend.start();
```
#### Follow-up: Config Defined Schedule
If you want to go a step further and increase the configurability of your new `FrobsProvider`, you can define the schedule that the task runs at in `app-config.yaml` instead of requiring code changes to adjust.
```yaml title="app-config.yaml"
catalog:
providers:
frobs-provider:
schedule:
initialDelay: { seconds: 30 }
frequency: { hours: 1 }
timeout: { minutes: 50 }
```
This approach will also allow you to customize the schedule per environment
#### New Backend
```ts title="packages/backend/src/index.ts"
import {
SchedulerServiceTaskScheduleDefinition,
/* highlight-add-start */
readSchedulerServiceTaskScheduleDefinitionFromConfig,
/* highlight-add-end */
} from '@backstage/backend-plugin-api';
export const catalogModuleFrobsProvider = createBackendModule({
pluginId: 'catalog',
moduleId: 'frobs-provider',
register(env) {
env.registerInit({
deps: {
// ... other deps
/* highlight-add-start */
rootConfig: coreServices.rootConfig,
/* highlight-add-end */
},
async init({ catalog, reader, scheduler, rootConfig }) {
/* highlight-add-start */
const config = rootConfig.getConfig('catalog.providers.frobs-provider'); // Generally, catalog config goes under catalog.providers.pluginId
// Add a default schedule if you don't define one in config.
const schedule = config.has('schedule')
? readSchedulerServiceTaskScheduleDefinitionFromConfig(
config.getConfig('schedule'),
)
: {
frequency: { minutes: 30 },
timeout: { minutes: 10 },
};
const taskRunner: SchedulerServiceTaskRunner =
scheduler.createScheduledTaskRunner(schedule);
/* highlight-add-end */
// rest of your code
},
});
},
});
```
#### Old Backend
```ts title="packages/backend/src/plugins/catalog.ts"
/* highlight-add-next-line */
import { FrobsProvider } from '../path/to/class';
import {
/* highlight-add-start */
readSchedulerServiceTaskScheduleDefinitionFromConfig,
/* highlight-add-end */
} from '@backstage/backend-plugin-api';
export default async function createPlugin(
env: PluginEnvironment,
): Promise<Router> {
/* highlight-add-start */
const config = env.config.getConfig('catalog.providers.frobs-provider'); // Generally, catalog config goes under catalog.providers.pluginId
// Add a default schedule if you don't define one in config.
const schedule = config.has('schedule')
? readSchedulerServiceTaskScheduleDefinitionFromConfig(
config.getConfig('schedule'),
)
: {
frequency: { minutes: 30 },
timeout: { minutes: 10 },
};
const taskRunner = env.scheduler.createScheduledTaskRunner(schedule);
/* highlight-add-end */
// ..
}
```
### Example User Entity Provider
If you have a 3rd party entity provider such as an internal HR system that you wish to use you are not limited to using our entity providers, (or simply wish to add to existing entity providers with your own data).
docs/integrations/aws-s3/discovery--old.md
+89
View File
@@ -0,0 +1,89 @@
---
id: discovery--old
title: AWS S3 Discovery
sidebar_label: Discovery
# prettier-ignore
description: Automatically discovering catalog entities from an AWS S3 Bucket
---
:::info
This documentation is written for the old backend which has been replaced by [the new backend system](../../backend-system/index.md), being the default since Backstage [version 1.24](../../releases/v1.24.0.md). If have migrated to the new backend system, you may want to read [its own article](./discovery.md) instead. Otherwise, [consider migrating](../../backend-system/building-backends/08-migrating.md)!
:::
The AWS S3 integration has a special entity provider for discovering catalog
entities located in an S3 Bucket. If you have a bucket that contains multiple
catalog files, and you want to automatically discover them, you can use this
provider. The provider will crawl your S3 bucket and register entities
matching the configured path. This can be useful as an alternative to static
locations or manually adding things to the catalog.
To use the entity provider, you'll need an AWS S3 integration
[set up](locations.md) with `accessKeyId` and `secretAccessKey`, and/or
a `roleArn` or none of these (e.g., profile- or instance-based credentials).
At production deployments, you likely manage these with the permissions attached
to your instance.
In your configuration, you add a provider config per bucket:
```yaml
# app-config.yaml
catalog:
providers:
awsS3:
yourProviderId: # identifies your dataset / provider independent of config changes
bucketName: sample-bucket
prefix: prefix/ # optional
region: us-east-2 # optional, uses the default region otherwise
schedule: # same options as in TaskScheduleDefinition
# supports cron, ISO duration, "human duration" as used in code
frequency: { minutes: 30 }
# supports ISO duration, "human duration" as used in code
timeout: { minutes: 3 }
```
For simple setups, you can omit the provider ID at the config
which has the same effect as using `default` for it.
```yaml
# app-config.yaml
catalog:
providers:
awsS3:
# uses "default" as provider ID
bucketName: sample-bucket
prefix: prefix/ # optional
region: us-east-2 # optional, uses the default region otherwise
schedule: # same options as in TaskScheduleDefinition
# supports cron, ISO duration, "human duration" as used in code
frequency: { minutes: 30 }
# supports ISO duration, "human duration" as used in code
timeout: { minutes: 3 }
```
As this provider is not one of the default providers, you will first need to install
the AWS catalog plugin:
```bash
# From your Backstage root directory
yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-aws
```
Once you've done that, you'll also need to add the segment below to `packages/backend/src/plugins/catalog.ts`:
```ts
/* packages/backend/src/plugins/catalog.ts */
import { AwsS3EntityProvider } from '@backstage/plugin-catalog-backend-module-aws';
const builder = await CatalogBuilder.create(env);
/** ... other processors and/or providers ... */
builder.addEntityProvider(
AwsS3EntityProvider.fromConfig(env.config, {
logger: env.logger,
scheduler: env.scheduler,
}),
);
```
docs/integrations/aws-s3/discovery.md
+11 -15
View File
@@ -6,6 +6,10 @@ sidebar_label: Discovery
description: Automatically discovering catalog entities from an AWS S3 Bucket
---
:::info
This documentation is written for [the new backend system](../../backend-system/index.md) which is the default since Backstage [version 1.24](../../releases/v1.24.0.md). If you are still on the old backend system, you may want to read [its own article](./discovery--old.md) instead, and [consider migrating](../../backend-system/building-backends/08-migrating.md)!
:::
The AWS S3 integration has a special entity provider for discovering catalog
entities located in an S3 Bucket. If you have a bucket that contains multiple
catalog files, and you want to automatically discover them, you can use this
@@ -20,7 +24,7 @@ a `roleArn` or none of these (e.g., profile- or instance-based credentials).
At production deployments, you likely manage these with the permissions attached
to your instance.
At your configuration, you add a provider config per bucket:
In your configuration, you add a provider config per bucket:
```yaml
# app-config.yaml
@@ -67,19 +71,11 @@ the AWS catalog plugin:
yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-aws
```
Once you've done that, you'll also need to add the segment below to `packages/backend/src/plugins/catalog.ts`:
Then update your backend by adding the following line:
```ts
/* packages/backend/src/plugins/catalog.ts */
import { AwsS3EntityProvider } from '@backstage/plugin-catalog-backend-module-aws';
const builder = await CatalogBuilder.create(env);
/** ... other processors and/or providers ... */
builder.addEntityProvider(
AwsS3EntityProvider.fromConfig(env.config, {
logger: env.logger,
scheduler: env.scheduler,
}),
);
```ts title="packages/backend/src/index.ts"
backend.add(import('@backstage/plugin-catalog-backend/alpha'));
/* highlight-add-start */
backend.add(import('@backstage/plugin-catalog-backend-module-aws/alpha'));
/* highlight-add-end */
```
docs/releases/v1.29.0-next.1-changelog.md
+1952
View File
File diff suppressed because it is too large Load Diff