diff --git a/app-config.yaml b/app-config.yaml index 1486e8b3ec..456fd17974 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -257,6 +257,7 @@ catalog: - Location providers: azureBlob: + accountName: ${ACCOUNT_NAME} containerName: ${CONTAINER_NAME} schedule: # same options as in TaskScheduleDefinition # supports cron, ISO duration, "human duration" as used in code diff --git a/docs/integrations/azure-blobStorage/discovery.md b/docs/integrations/azure-blobStorage/discovery.md new file mode 100644 index 0000000000..a62333ff69 --- /dev/null +++ b/docs/integrations/azure-blobStorage/discovery.md @@ -0,0 +1,76 @@ +--- +id: discovery +title: Azure Blob Storage Discovery +sidebar_label: Discovery +# prettier-ignore +description: Automatically discovering catalog entities from an Azure Blob Storage account +--- + +:::info +This documentation is written for [the new backend system](../../backend-system/index.md) which is the default since Backstage [version 1.24](../../releases/v1.24.0.md). If you are still on the old backend system, you may want to read [its own article](./discovery--old.md) instead, and [consider migrating](../../backend-system/building-backends/08-migrating.md)! +::: + +The Azure Blob Storage account integration has a special entity provider for discovering catalog +entities located in a stroage account container. If you have a conatiner that contains multiple +catalog files, and you want to automatically discover them, you can use this +provider. The provider will crawl your Blob Storage account container and register entities +matching the configured path. This can be useful as an alternative to static +locations or manually adding things to the catalog. + +To use the entity provider, you'll need an Azure Blob Storage account integration +[set up](locations.md) with `accountName` and either `aadCredential`, `sasToken`, or `accountKey` + +At production deployments, you likely manage these with the permissions attached +to your instance. + +In your configuration, you add a provider config per bucket: + +```yaml +# app-config.yaml + +catalog: + providers: + azureBlob: + providerId: + accountName: ${ACCOUNT_NAME} + containerName: ${CONTAINER_NAME} + schedule: # same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } +``` + +For simple setups, you can omit the provider ID at the config +which has the same effect as using `default` for it. + +```yaml +# app-config.yaml + +catalog: + providers: + azureBlob: + accountName: ${ACCOUNT_NAME} + containerName: ${CONTAINER_NAME} + schedule: # same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } +``` + +As this provider is not one of the default providers, you will first need to install +the Azure catalog plugin: + +```bash title="From your Backstage root directory" +yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-azure +``` + +Then updated your backend by adding the following line: + +```ts title="packages/backend/src/index.ts" +backend.add(import('@backstage/plugin-catalog-backend')); +/* highlight-add-start */ +backend.add(import('@backstage/plugin-catalog-backend-module-azure')); +/* highlight-add-end */ +``` diff --git a/docs/integrations/azure-blobStorage/locations.md b/docs/integrations/azure-blobStorage/locations.md new file mode 100644 index 0000000000..38d8555c2e --- /dev/null +++ b/docs/integrations/azure-blobStorage/locations.md @@ -0,0 +1,51 @@ +--- +id: locations +sidebar_label: Locations +title: Azure Blob Storage account Locations +# prettier-ignore +description: Setting up an integration with Azure Blob Storage account +--- + +The Azure Blob Storage account integration supports loading catalog entities from an blob storage account container. +Entities can be added to +[static catalog configuration](../../features/software-catalog/configuration.md), +or registered with the +[catalog-import](https://github.com/backstage/backstage/tree/master/plugins/catalog-import) +plugin. + +## Configuration + +To use this integration, add configuration to your `app-config.yaml`: + +Using Azure active directory credentials: + +```yaml +integrations: + azureBlobStorage: + - accountName: ${ACCOUNT_NAME} # required + endpoint: ${CUSTOM_ENDPOINT} # custom endpoint will require either aadCredentials or sasToken + aadCredential: + clientId: ${CLIENT_ID} + tenantId: ${TENANT_ID} + clientSecret: ${CLIENT_SECRET} +``` + +Using Azure storage account SAS token: + +```yaml +integrations: + azureBlobStorage: + - accountName: ${ACCOUNT_NAME} # required + endpoint: ${CUSTOM_ENDPOINT} # custom endpoint will require either aadCredentials or sasToken + sasToken: ${SAS_TOKEN} +``` + +Using Azure storage account access key: + +```yaml +integrations: + azureBlobStorage: + - accountName: ${ACCOUNT_NAME} # required + endpoint: ${CUSTOM_ENDPOINT} # custom endpoint will require either aadCredentials or sasToken + accountKey: ${ACCOUNT_KEY} +``` diff --git a/packages/backend-defaults/report-urlReader.api.md b/packages/backend-defaults/report-urlReader.api.md index f8b8db2954..4813643d68 100644 --- a/packages/backend-defaults/report-urlReader.api.md +++ b/packages/backend-defaults/report-urlReader.api.md @@ -7,6 +7,8 @@ import { AwsCredentialsManager } from '@backstage/integration-aws-node'; import { AwsS3Integration } from '@backstage/integration'; +import { AzureBlobStorageIntergation } from '@backstage/integration'; +import { AzureCredentialsManager } from '@backstage/integration'; import { AzureDevOpsCredentialsProvider } from '@backstage/integration'; import { AzureIntegration } from '@backstage/integration'; import { BitbucketCloudIntegration } from '@backstage/integration'; @@ -60,6 +62,35 @@ export class AwsS3UrlReader implements UrlReaderService { toString(): string; } +// @public +export class AzureBlobStorageUrlReader implements UrlReaderService { + constructor( + credsManager: AzureCredentialsManager, + integration: AzureBlobStorageIntergation, + deps: { + treeResponseFactory: ReadTreeResponseFactory; + }, + ); + // (undocumented) + static factory: ReaderFactory; + // (undocumented) + read(url: string): Promise; + // (undocumented) + readTree( + url: string, + options?: UrlReaderServiceReadTreeOptions, + ): Promise; + // (undocumented) + readUrl( + url: string, + options?: UrlReaderServiceReadUrlOptions, + ): Promise; + // (undocumented) + search(): Promise; + // (undocumented) + toString(): string; +} + // @public export class AzureUrlReader implements UrlReaderService { constructor( diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureBlobStorageUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureBlobStorageUrlReader.ts index 4de5f79f4e..069b772a15 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureBlobStorageUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureBlobStorageUrlReader.ts @@ -54,6 +54,12 @@ export function parseUrl(url: string): { path: string; container: string } { return { path, container }; } + +/** + * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for Azure storage accounts urls. + * + * @public + */ export class AzureBlobStorageUrlReader implements UrlReaderService { static factory: ReaderFactory = ({ config, treeResponseFactory }) => { const integrations = ScmIntegrations.fromConfig(config); @@ -71,7 +77,9 @@ export class AzureBlobStorageUrlReader implements UrlReaderService { ); const predicate = (url: URL) => - url.host.endsWith(integrationConfig.config.host); + url.host.endsWith( + `${integrationConfig.config.accountName}.${integrationConfig.config.host}`, + ); return { reader, predicate }; }); }; @@ -141,16 +149,13 @@ export class AzureBlobStorageUrlReader implements UrlReaderService { const containerClient = await this.createContainerClient(container); const blobClient = containerClient.getBlobClient(path); - const abortController = new AbortController(); - const getBlobOptions: BlobDownloadOptions = { - abortSignal: abortController.signal, + abortSignal: options?.signal, conditions: { ...(etag && { ifNoneMatch: etag }), ...(lastModifiedAfter && { ifModifiedSince: lastModifiedAfter }), }, }; - options?.signal?.addEventListener('abort', () => abortController.abort()); const downloadBlockBlobResponse = await blobClient.download( 0, @@ -189,13 +194,14 @@ export class AzureBlobStorageUrlReader implements UrlReaderService { const blobs = containerClient.listBlobsFlat({ prefix: path }); const responses = []; - const abortController = new AbortController(); + for await (const blob of blobs) { const blobClient = containerClient.getBlobClient(blob.name); - options?.signal?.addEventListener('abort', () => - abortController.abort(), + const downloadBlockBlobResponse = await blobClient.download( + undefined, + undefined, + { abortSignal: options?.signal }, ); - const downloadBlockBlobResponse = await blobClient.download(); responses.push({ data: Readable.from( diff --git a/packages/backend/package.json b/packages/backend/package.json index 34de57574d..6b0ca47f98 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -39,6 +39,7 @@ "@backstage/plugin-auth-backend-module-guest-provider": "workspace:^", "@backstage/plugin-auth-node": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", + "@backstage/plugin-catalog-backend-module-azure": "workspace:^", "@backstage/plugin-catalog-backend-module-backstage-openapi": "workspace:^", "@backstage/plugin-catalog-backend-module-openapi": "workspace:^", "@backstage/plugin-catalog-backend-module-scaffolder-entity-model": "workspace:^", diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index b9130d452f..80537f1751 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -56,5 +56,6 @@ backend.add(searchLoader); backend.add(import('@backstage/plugin-techdocs-backend')); backend.add(import('@backstage/plugin-signals-backend')); backend.add(import('@backstage/plugin-notifications-backend')); +backend.add(import('@backstage/plugin-catalog-backend-module-azure')); backend.start(); diff --git a/packages/integration/report.api.md b/packages/integration/report.api.md index e9b67bb6c1..5129dbb7c2 100644 --- a/packages/integration/report.api.md +++ b/packages/integration/report.api.md @@ -3,9 +3,12 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { AnonymousCredential } from '@azure/storage-blob'; import { Config } from '@backstage/config'; import { ConsumedResponse } from '@backstage/errors'; import { RestEndpointMethodTypes } from '@octokit/rest'; +import { StorageSharedKeyCredential } from '@azure/storage-blob'; +import { TokenCredential } from '@azure/identity'; // @public export class AwsCodeCommitIntegration implements ScmIntegration { @@ -70,6 +73,43 @@ export type AwsS3IntegrationConfig = { externalId?: string; }; +// @public +export type AzureBlobStorageIntegrationConfig = { + accountName?: string; + accountKey?: string; + sasToken?: string; + connectionString?: string; + endpointSuffix?: string; + host: string; + endpoint?: string; + aadCredential?: { + clientId: string; + tenantId: string; + clientSecret: string; + }; +}; + +// @public +export class AzureBlobStorageIntergation implements ScmIntegration { + constructor(integrationConfig: AzureBlobStorageIntegrationConfig); + // (undocumented) + get config(): AzureBlobStorageIntegrationConfig; + // (undocumented) + static factory: ScmIntegrationsFactory; + // (undocumented) + resolveEditUrl(url: string): string; + // (undocumented) + resolveUrl(options: { + url: string; + base: string; + lineNumber?: number | undefined; + }): string; + // (undocumented) + get title(): string; + // (undocumented) + get type(): string; +} + // @public export type AzureClientSecretCredential = AzureCredentialBase & { kind: 'ClientSecret'; @@ -84,6 +124,16 @@ export type AzureCredentialBase = { organizations?: string[]; }; +// @public +export interface AzureCredentialsManager { + // (undocumented) + getCredentials( + accountName: string, + ): Promise< + TokenCredential | StorageSharedKeyCredential | AnonymousCredential + >; +} + // @public export type AzureDevOpsCredential = | AzureClientSecretCredential @@ -257,6 +307,15 @@ export function buildGerritGitilesArchiveUrl( filePath: string, ): string; +// @public +export class DefaultAzureCredentialsManager implements AzureCredentialsManager { + static fromIntegrations( + integrations: ScmIntegrationRegistry, + ): DefaultAzureCredentialsManager; + // (undocumented) + getCredentials(accountName: string): Promise; +} + // @public export class DefaultAzureDevOpsCredentialsProvider implements AzureDevOpsCredentialsProvider @@ -723,6 +782,8 @@ export interface IntegrationsByType { awsS3: ScmIntegrationsGroup; // (undocumented) azure: ScmIntegrationsGroup; + // (undocumented) + azureBlobStorage: ScmIntegrationsGroup; // @deprecated (undocumented) bitbucket: ScmIntegrationsGroup; // (undocumented) @@ -826,6 +887,16 @@ export function readAwsS3IntegrationConfigs( configs: Config[], ): AwsS3IntegrationConfig[]; +// @public +export function readAzureBlobStorageIntegrationConfig( + config: Config, +): AzureBlobStorageIntegrationConfig; + +// @public +export function readAzureBlobStorageIntegrationConfigs( + configs: Config[], +): AzureBlobStorageIntegrationConfig[]; + // @public export function readAzureIntegrationConfig( config: Config, @@ -940,6 +1011,8 @@ export interface ScmIntegrationRegistry awsS3: ScmIntegrationsGroup; // (undocumented) azure: ScmIntegrationsGroup; + // (undocumented) + azureBlobStorage: ScmIntegrationsGroup; // @deprecated (undocumented) bitbucket: ScmIntegrationsGroup; // (undocumented) @@ -973,6 +1046,8 @@ export class ScmIntegrations implements ScmIntegrationRegistry { get awsS3(): ScmIntegrationsGroup; // (undocumented) get azure(): ScmIntegrationsGroup; + // (undocumented) + get azureBlobStorage(): ScmIntegrationsGroup; // @deprecated (undocumented) get bitbucket(): ScmIntegrationsGroup; // (undocumented) diff --git a/packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.ts b/packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.ts index 01e6bbc317..27061f7d9d 100644 --- a/packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.ts +++ b/packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.ts @@ -21,6 +21,11 @@ import { readAzureBlobStorageIntegrationConfigs, } from './config'; +/** + * Microsoft Azure Blob storage based integration. + * + * @public + */ export class AzureBlobStorageIntergation implements ScmIntegration { static factory: ScmIntegrationsFactory = ({ config, diff --git a/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsProvider.ts b/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsProvider.ts index 8971e560ad..3861329300 100644 --- a/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsProvider.ts +++ b/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsProvider.ts @@ -23,26 +23,58 @@ import { AzureBlobStorageIntegrationConfig } from './config'; import { AzureCredentialsManager } from './types'; import { ScmIntegrationRegistry } from '../registry'; +/** + * Default implementation of AzureCredentialsManager that supports multiple Azure Blob Storage integrations. + * @public + */ export class DefaultAzureCredentialsManager implements AzureCredentialsManager { - private config: AzureBlobStorageIntegrationConfig; private cachedCredentials: Map; - constructor(config: AzureBlobStorageIntegrationConfig) { - this.config = config; + private constructor( + private readonly configProviders: Map< + string, + AzureBlobStorageIntegrationConfig + >, + ) { this.cachedCredentials = new Map(); } /** - * Creates an instance of DefaultAzureCredentialsManager from a Backstage Config. + * Creates an instance of DefaultAzureCredentialsManager from a Backstage integration registry. */ static fromIntegrations( - integration: ScmIntegrationRegistry, + integrations: ScmIntegrationRegistry, ): DefaultAzureCredentialsManager { - const azureConfig = integration.azureBlobStorage.list().length - ? integration.azureBlobStorage.list()[0].config - : { host: 'blob.core.windows.net' }; // Default to Azure Blob Storage host if no config found + const configProviders = integrations.azureBlobStorage + .list() + .reduce((acc, integration) => { + acc.set( + integration.config.accountName || 'default', + integration.config, + ); + return acc; + }, new Map()); - return new DefaultAzureCredentialsManager(azureConfig); + return new DefaultAzureCredentialsManager(configProviders); + } + + private createCredential( + config: AzureBlobStorageIntegrationConfig, + ): TokenCredential { + if ( + config.aadCredential && + config.aadCredential.clientId && + config.aadCredential.clientSecret && + config.aadCredential.tenantId + ) { + return new ClientSecretCredential( + config.aadCredential.tenantId, + config.aadCredential.clientId, + config.aadCredential.clientSecret, + ); + } + + return new DefaultAzureCredential(); } async getCredentials(accountName: string): Promise { @@ -50,23 +82,13 @@ export class DefaultAzureCredentialsManager implements AzureCredentialsManager { return this.cachedCredentials.get(accountName)!; } - let credential: TokenCredential; - - if ( - this.config.aadCredential && - this.config.aadCredential.clientId && - this.config.aadCredential.clientSecret && - this.config.aadCredential.tenantId - ) { - credential = new ClientSecretCredential( - this.config.aadCredential.tenantId, - this.config.aadCredential.clientId, - this.config.aadCredential.clientSecret, - ); - } else { - credential = new DefaultAzureCredential(); + const config = this.configProviders.get(accountName); + if (!config) { + throw new Error(`No configuration found for account: ${accountName}`); } + const credential = this.createCredential(config); + // Cache the credentials for future use this.cachedCredentials.set(accountName, credential); diff --git a/packages/integration/src/azureBlobStorage/types.ts b/packages/integration/src/azureBlobStorage/types.ts index 99bcbec57f..90c2725d05 100644 --- a/packages/integration/src/azureBlobStorage/types.ts +++ b/packages/integration/src/azureBlobStorage/types.ts @@ -20,6 +20,12 @@ import { AnonymousCredential, } from '@azure/storage-blob'; +/** + * This allows implementations to be provided to retrieve Azure Storage accounts credentials. + * + * @public + * + */ export interface AzureCredentialsManager { getCredentials( accountName: string, diff --git a/plugins/catalog-backend-module-azure/report.api.md b/plugins/catalog-backend-module-azure/report.api.md index 19b79a6d0d..252513aa0a 100644 --- a/plugins/catalog-backend-module-azure/report.api.md +++ b/plugins/catalog-backend-module-azure/report.api.md @@ -15,9 +15,7 @@ import { SchedulerService } from '@backstage/backend-plugin-api'; import { SchedulerServiceTaskRunner } from '@backstage/backend-plugin-api'; import { ScmIntegrationRegistry } from '@backstage/integration'; -// Warning: (ae-missing-release-tag) "AzureBlobStorageEntityProvider" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export class AzureBlobStorageEntityProvider implements EntityProvider { // (undocumented) connect(connection: EntityProviderConnection): Promise; diff --git a/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.ts b/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.ts index f30675cc9e..3e2d023e76 100644 --- a/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.ts +++ b/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.ts @@ -42,6 +42,13 @@ import { import { TokenCredential } from '@azure/identity'; import { AzureBlobStorageConfig } from './types'; +/** + * Provider which discovers catalog files within an Azure Storage accounts. + * + * Use `AzureBlobStorageEntityProvider.fromConfig(...)` to create instances. + * + * @public + */ export class AzureBlobStorageEntityProvider implements EntityProvider { private readonly logger: LoggerService; private connection?: EntityProviderConnection; @@ -66,7 +73,12 @@ export class AzureBlobStorageEntityProvider implements EntityProvider { } return providerConfigs.map(providerConfig => { - const integration = scmIntegration.azureBlobStorage.list()[0]; + const integration = scmIntegration.azureBlobStorage + .list() + .filter( + azureIntegration => + azureIntegration.config.accountName === providerConfig.accountName, + )[0]; if (!integration) { throw new Error( `There is no Azure blob storage integration for host. Please add a configuration entry for it under integrations.azure`, diff --git a/plugins/catalog-backend-module-azure/src/providers/config.ts b/plugins/catalog-backend-module-azure/src/providers/config.ts index d082726b14..8ec1cbfb1b 100644 --- a/plugins/catalog-backend-module-azure/src/providers/config.ts +++ b/plugins/catalog-backend-module-azure/src/providers/config.ts @@ -98,6 +98,7 @@ function readAzureBlobStorageConfig( config: Config, ): AzureBlobStorageConfig { const containerName = config.getString('containerName'); + const accountName = config.getString('accountName'); const schedule = config.has('schedule') ? readSchedulerServiceTaskScheduleDefinitionFromConfig( config.getConfig('schedule'), @@ -107,6 +108,7 @@ function readAzureBlobStorageConfig( return { id, containerName, + accountName, schedule, }; } diff --git a/plugins/catalog-backend-module-azure/src/providers/types.ts b/plugins/catalog-backend-module-azure/src/providers/types.ts index eb16dbd6ea..e296d50ca7 100644 --- a/plugins/catalog-backend-module-azure/src/providers/types.ts +++ b/plugins/catalog-backend-module-azure/src/providers/types.ts @@ -30,5 +30,6 @@ export type AzureDevOpsConfig = { export type AzureBlobStorageConfig = { id: string; containerName: string; + accountName: string; schedule?: SchedulerServiceTaskScheduleDefinition; }; diff --git a/yarn.lock b/yarn.lock index a76d05d9db..7a68420f41 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5395,7 +5395,7 @@ __metadata: languageName: unknown linkType: soft -"@backstage/plugin-catalog-backend-module-azure@workspace:plugins/catalog-backend-module-azure": +"@backstage/plugin-catalog-backend-module-azure@workspace:^, @backstage/plugin-catalog-backend-module-azure@workspace:plugins/catalog-backend-module-azure": version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-backend-module-azure@workspace:plugins/catalog-backend-module-azure" dependencies: @@ -27070,6 +27070,7 @@ __metadata: "@backstage/plugin-auth-backend-module-guest-provider": "workspace:^" "@backstage/plugin-auth-node": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" + "@backstage/plugin-catalog-backend-module-azure": "workspace:^" "@backstage/plugin-catalog-backend-module-backstage-openapi": "workspace:^" "@backstage/plugin-catalog-backend-module-openapi": "workspace:^" "@backstage/plugin-catalog-backend-module-scaffolder-entity-model": "workspace:^"