diff --git a/.changeset/cyan-rocks-divide.md b/.changeset/cyan-rocks-divide.md new file mode 100644 index 0000000000..8576e515d8 --- /dev/null +++ b/.changeset/cyan-rocks-divide.md @@ -0,0 +1,5 @@ +--- +'@backstage/integration': minor +--- + +Add the integration for Azure blob storage to read the credentials to access the storage account and provide the default credential provider. diff --git a/.changeset/fluffy-cobras-sort.md b/.changeset/fluffy-cobras-sort.md new file mode 100644 index 0000000000..363281a6c5 --- /dev/null +++ b/.changeset/fluffy-cobras-sort.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-defaults': minor +--- + +Implemented `AzureBlobStorageUrlReader` to read from the url of committed location from the entity provider diff --git a/.changeset/tall-comics-heal.md b/.changeset/tall-comics-heal.md new file mode 100644 index 0000000000..3d65caee67 --- /dev/null +++ b/.changeset/tall-comics-heal.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-azure': minor +--- + +Added the Azure Blob Storage as catalog entity provider to import all the desired entities from storage account provided in app-config.yaml diff --git a/app-config.yaml b/app-config.yaml index cb70170541..456fd17974 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -221,6 +221,17 @@ integrations: # apiBaseUrl: server.bitbucket.com # username: ${BITBUCKET_SERVER_USERNAME} # appPassword: ${BITBUCKET_SERVER_APP_PASSWORD} + + azureBlobStorage: + - accountName: ${ACCOUNT_NAME} # required + endpoint: ${CUSTOM_ENDPOINT} # custom endpoint will require either aadCredentials or sasToken + sasToken: ${SAS_TOKEN} + aadCredential: + clientId: ${CLIENT_ID} + tenantId: ${TENANT_ID} + clientSecret: ${CLIENT_SECRET} + accountKey: ${ACCOUNT_KEY} + azure: - host: dev.azure.com token: ${AZURE_TOKEN} @@ -245,6 +256,14 @@ catalog: - Domain - Location providers: + azureBlob: + accountName: ${ACCOUNT_NAME} + containerName: ${CONTAINER_NAME} + schedule: # same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } backstageOpenapi: plugins: - catalog diff --git a/docs/integrations/azure-blobStorage/discovery.md b/docs/integrations/azure-blobStorage/discovery.md new file mode 100644 index 0000000000..32612daf75 --- /dev/null +++ b/docs/integrations/azure-blobStorage/discovery.md @@ -0,0 +1,76 @@ +--- +id: discovery +title: Azure Blob Storage Discovery +sidebar_label: Discovery +# prettier-ignore +description: Automatically discovering catalog entities from an Azure Blob Storage account +--- + +:::info +This documentation is written for [the new backend system](../../backend-system/index.md) which is the default since Backstage [version 1.24](../../releases/v1.24.0.md). +::: + +The Azure Blob Storage account integration has a special entity provider for discovering catalog +entities located in a storage account container. If you have a container that contains multiple +catalog files, and you want to automatically discover them, you can use this +provider. The provider will crawl your Blob Storage account container and register entities +matching the configured path. This can be useful as an alternative to static +locations or manually adding things to the catalog. + +To use the entity provider, you'll need an Azure Blob Storage account integration +[set up](locations.md) with `accountName` and either `aadCredential`, `sasToken`, or `accountKey` + +At production deployments, you likely manage these with the permissions attached +to your instance. + +In your configuration, you add a provider config per bucket: + +```yaml +# app-config.yaml + +catalog: + providers: + azureBlob: + providerId: + accountName: ${ACCOUNT_NAME} + containerName: ${CONTAINER_NAME} + schedule: # same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } +``` + +For simple setups, you can omit the provider ID at the config +which has the same effect as using `default` for it. + +```yaml +# app-config.yaml + +catalog: + providers: + azureBlob: + accountName: ${ACCOUNT_NAME} + containerName: ${CONTAINER_NAME} + schedule: # same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } +``` + +As this provider is not one of the default providers, you will first need to install +the Azure catalog plugin: + +```bash title="From your Backstage root directory" +yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-azure +``` + +Then updated your backend by adding the following line: + +```ts title="packages/backend/src/index.ts" +backend.add(import('@backstage/plugin-catalog-backend')); +/* highlight-add-start */ +backend.add(import('@backstage/plugin-catalog-backend-module-azure')); +/* highlight-add-end */ +``` diff --git a/docs/integrations/azure-blobStorage/locations.md b/docs/integrations/azure-blobStorage/locations.md new file mode 100644 index 0000000000..38d8555c2e --- /dev/null +++ b/docs/integrations/azure-blobStorage/locations.md @@ -0,0 +1,51 @@ +--- +id: locations +sidebar_label: Locations +title: Azure Blob Storage account Locations +# prettier-ignore +description: Setting up an integration with Azure Blob Storage account +--- + +The Azure Blob Storage account integration supports loading catalog entities from an blob storage account container. +Entities can be added to +[static catalog configuration](../../features/software-catalog/configuration.md), +or registered with the +[catalog-import](https://github.com/backstage/backstage/tree/master/plugins/catalog-import) +plugin. + +## Configuration + +To use this integration, add configuration to your `app-config.yaml`: + +Using Azure active directory credentials: + +```yaml +integrations: + azureBlobStorage: + - accountName: ${ACCOUNT_NAME} # required + endpoint: ${CUSTOM_ENDPOINT} # custom endpoint will require either aadCredentials or sasToken + aadCredential: + clientId: ${CLIENT_ID} + tenantId: ${TENANT_ID} + clientSecret: ${CLIENT_SECRET} +``` + +Using Azure storage account SAS token: + +```yaml +integrations: + azureBlobStorage: + - accountName: ${ACCOUNT_NAME} # required + endpoint: ${CUSTOM_ENDPOINT} # custom endpoint will require either aadCredentials or sasToken + sasToken: ${SAS_TOKEN} +``` + +Using Azure storage account access key: + +```yaml +integrations: + azureBlobStorage: + - accountName: ${ACCOUNT_NAME} # required + endpoint: ${CUSTOM_ENDPOINT} # custom endpoint will require either aadCredentials or sasToken + accountKey: ${ACCOUNT_KEY} +``` diff --git a/microsite/sidebars.js b/microsite/sidebars.js index 954b3630b1..61329f4985 100644 --- a/microsite/sidebars.js +++ b/microsite/sidebars.js @@ -147,6 +147,14 @@ module.exports = { 'integrations/aws-s3/discovery', ], }, + { + type: 'category', + label: 'Azure Blob Storage', + items: [ + 'integrations/azure-blobStorage/locations', + 'integrations/azure-blobStorage/discovery', + ], + }, { type: 'category', label: 'Azure', diff --git a/packages/backend-defaults/package.json b/packages/backend-defaults/package.json index ea5d0abd94..cbd2847cab 100644 --- a/packages/backend-defaults/package.json +++ b/packages/backend-defaults/package.json @@ -119,6 +119,8 @@ "@aws-sdk/client-s3": "^3.350.0", "@aws-sdk/credential-providers": "^3.350.0", "@aws-sdk/types": "^3.347.0", + "@azure/identity": "^4.0.0", + "@azure/storage-blob": "^12.5.0", "@backstage/backend-app-api": "workspace:^", "@backstage/backend-dev-utils": "workspace:^", "@backstage/backend-plugin-api": "workspace:^", diff --git a/packages/backend-defaults/report-urlReader.api.md b/packages/backend-defaults/report-urlReader.api.md index f8b8db2954..4813643d68 100644 --- a/packages/backend-defaults/report-urlReader.api.md +++ b/packages/backend-defaults/report-urlReader.api.md @@ -7,6 +7,8 @@ import { AwsCredentialsManager } from '@backstage/integration-aws-node'; import { AwsS3Integration } from '@backstage/integration'; +import { AzureBlobStorageIntergation } from '@backstage/integration'; +import { AzureCredentialsManager } from '@backstage/integration'; import { AzureDevOpsCredentialsProvider } from '@backstage/integration'; import { AzureIntegration } from '@backstage/integration'; import { BitbucketCloudIntegration } from '@backstage/integration'; @@ -60,6 +62,35 @@ export class AwsS3UrlReader implements UrlReaderService { toString(): string; } +// @public +export class AzureBlobStorageUrlReader implements UrlReaderService { + constructor( + credsManager: AzureCredentialsManager, + integration: AzureBlobStorageIntergation, + deps: { + treeResponseFactory: ReadTreeResponseFactory; + }, + ); + // (undocumented) + static factory: ReaderFactory; + // (undocumented) + read(url: string): Promise; + // (undocumented) + readTree( + url: string, + options?: UrlReaderServiceReadTreeOptions, + ): Promise; + // (undocumented) + readUrl( + url: string, + options?: UrlReaderServiceReadUrlOptions, + ): Promise; + // (undocumented) + search(): Promise; + // (undocumented) + toString(): string; +} + // @public export class AzureUrlReader implements UrlReaderService { constructor( diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureBlobStorageUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureBlobStorageUrlReader.ts new file mode 100644 index 0000000000..069b772a15 --- /dev/null +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureBlobStorageUrlReader.ts @@ -0,0 +1,235 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + BlobDownloadOptions, + BlobServiceClient, + ContainerClient, + StorageSharedKeyCredential, +} from '@azure/storage-blob'; +import { ReaderFactory, ReadTreeResponseFactory } from './types'; +import { ForwardedError, NotModifiedError } from '@backstage/errors'; +import { Readable } from 'stream'; +import { relative } from 'path/posix'; +import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; +import { + AzureBlobStorageIntergation, + AzureCredentialsManager, + DefaultAzureCredentialsManager, + ScmIntegrations, +} from '@backstage/integration'; +import { + UrlReaderService, + UrlReaderServiceReadTreeOptions, + UrlReaderServiceReadTreeResponse, + UrlReaderServiceReadUrlOptions, + UrlReaderServiceReadUrlResponse, + UrlReaderServiceSearchResponse, +} from '@backstage/backend-plugin-api'; + +export function parseUrl(url: string): { path: string; container: string } { + const parsedUrl = new URL(url); + const pathSegments = parsedUrl.pathname.split('/').filter(Boolean); + + if (pathSegments.length < 2) { + throw new Error(`Invalid Azure Blob Storage URL format: ${url}`); + } + + // First segment is the container name, rest is the blob path + const container = pathSegments[0]; + const path = pathSegments.slice(1).join('/'); + + return { path, container }; +} + +/** + * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for Azure storage accounts urls. + * + * @public + */ +export class AzureBlobStorageUrlReader implements UrlReaderService { + static factory: ReaderFactory = ({ config, treeResponseFactory }) => { + const integrations = ScmIntegrations.fromConfig(config); + + const credsManager = + DefaultAzureCredentialsManager.fromIntegrations(integrations); + + return integrations.azureBlobStorage.list().map(integrationConfig => { + const reader = new AzureBlobStorageUrlReader( + credsManager, + integrationConfig, + { + treeResponseFactory, + }, + ); + + const predicate = (url: URL) => + url.host.endsWith( + `${integrationConfig.config.accountName}.${integrationConfig.config.host}`, + ); + return { reader, predicate }; + }); + }; + + // private readonly blobServiceClient: BlobServiceClient; + + constructor( + private readonly credsManager: AzureCredentialsManager, + private readonly integration: AzureBlobStorageIntergation, + private readonly deps: { + treeResponseFactory: ReadTreeResponseFactory; + }, + ) {} + + private async createContainerClient( + containerName: string, + ): Promise { + const accountName = this.integration.config.accountName; // Use the account name from the integration config + const accountKey = this.integration.config.accountKey; // Get the account key if it exists + + if (accountKey && accountName) { + const creds = new StorageSharedKeyCredential(accountName, accountKey); + const blobServiceClient = new BlobServiceClient( + `https://${accountName}.${this.integration.config.host}`, + creds, + ); + return blobServiceClient.getContainerClient(containerName); + } + // Use the credentials manager to get the correct credentials + const credential = await this.credsManager.getCredentials( + accountName as string, + ); + + let blobServiceClientUrl: string; + + if (this.integration.config.endpoint) { + if (this.integration.config.sasToken) { + blobServiceClientUrl = `${this.integration.config.endpoint}?${this.integration.config.sasToken}`; + } else { + blobServiceClientUrl = `${this.integration.config.endpoint}`; + } + } else { + blobServiceClientUrl = `https://${this.integration.config.accountName}.${this.integration.config.host}`; + } + + const blobServiceClient = new BlobServiceClient( + blobServiceClientUrl, + credential, + ); + return blobServiceClient.getContainerClient(containerName); + } + + async read(url: string): Promise { + const response = await this.readUrl(url); + return response.buffer(); + } + + async readUrl( + url: string, + options?: UrlReaderServiceReadUrlOptions, + ): Promise { + const { etag, lastModifiedAfter } = options ?? {}; + + try { + const { path, container } = parseUrl(url); + + const containerClient = await this.createContainerClient(container); + const blobClient = containerClient.getBlobClient(path); + + const getBlobOptions: BlobDownloadOptions = { + abortSignal: options?.signal, + conditions: { + ...(etag && { ifNoneMatch: etag }), + ...(lastModifiedAfter && { ifModifiedSince: lastModifiedAfter }), + }, + }; + + const downloadBlockBlobResponse = await blobClient.download( + 0, + undefined, + getBlobOptions, + ); + + return ReadUrlResponseFactory.fromReadable( + downloadBlockBlobResponse.readableStreamBody as Readable, + { + etag: downloadBlockBlobResponse.etag, + lastModifiedAt: downloadBlockBlobResponse.lastModified, + }, + ); + } catch (e) { + if (e.$metadata && e.$metadata.httpStatusCode === 304) { + throw new NotModifiedError(); + } + + throw new ForwardedError( + 'Could not retrieve file from Azure Blob Storage', + e, + ); + } + } + + async readTree( + url: string, + options?: UrlReaderServiceReadTreeOptions, + ): Promise { + try { + const { path, container } = parseUrl(url); + + const containerClient = await this.createContainerClient(container); + + const blobs = containerClient.listBlobsFlat({ prefix: path }); + + const responses = []; + + for await (const blob of blobs) { + const blobClient = containerClient.getBlobClient(blob.name); + const downloadBlockBlobResponse = await blobClient.download( + undefined, + undefined, + { abortSignal: options?.signal }, + ); + + responses.push({ + data: Readable.from( + downloadBlockBlobResponse.readableStreamBody as Readable, + ), + path: relative(path, blob.name), + lastModifiedAt: blob.properties.lastModified, + }); + } + + return this.deps.treeResponseFactory.fromReadableArray(responses); + } catch (e) { + throw new ForwardedError( + 'Could not retrieve file tree from Azure Blob Storage', + e, + ); + } + } + + async search(): Promise { + throw new Error('AzureBlobStorageUrlReader does not implement search'); + } + + toString() { + const accountName = this.integration.config.accountName; + const accountKey = this.integration.config.accountKey; + return `azureBlobStorage{accountName=${accountName},authed=${Boolean( + accountKey, + )}}`; + } +} diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/UrlReaders.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/UrlReaders.ts index 96e9ed6509..229bdd4903 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/UrlReaders.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/UrlReaders.ts @@ -35,6 +35,7 @@ import { AwsS3UrlReader } from './AwsS3UrlReader'; import { GiteaUrlReader } from './GiteaUrlReader'; import { AwsCodeCommitUrlReader } from './AwsCodeCommitUrlReader'; import { HarnessUrlReader } from './HarnessUrlReader'; +import { AzureBlobStorageUrlReader } from './AzureBlobStorageUrlReader'; /** * Creation options for {@link @backstage/backend-plugin-api#UrlReaderService}. @@ -99,6 +100,7 @@ export class UrlReaders { GoogleGcsUrlReader.factory, HarnessUrlReader.factory, AwsS3UrlReader.factory, + AzureBlobStorageUrlReader.factory, AwsCodeCommitUrlReader.factory, FetchUrlReader.factory, ]), diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/index.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/index.ts index 678c6ef64b..fc33efbc3a 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/index.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/index.ts @@ -24,6 +24,7 @@ export { GitlabUrlReader } from './GitlabUrlReader'; export { GiteaUrlReader } from './GiteaUrlReader'; export { HarnessUrlReader } from './HarnessUrlReader'; export { AwsS3UrlReader } from './AwsS3UrlReader'; +export { AzureBlobStorageUrlReader } from './AzureBlobStorageUrlReader'; export { FetchUrlReader } from './FetchUrlReader'; export { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; export type { diff --git a/packages/integration-react/src/api/ScmIntegrationsApi.test.ts b/packages/integration-react/src/api/ScmIntegrationsApi.test.ts index fb53bbdb12..0a1f1b1b8f 100644 --- a/packages/integration-react/src/api/ScmIntegrationsApi.test.ts +++ b/packages/integration-react/src/api/ScmIntegrationsApi.test.ts @@ -26,6 +26,6 @@ describe('scmIntegrationsApiRef', () => { it('should be instantiated', () => { const i = ScmIntegrationsApi.fromConfig(new ConfigReader({})); - expect(i.list().length).toBe(6); // The default ones + expect(i.list().length).toBe(7); // The default ones }); }); diff --git a/packages/integration/package.json b/packages/integration/package.json index d14fdb0036..b606f3c2f3 100644 --- a/packages/integration/package.json +++ b/packages/integration/package.json @@ -38,6 +38,7 @@ }, "dependencies": { "@azure/identity": "^4.0.0", + "@azure/storage-blob": "^12.5.0", "@backstage/config": "workspace:^", "@backstage/errors": "workspace:^", "@octokit/auth-app": "^4.0.0", diff --git a/packages/integration/report.api.md b/packages/integration/report.api.md index e9b67bb6c1..5129dbb7c2 100644 --- a/packages/integration/report.api.md +++ b/packages/integration/report.api.md @@ -3,9 +3,12 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { AnonymousCredential } from '@azure/storage-blob'; import { Config } from '@backstage/config'; import { ConsumedResponse } from '@backstage/errors'; import { RestEndpointMethodTypes } from '@octokit/rest'; +import { StorageSharedKeyCredential } from '@azure/storage-blob'; +import { TokenCredential } from '@azure/identity'; // @public export class AwsCodeCommitIntegration implements ScmIntegration { @@ -70,6 +73,43 @@ export type AwsS3IntegrationConfig = { externalId?: string; }; +// @public +export type AzureBlobStorageIntegrationConfig = { + accountName?: string; + accountKey?: string; + sasToken?: string; + connectionString?: string; + endpointSuffix?: string; + host: string; + endpoint?: string; + aadCredential?: { + clientId: string; + tenantId: string; + clientSecret: string; + }; +}; + +// @public +export class AzureBlobStorageIntergation implements ScmIntegration { + constructor(integrationConfig: AzureBlobStorageIntegrationConfig); + // (undocumented) + get config(): AzureBlobStorageIntegrationConfig; + // (undocumented) + static factory: ScmIntegrationsFactory; + // (undocumented) + resolveEditUrl(url: string): string; + // (undocumented) + resolveUrl(options: { + url: string; + base: string; + lineNumber?: number | undefined; + }): string; + // (undocumented) + get title(): string; + // (undocumented) + get type(): string; +} + // @public export type AzureClientSecretCredential = AzureCredentialBase & { kind: 'ClientSecret'; @@ -84,6 +124,16 @@ export type AzureCredentialBase = { organizations?: string[]; }; +// @public +export interface AzureCredentialsManager { + // (undocumented) + getCredentials( + accountName: string, + ): Promise< + TokenCredential | StorageSharedKeyCredential | AnonymousCredential + >; +} + // @public export type AzureDevOpsCredential = | AzureClientSecretCredential @@ -257,6 +307,15 @@ export function buildGerritGitilesArchiveUrl( filePath: string, ): string; +// @public +export class DefaultAzureCredentialsManager implements AzureCredentialsManager { + static fromIntegrations( + integrations: ScmIntegrationRegistry, + ): DefaultAzureCredentialsManager; + // (undocumented) + getCredentials(accountName: string): Promise; +} + // @public export class DefaultAzureDevOpsCredentialsProvider implements AzureDevOpsCredentialsProvider @@ -723,6 +782,8 @@ export interface IntegrationsByType { awsS3: ScmIntegrationsGroup; // (undocumented) azure: ScmIntegrationsGroup; + // (undocumented) + azureBlobStorage: ScmIntegrationsGroup; // @deprecated (undocumented) bitbucket: ScmIntegrationsGroup; // (undocumented) @@ -826,6 +887,16 @@ export function readAwsS3IntegrationConfigs( configs: Config[], ): AwsS3IntegrationConfig[]; +// @public +export function readAzureBlobStorageIntegrationConfig( + config: Config, +): AzureBlobStorageIntegrationConfig; + +// @public +export function readAzureBlobStorageIntegrationConfigs( + configs: Config[], +): AzureBlobStorageIntegrationConfig[]; + // @public export function readAzureIntegrationConfig( config: Config, @@ -940,6 +1011,8 @@ export interface ScmIntegrationRegistry awsS3: ScmIntegrationsGroup; // (undocumented) azure: ScmIntegrationsGroup; + // (undocumented) + azureBlobStorage: ScmIntegrationsGroup; // @deprecated (undocumented) bitbucket: ScmIntegrationsGroup; // (undocumented) @@ -973,6 +1046,8 @@ export class ScmIntegrations implements ScmIntegrationRegistry { get awsS3(): ScmIntegrationsGroup; // (undocumented) get azure(): ScmIntegrationsGroup; + // (undocumented) + get azureBlobStorage(): ScmIntegrationsGroup; // @deprecated (undocumented) get bitbucket(): ScmIntegrationsGroup; // (undocumented) diff --git a/packages/integration/src/ScmIntegrations.test.ts b/packages/integration/src/ScmIntegrations.test.ts index c758816c03..0e0c181861 100644 --- a/packages/integration/src/ScmIntegrations.test.ts +++ b/packages/integration/src/ScmIntegrations.test.ts @@ -39,6 +39,10 @@ import { GiteaIntegration, GiteaIntegrationConfig } from './gitea'; import { AwsCodeCommitIntegration } from './awsCodeCommit/AwsCodeCommitIntegration'; import { AwsCodeCommitIntegrationConfig } from './awsCodeCommit'; import { HarnessIntegration, HarnessIntegrationConfig } from './harness'; +import { + AzureBlobStorageIntegrationConfig, + AzureBlobStorageIntergation, +} from './azureBlobStorage'; describe('ScmIntegrations', () => { const awsS3 = new AwsS3Integration({ @@ -53,6 +57,10 @@ describe('ScmIntegrations', () => { host: 'azure.local', } as AzureIntegrationConfig); + const azureBlob = new AzureBlobStorageIntergation({ + host: 'azureblobstorage.local', + } as AzureBlobStorageIntegrationConfig); + const bitbucket = new BitbucketIntegration({ host: 'bitbucket.local', } as BitbucketIntegrationConfig); @@ -89,6 +97,7 @@ describe('ScmIntegrations', () => { awsS3: basicIntegrations([awsS3], item => item.config.host), awsCodeCommit: basicIntegrations([awsCodeCommit], item => item.config.host), azure: basicIntegrations([azure], item => item.config.host), + azureBlobStorage: basicIntegrations([azureBlob], item => item.config.host), bitbucket: basicIntegrations([bitbucket], item => item.config.host), bitbucketCloud: basicIntegrations([bitbucketCloud], item => item.title), bitbucketServer: basicIntegrations( @@ -108,6 +117,9 @@ describe('ScmIntegrations', () => { awsCodeCommit, ); expect(i.azure.byUrl('https://azure.local')).toBe(azure); + expect(i.azureBlobStorage.byUrl('https://azureblobstorage.local')).toBe( + azureBlob, + ); expect(i.bitbucket.byUrl('https://bitbucket.local')).toBe(bitbucket); expect(i.bitbucketCloud.byUrl('https://bitbucket.org')).toBe( bitbucketCloud, @@ -128,6 +140,7 @@ describe('ScmIntegrations', () => { awsS3, awsCodeCommit, azure, + azureBlob, bitbucket, bitbucketCloud, bitbucketServer, @@ -144,6 +157,9 @@ describe('ScmIntegrations', () => { expect(i.byUrl('https://awss3.local')).toBe(awsS3); expect(i.byUrl('https://awscodecommit.local')).toBe(awsCodeCommit); expect(i.byUrl('https://azure.local')).toBe(azure); + expect(i.azureBlobStorage.byUrl('https://azureblobstorage.local')).toBe( + azureBlob, + ); expect(i.byUrl('https://bitbucket.local')).toBe(bitbucket); expect(i.byUrl('https://bitbucket.org')).toBe(bitbucketCloud); expect(i.byUrl('https://bitbucket-server.local')).toBe(bitbucketServer); @@ -156,6 +172,7 @@ describe('ScmIntegrations', () => { expect(i.byHost('awss3.local')).toBe(awsS3); expect(i.byHost('awscodecommit.local')).toBe(awsCodeCommit); expect(i.byHost('azure.local')).toBe(azure); + expect(i.byHost('azureblobstorage.local')).toBe(azureBlob); expect(i.byHost('bitbucket.local')).toBe(bitbucket); expect(i.byHost('bitbucket.org')).toBe(bitbucketCloud); expect(i.byHost('bitbucket-server.local')).toBe(bitbucketServer); diff --git a/packages/integration/src/ScmIntegrations.ts b/packages/integration/src/ScmIntegrations.ts index e0d430d834..f11cb3c17e 100644 --- a/packages/integration/src/ScmIntegrations.ts +++ b/packages/integration/src/ScmIntegrations.ts @@ -29,6 +29,7 @@ import { ScmIntegration, ScmIntegrationsGroup } from './types'; import { ScmIntegrationRegistry } from './registry'; import { GiteaIntegration } from './gitea'; import { HarnessIntegration } from './harness/HarnessIntegration'; +import { AzureBlobStorageIntergation } from './azureBlobStorage'; /** * The set of supported integrations. @@ -38,6 +39,7 @@ import { HarnessIntegration } from './harness/HarnessIntegration'; export interface IntegrationsByType { awsS3: ScmIntegrationsGroup; awsCodeCommit: ScmIntegrationsGroup; + azureBlobStorage: ScmIntegrationsGroup; azure: ScmIntegrationsGroup; /** * @deprecated in favor of `bitbucketCloud` and `bitbucketServer` @@ -64,6 +66,7 @@ export class ScmIntegrations implements ScmIntegrationRegistry { return new ScmIntegrations({ awsS3: AwsS3Integration.factory({ config }), awsCodeCommit: AwsCodeCommitIntegration.factory({ config }), + azureBlobStorage: AzureBlobStorageIntergation.factory({ config }), azure: AzureIntegration.factory({ config }), bitbucket: BitbucketIntegration.factory({ config }), bitbucketCloud: BitbucketCloudIntegration.factory({ config }), @@ -88,6 +91,10 @@ export class ScmIntegrations implements ScmIntegrationRegistry { return this.byType.awsCodeCommit; } + get azureBlobStorage(): ScmIntegrationsGroup { + return this.byType.azureBlobStorage; + } + get azure(): ScmIntegrationsGroup { return this.byType.azure; } diff --git a/packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.test.ts b/packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.test.ts new file mode 100644 index 0000000000..e7f7703be9 --- /dev/null +++ b/packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.test.ts @@ -0,0 +1,78 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { AzureBlobStorageIntergation } from './AzureBlobStorageIntegration'; + +describe('AzureBlobStorageIntegration', () => { + it('has a working factory', () => { + const integrations = AzureBlobStorageIntergation.factory({ + config: new ConfigReader({ + integrations: { + azureBlobStorage: [ + { + endpoint: 'https://myaccount.blob.core.windows.net', + accountName: 'myaccount', + accountKey: 'someAccountKey', + }, + ], + }, + }), + }); + expect(integrations.list().length).toBe(2); // including default + expect(integrations.list()[0].config.host).toBe( + 'myaccount.blob.core.windows.net', + ); + expect(integrations.list()[1].config.host).toBe('blob.core.windows.net'); // default integration + }); + + it('returns the basics', () => { + const integration = new AzureBlobStorageIntergation({ + host: 'myaccount.blob.core.windows.net', + } as any); + expect(integration.type).toBe('azureBlobStorage'); + expect(integration.title).toBe('myaccount.blob.core.windows.net'); + }); + + describe('resolveUrl', () => { + it('works for valid URLs', () => { + const integration = new AzureBlobStorageIntergation({ + host: 'blob.core.windows.net', + } as any); + + expect( + integration.resolveUrl({ + url: 'https://myaccount.blob.core.windows.net/container/file.yaml', + base: 'https://myaccount.blob.core.windows.net/container/file.yaml', + }), + ).toBe('https://myaccount.blob.core.windows.net/container/file.yaml'); + }); + }); + + it('resolve edit URL', () => { + const integration = new AzureBlobStorageIntergation({ + host: 'myaccount.blob.core.windows.net', + } as any); + + // TODO: The Azure Blob Storage integration doesn't support resolving an edit URL, + // instead we keep the input URL. + expect( + integration.resolveEditUrl( + 'https://myaccount.blob.core.windows.net/container/file.yaml', + ), + ).toBe('https://myaccount.blob.core.windows.net/container/file.yaml'); + }); +}); diff --git a/packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.ts b/packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.ts new file mode 100644 index 0000000000..27061f7d9d --- /dev/null +++ b/packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.ts @@ -0,0 +1,71 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { basicIntegrations, defaultScmResolveUrl } from '../helpers'; +import { ScmIntegration, ScmIntegrationsFactory } from '../types'; +import { + AzureBlobStorageIntegrationConfig, + readAzureBlobStorageIntegrationConfigs, +} from './config'; + +/** + * Microsoft Azure Blob storage based integration. + * + * @public + */ +export class AzureBlobStorageIntergation implements ScmIntegration { + static factory: ScmIntegrationsFactory = ({ + config, + }) => { + const configs = readAzureBlobStorageIntegrationConfigs( + config.getOptionalConfigArray('integrations.azureBlobStorage') ?? [], + ); + return basicIntegrations( + configs.map(c => new AzureBlobStorageIntergation(c)), + i => i.config.host, + ); + }; + + get type(): string { + return 'azureBlobStorage'; + } + + get title(): string { + return this.integrationConfig.host; + } + + get config(): AzureBlobStorageIntegrationConfig { + return this.integrationConfig; + } + + constructor( + private readonly integrationConfig: AzureBlobStorageIntegrationConfig, + ) {} + + resolveUrl(options: { + url: string; + base: string; + lineNumber?: number | undefined; + }): string { + const resolved = defaultScmResolveUrl(options); + return resolved; + } + + resolveEditUrl(url: string): string { + // TODO: Implement edit URL for azureBlobStorage + return url; + } +} diff --git a/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsManager.test.ts b/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsManager.test.ts new file mode 100644 index 0000000000..d8e8be2919 --- /dev/null +++ b/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsManager.test.ts @@ -0,0 +1,149 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + AccessToken, + ClientSecretCredential, + DefaultAzureCredential, +} from '@azure/identity'; +import { ScmIntegrationRegistry } from '../registry'; +import { ConfigReader } from '@backstage/config'; +import { DefaultAzureCredentialsManager } from './DefaultAzureCredentialsProvider'; +import { ScmIntegrations } from '../ScmIntegrations'; +import { DateTime } from 'luxon'; + +const MockedClientSecretCredential = ClientSecretCredential as jest.MockedClass< + typeof ClientSecretCredential +>; + +jest.mock('@azure/identity'); + +describe('DefaultAzureCredentialsManager', () => { + let mockIntegration: ScmIntegrationRegistry; + + const buildProvider = (azureIntegrations: any[]) => + DefaultAzureCredentialsManager.fromIntegrations( + ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + azureBlobStorage: azureIntegrations, + }, + }), + ), + ); + + beforeEach(() => { + mockIntegration = { + azureBlobStorage: { + list: jest.fn().mockReturnValue([ + { + config: { + accountName: 'testaccount', + aadCredential: { + clientId: 'someClientId', + tenantId: 'someTenantId', + clientSecret: 'someClientSecret', + }, + }, + }, + ]), + }, + } as unknown as ScmIntegrationRegistry; + + MockedClientSecretCredential.prototype.getToken.mockImplementation(() => + Promise.resolve({ + expiresOnTimestamp: DateTime.local().plus({ days: 1 }).toSeconds(), + token: 'fake-client-secret-token', + } as AccessToken), + ); + }); + + it('should create an instance from ScmIntegrationRegistry', () => { + const manager = + DefaultAzureCredentialsManager.fromIntegrations(mockIntegration); + expect(manager).toBeInstanceOf(DefaultAzureCredentialsManager); + }); + + it('should return cached credentials if available', async () => { + const manager = buildProvider([ + { + accountName: 'testaccount', + aadCredential: { + clientId: 'someClientId', + tenantId: 'someTenantId', + clientSecret: 'someClientSecret', + }, + }, + ]); + + const mockCredential = new MockedClientSecretCredential( + 'someTenantId', + 'someClientId', + 'someClientSecret', + ); + + const credential = await manager.getCredentials('testaccount'); + + const scopes = ['https://storage.azure.com/.default']; + + const expectedToken = await mockCredential.getToken(scopes); + const receivedToken = await credential.getToken(scopes); + + expect(receivedToken?.token).toEqual(expectedToken.token); + }); + + it('should use Azure AD credentials if aadCredential is provided', async () => { + const manager = buildProvider([ + { + accountName: 'testaccount', + aadCredential: { + clientId: 'someClientId', + tenantId: 'someTenantId', + clientSecret: 'someClientSecret', + }, + }, + ]); + + const credential = await manager.getCredentials('testaccount'); + + expect(credential).toBeInstanceOf(ClientSecretCredential); + }); + + it('should use DefaultAzureCredential if no aadCredential is provided', async () => { + const manager = buildProvider([ + { + accountName: 'testaccount', + }, + ]); + + const credential = await manager.getCredentials('testaccount'); + + expect(credential).toBeInstanceOf(DefaultAzureCredential); + }); + + it('should cache credentials after first retrieval', async () => { + const manager = buildProvider([ + { + accountName: 'testaccount', + }, + ]); + + const credential = await manager.getCredentials('testaccount'); + + const cachedCredential = await manager.getCredentials('testaccount'); + expect(cachedCredential).toBe(credential); + }); +}); diff --git a/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsProvider.ts b/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsProvider.ts new file mode 100644 index 0000000000..3861329300 --- /dev/null +++ b/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsProvider.ts @@ -0,0 +1,97 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + DefaultAzureCredential, + ClientSecretCredential, + TokenCredential, +} from '@azure/identity'; +import { AzureBlobStorageIntegrationConfig } from './config'; +import { AzureCredentialsManager } from './types'; +import { ScmIntegrationRegistry } from '../registry'; + +/** + * Default implementation of AzureCredentialsManager that supports multiple Azure Blob Storage integrations. + * @public + */ +export class DefaultAzureCredentialsManager implements AzureCredentialsManager { + private cachedCredentials: Map; + + private constructor( + private readonly configProviders: Map< + string, + AzureBlobStorageIntegrationConfig + >, + ) { + this.cachedCredentials = new Map(); + } + + /** + * Creates an instance of DefaultAzureCredentialsManager from a Backstage integration registry. + */ + static fromIntegrations( + integrations: ScmIntegrationRegistry, + ): DefaultAzureCredentialsManager { + const configProviders = integrations.azureBlobStorage + .list() + .reduce((acc, integration) => { + acc.set( + integration.config.accountName || 'default', + integration.config, + ); + return acc; + }, new Map()); + + return new DefaultAzureCredentialsManager(configProviders); + } + + private createCredential( + config: AzureBlobStorageIntegrationConfig, + ): TokenCredential { + if ( + config.aadCredential && + config.aadCredential.clientId && + config.aadCredential.clientSecret && + config.aadCredential.tenantId + ) { + return new ClientSecretCredential( + config.aadCredential.tenantId, + config.aadCredential.clientId, + config.aadCredential.clientSecret, + ); + } + + return new DefaultAzureCredential(); + } + + async getCredentials(accountName: string): Promise { + if (this.cachedCredentials.has(accountName)) { + return this.cachedCredentials.get(accountName)!; + } + + const config = this.configProviders.get(accountName); + if (!config) { + throw new Error(`No configuration found for account: ${accountName}`); + } + + const credential = this.createCredential(config); + + // Cache the credentials for future use + this.cachedCredentials.set(accountName, credential); + + return credential; + } +} diff --git a/packages/integration/src/azureBlobStorage/config.test.ts b/packages/integration/src/azureBlobStorage/config.test.ts new file mode 100644 index 0000000000..f17bdf3efb --- /dev/null +++ b/packages/integration/src/azureBlobStorage/config.test.ts @@ -0,0 +1,264 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Config, ConfigReader } from '@backstage/config'; +import { + AzureBlobStorageIntegrationConfig, + readAzureBlobStorageIntegrationConfig, + readAzureBlobStorageIntegrationConfigs, +} from './config'; + +describe('readAzureBlobStorageIntegrationConfig', () => { + function buildConfig( + data: Partial, + ): Config { + return new ConfigReader(data); + } + + it('reads valid configuration with accountKey', () => { + const output = readAzureBlobStorageIntegrationConfig( + buildConfig({ + accountName: 'mystorageaccount', + accountKey: 'someAccountKey', + }), + ); + expect(output).toEqual({ + host: 'blob.core.windows.net', + endpoint: undefined, + accountName: 'mystorageaccount', + accountKey: 'someAccountKey', + sasToken: undefined, + connectionString: undefined, + endpointSuffix: undefined, + aadCredential: undefined, + }); + }); + + it('reads valid configuration with sasToken', () => { + const output = readAzureBlobStorageIntegrationConfig( + buildConfig({ + endpoint: 'https://blob.core.windows.net', + accountName: 'mystorageaccount', + sasToken: 'someSASToken', + }), + ); + expect(output).toEqual({ + host: 'blob.core.windows.net', + endpoint: 'https://blob.core.windows.net', + accountName: 'mystorageaccount', + accountKey: undefined, + sasToken: 'someSASToken', + connectionString: undefined, + endpointSuffix: undefined, + aadCredential: undefined, + }); + }); + + it('reads valid configuration with Azure AD credentials', () => { + const output = readAzureBlobStorageIntegrationConfig( + buildConfig({ + accountName: 'mystorageaccount', + aadCredential: { + clientId: 'someClientId', + tenantId: 'someTenantId', + clientSecret: 'someClientSecret', + }, + }), + ); + expect(output).toEqual({ + host: 'blob.core.windows.net', + endpoint: undefined, + accountName: 'mystorageaccount', + accountKey: undefined, + sasToken: undefined, + connectionString: undefined, + endpointSuffix: undefined, + aadCredential: { + clientId: 'someClientId', + tenantId: 'someTenantId', + clientSecret: 'someClientSecret', + }, + }); + }); + + it('reads valid configuration with a custom endpoint', () => { + const output = readAzureBlobStorageIntegrationConfig( + buildConfig({ + endpoint: 'https://custom.blob.core.windows.net', + accountName: 'customaccount', + }), + ); + expect(output).toEqual({ + host: 'custom.blob.core.windows.net', + endpoint: 'https://custom.blob.core.windows.net', + accountName: 'customaccount', + accountKey: undefined, + sasToken: undefined, + connectionString: undefined, + endpointSuffix: undefined, + aadCredential: undefined, + }); + }); + + it('throws error for invalid endpoint URL', () => { + const config = buildConfig({ + endpoint: 'invalid-url', + accountName: 'invalidaccount', + }); + + expect(() => readAzureBlobStorageIntegrationConfig(config)).toThrow( + `invalid azureBlobStorage integration config, endpoint 'invalid-url' is not a valid URL`, + ); + }); + + it('throws error if endpoint has a path', () => { + const config = buildConfig({ + endpoint: 'https://blob.core.windows.net/path', + accountName: 'accountwithpath', + }); + + expect(() => readAzureBlobStorageIntegrationConfig(config)).toThrow( + `invalid azureBlobStorage integration config, endpoints cannot contain path, got 'https://blob.core.windows.net/path'`, + ); + }); + + it('throws error if both accountKey and sasToken are provided', () => { + const config = buildConfig({ + accountName: 'mystorageaccount', + accountKey: 'someAccountKey', + sasToken: 'someSASToken', + }); + + expect(() => readAzureBlobStorageIntegrationConfig(config)).toThrow( + `Invalid Azure Blob Storage config for mystorageaccount: Both account key and SAS token cannot be used simultaneously.`, + ); + }); + + it('throws error if both aadCredential and accountKey/sasToken are provided', () => { + const config = buildConfig({ + accountName: 'mystorageaccount', + accountKey: 'someAccountKey', + aadCredential: { + clientId: 'someClientId', + tenantId: 'someTenantId', + clientSecret: 'someClientSecret', + }, + }); + + expect(() => readAzureBlobStorageIntegrationConfig(config)).toThrow( + `Invalid Azure Blob Storage config for mystorageaccount: Cannot use both Azure AD credentials and account keys/SAS tokens for the same account.`, + ); + }); +}); + +describe('readAzureBlobStorageIntegrationConfigs', () => { + function buildConfigs( + data: Partial[], + ): Config[] { + return data.map(item => new ConfigReader(item)); + } + + it('reads all provided configurations', () => { + const output = readAzureBlobStorageIntegrationConfigs( + buildConfigs([ + { + host: 'blob.core.windows.net', + accountName: 'account1', + accountKey: 'someAccountKey', + }, + { + endpoint: 'https://custom.blob.core.windows.net', + accountName: 'account2', + }, + ]), + ); + expect(output).toEqual([ + { + host: 'blob.core.windows.net', + endpoint: undefined, + accountName: 'account1', + accountKey: 'someAccountKey', + sasToken: undefined, + connectionString: undefined, + endpointSuffix: undefined, + aadCredential: undefined, + }, + { + host: 'custom.blob.core.windows.net', + endpoint: 'https://custom.blob.core.windows.net', + accountName: 'account2', + accountKey: undefined, + sasToken: undefined, + connectionString: undefined, + endpointSuffix: undefined, + aadCredential: undefined, + }, + ]); + }); + + it('adds default integration for blob.core.windows.net when missing', () => { + const output = readAzureBlobStorageIntegrationConfigs( + buildConfigs([ + { + endpoint: 'https://custom.blob.core.windows.net', + accountName: 'account2', + }, + ]), + ); + + expect(output).toEqual([ + { + host: 'custom.blob.core.windows.net', + endpoint: 'https://custom.blob.core.windows.net', + accountName: 'account2', + accountKey: undefined, + sasToken: undefined, + connectionString: undefined, + endpointSuffix: undefined, + aadCredential: undefined, + }, + { + host: 'blob.core.windows.net', + endpoint: undefined, + accountName: undefined, + accountKey: undefined, + sasToken: undefined, + connectionString: undefined, + endpointSuffix: undefined, + aadCredential: undefined, + }, + ]); + }); + + it('does not add default integration if blob.core.windows.net already exists', () => { + const output = readAzureBlobStorageIntegrationConfigs( + buildConfigs([ + { host: 'blob.core.windows.net', accountName: 'account1' }, + ]), + ); + expect(output).toEqual([ + { + host: 'blob.core.windows.net', + endpoint: undefined, + accountName: 'account1', + accountKey: undefined, + sasToken: undefined, + connectionString: undefined, + endpointSuffix: undefined, + aadCredential: undefined, + }, + ]); + }); +}); diff --git a/packages/integration/src/azureBlobStorage/config.ts b/packages/integration/src/azureBlobStorage/config.ts new file mode 100644 index 0000000000..10eccc252c --- /dev/null +++ b/packages/integration/src/azureBlobStorage/config.ts @@ -0,0 +1,170 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; + +const AZURE_HOST = 'blob.core.windows.net'; + +/** + * The configuration parameters for a single Azure Blob Storage account. + * + * @public + */ +export type AzureBlobStorageIntegrationConfig = { + /** + * The name of the Azure Storage Account, e.g., "mystorageaccount". + */ + accountName?: string; + + /** + * The primary or secondary key for the Azure Storage Account. + * Only required if connectionString or SAS token are not specified. + */ + accountKey?: string; + + /** + * A Shared Access Signature (SAS) token for limited access to resources. + */ + sasToken?: string; + + /** + * A full connection string for the Azure Storage Account. + * This includes the account name, key, and endpoint details. + */ + connectionString?: string; + + /** + * Optional endpoint suffix for custom domains or sovereign clouds. + * e.g., "core.windows.net" for public Azure or "core.usgovcloudapi.net" for US Government cloud. + */ + endpointSuffix?: string; + + /** + * The host of the target that this matches on, e.g., "blob.core.windows.net". + */ + host: string; + + endpoint?: string; + /** + * Optional credential to use for Azure Active Directory authentication. + */ + aadCredential?: { + /** + * The client ID of the Azure AD application. + */ + clientId: string; + + /** + * The tenant ID for Azure AD. + */ + tenantId: string; + + /** + * The client secret for the Azure AD application. + */ + clientSecret: string; + }; +}; + +/** + * Reads a single Azure Blob Storage integration config. + * + * @param config - The config object of a single integration. + * @public + */ +export function readAzureBlobStorageIntegrationConfig( + config: Config, +): AzureBlobStorageIntegrationConfig { + const endpoint = config.getOptionalString('endpoint'); + const accountName = config.getString('accountName'); + const accountKey = config.getOptionalString('accountKey')?.trim(); + const sasToken = config.getOptionalString('sasToken')?.trim(); + const connectionString = config.getOptionalString('connectionString')?.trim(); + const endpointSuffix = config.getOptionalString('endpointSuffix')?.trim(); + + let host; + let pathname; + if (endpoint) { + try { + const url = new URL(endpoint); + host = url.host; + pathname = url.pathname; + } catch { + throw new Error( + `invalid azureBlobStorage integration config, endpoint '${endpoint}' is not a valid URL`, + ); + } + if (pathname !== '/') { + throw new Error( + `invalid azureBlobStorage integration config, endpoints cannot contain path, got '${endpoint}'`, + ); + } + } else { + host = AZURE_HOST; + } + const aadCredential = config.has('aadCredential') + ? { + clientId: config.getString('aadCredential.clientId'), + tenantId: config.getString('aadCredential.tenantId'), + clientSecret: config.getString('aadCredential.clientSecret')?.trim(), + } + : undefined; + + if (accountKey && sasToken) { + throw new Error( + `Invalid Azure Blob Storage config for ${accountName}: Both account key and SAS token cannot be used simultaneously.`, + ); + } + + if (aadCredential && (accountKey || sasToken)) { + throw new Error( + `Invalid Azure Blob Storage config for ${accountName}: Cannot use both Azure AD credentials and account keys/SAS tokens for the same account.`, + ); + } + + return { + host, + endpoint, + accountName, + accountKey, + sasToken, + connectionString, + endpointSuffix, + aadCredential, + }; +} + +/** + * Reads a set of Azure Blob Storage integration configs. + * + * @param configs - All of the integration config objects. + * @public + */ +export function readAzureBlobStorageIntegrationConfigs( + configs: Config[], +): AzureBlobStorageIntegrationConfig[] { + // First read all the explicit integrations + const result = configs.map(readAzureBlobStorageIntegrationConfig); + + // If no explicit blob.core.windows.net integration was added, put one in the list as + // a convenience + if (!result.some(c => c.host === AZURE_HOST)) { + result.push({ + host: AZURE_HOST, + }); + } + return result; +} diff --git a/packages/integration/src/azureBlobStorage/index.ts b/packages/integration/src/azureBlobStorage/index.ts new file mode 100644 index 0000000000..f55e9aaa79 --- /dev/null +++ b/packages/integration/src/azureBlobStorage/index.ts @@ -0,0 +1,23 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export { AzureBlobStorageIntergation } from './AzureBlobStorageIntegration'; +export { + readAzureBlobStorageIntegrationConfig, + readAzureBlobStorageIntegrationConfigs, +} from './config'; +export type { AzureBlobStorageIntegrationConfig } from './config'; +export { DefaultAzureCredentialsManager } from './DefaultAzureCredentialsProvider'; +export type { AzureCredentialsManager } from './types'; diff --git a/packages/integration/src/azureBlobStorage/types.ts b/packages/integration/src/azureBlobStorage/types.ts new file mode 100644 index 0000000000..90c2725d05 --- /dev/null +++ b/packages/integration/src/azureBlobStorage/types.ts @@ -0,0 +1,35 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TokenCredential } from '@azure/identity'; +import { + StorageSharedKeyCredential, + AnonymousCredential, +} from '@azure/storage-blob'; + +/** + * This allows implementations to be provided to retrieve Azure Storage accounts credentials. + * + * @public + * + */ +export interface AzureCredentialsManager { + getCredentials( + accountName: string, + ): Promise< + TokenCredential | StorageSharedKeyCredential | AnonymousCredential + >; +} diff --git a/packages/integration/src/index.ts b/packages/integration/src/index.ts index 32c573abef..a625f5d83c 100644 --- a/packages/integration/src/index.ts +++ b/packages/integration/src/index.ts @@ -22,6 +22,7 @@ export * from './awsS3'; export * from './awsCodeCommit'; +export * from './azureBlobStorage'; export * from './azure'; export * from './bitbucket'; export * from './bitbucketCloud'; diff --git a/packages/integration/src/registry.ts b/packages/integration/src/registry.ts index 7e4b34cc34..2e8111903b 100644 --- a/packages/integration/src/registry.ts +++ b/packages/integration/src/registry.ts @@ -26,6 +26,7 @@ import { GithubIntegration } from './github/GithubIntegration'; import { GitLabIntegration } from './gitlab/GitLabIntegration'; import { GiteaIntegration } from './gitea/GiteaIntegration'; import { HarnessIntegration } from './harness/HarnessIntegration'; +import { AzureBlobStorageIntergation } from './azureBlobStorage'; /** * Holds all registered SCM integrations, of all types. @@ -36,6 +37,7 @@ export interface ScmIntegrationRegistry extends ScmIntegrationsGroup { awsS3: ScmIntegrationsGroup; awsCodeCommit: ScmIntegrationsGroup; + azureBlobStorage: ScmIntegrationsGroup; azure: ScmIntegrationsGroup; /** * @deprecated in favor of `bitbucketCloud` and `bitbucketServer` diff --git a/plugins/catalog-backend-module-azure/package.json b/plugins/catalog-backend-module-azure/package.json index 0129477902..f8edf65e55 100644 --- a/plugins/catalog-backend-module-azure/package.json +++ b/plugins/catalog-backend-module-azure/package.json @@ -51,6 +51,8 @@ "test": "backstage-cli package test" }, "dependencies": { + "@azure/identity": "^4.0.0", + "@azure/storage-blob": "^12.5.0", "@backstage/backend-plugin-api": "workspace:^", "@backstage/config": "workspace:^", "@backstage/integration": "workspace:^", diff --git a/plugins/catalog-backend-module-azure/report.api.md b/plugins/catalog-backend-module-azure/report.api.md index d11197d7f2..60b7ee135b 100644 --- a/plugins/catalog-backend-module-azure/report.api.md +++ b/plugins/catalog-backend-module-azure/report.api.md @@ -56,6 +56,6 @@ export class AzureDevOpsEntityProvider implements EntityProvider { } // @public -const catalogModuleAzureDevOpsEntityProvider: BackendFeature; -export default catalogModuleAzureDevOpsEntityProvider; +const catalogModuleAzureEntityProvider: BackendFeature; +export default catalogModuleAzureEntityProvider; ``` diff --git a/plugins/catalog-backend-module-azure/src/module/catalogModuleAzureDevOpsEntityProvider.test.ts b/plugins/catalog-backend-module-azure/src/module/catalogModuleAzureDevOpsEntityProvider.test.ts index 32eb190460..9c5d1baef7 100644 --- a/plugins/catalog-backend-module-azure/src/module/catalogModuleAzureDevOpsEntityProvider.test.ts +++ b/plugins/catalog-backend-module-azure/src/module/catalogModuleAzureDevOpsEntityProvider.test.ts @@ -17,8 +17,11 @@ import { SchedulerServiceTaskScheduleDefinition } from '@backstage/backend-plugin-api'; import { mockServices, startTestBackend } from '@backstage/backend-test-utils'; import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; -import { catalogModuleAzureDevOpsEntityProvider } from './catalogModuleAzureDevOpsEntityProvider'; -import { AzureDevOpsEntityProvider } from '../providers'; +import { catalogModuleAzureEntityProvider } from './catalogModuleAzureDevOpsEntityProvider'; +import { + AzureBlobStorageEntityProvider, + AzureDevOpsEntityProvider, +} from '../providers'; describe('catalogModuleAzureDevOpsEntityProvider', () => { it('should register provider at the catalog extension point', async () => { @@ -58,7 +61,7 @@ describe('catalogModuleAzureDevOpsEntityProvider', () => { await startTestBackend({ extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]], features: [ - catalogModuleAzureDevOpsEntityProvider, + catalogModuleAzureEntityProvider, mockServices.rootConfig.factory({ data: config }), mockServices.logger.factory(), scheduler.factory, @@ -73,4 +76,63 @@ describe('catalogModuleAzureDevOpsEntityProvider', () => { ); expect(runner).not.toHaveBeenCalled(); }); + + it('should register azure blob storage provider at the catalog extension point', async () => { + let addedProviders: Array | undefined; + let usedSchedule: SchedulerServiceTaskScheduleDefinition | undefined; + + const extensionPoint = { + addEntityProvider: (providers: any) => { + addedProviders = providers; + }, + }; + const runner = jest.fn(); + const scheduler = mockServices.scheduler.mock({ + createScheduledTaskRunner(schedule) { + usedSchedule = schedule; + return { run: runner }; + }, + }); + + const config = { + integrations: { + azureBlobStorage: [ + { + accountName: 'test', + accountKey: 'test', + }, + ], + }, + catalog: { + providers: { + azureBlob: { + containerName: 'test', + accountName: 'test', + schedule: { + frequency: 'P1M', + timeout: 'PT3M', + }, + }, + }, + }, + }; + + await startTestBackend({ + extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]], + features: [ + catalogModuleAzureEntityProvider, + mockServices.rootConfig.factory({ data: config }), + mockServices.logger.factory(), + scheduler.factory, + ], + }); + + expect(usedSchedule?.frequency).toEqual({ months: 1 }); + expect(usedSchedule?.timeout).toEqual({ minutes: 3 }); + expect(addedProviders?.length).toEqual(1); + expect(addedProviders?.pop()?.getProviderName()).toEqual( + 'azureBlobStorage-provider:default', + ); + expect(runner).not.toHaveBeenCalled(); + }); }); diff --git a/plugins/catalog-backend-module-azure/src/module/catalogModuleAzureDevOpsEntityProvider.ts b/plugins/catalog-backend-module-azure/src/module/catalogModuleAzureDevOpsEntityProvider.ts index 3aaa1b5b08..cc808292f3 100644 --- a/plugins/catalog-backend-module-azure/src/module/catalogModuleAzureDevOpsEntityProvider.ts +++ b/plugins/catalog-backend-module-azure/src/module/catalogModuleAzureDevOpsEntityProvider.ts @@ -19,16 +19,19 @@ import { createBackendModule, } from '@backstage/backend-plugin-api'; import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; -import { AzureDevOpsEntityProvider } from '../providers'; +import { + AzureBlobStorageEntityProvider, + AzureDevOpsEntityProvider, +} from '../providers'; /** * Registers the AzureDevOpsEntityProvider with the catalog processing extension point. * * @public */ -export const catalogModuleAzureDevOpsEntityProvider = createBackendModule({ +export const catalogModuleAzureEntityProvider = createBackendModule({ pluginId: 'catalog', - moduleId: 'azure-dev-ops-entity-provider', + moduleId: 'azure-providers', register(env) { env.registerInit({ deps: { @@ -38,12 +41,25 @@ export const catalogModuleAzureDevOpsEntityProvider = createBackendModule({ scheduler: coreServices.scheduler, }, async init({ config, catalog, logger, scheduler }) { - catalog.addEntityProvider( - AzureDevOpsEntityProvider.fromConfig(config, { - logger, - scheduler, - }), - ); + // Check for Azure Blob Storage provider configuration and register it + if (config.has('catalog.providers.azureBlob')) { + catalog.addEntityProvider( + AzureBlobStorageEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ); + } + + // Check for Azure DevOps provider configuration and register it + if (config.has('catalog.providers.azureDevOps')) { + catalog.addEntityProvider( + AzureDevOpsEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ); + } }, }); }, diff --git a/plugins/catalog-backend-module-azure/src/module/index.ts b/plugins/catalog-backend-module-azure/src/module/index.ts index 329ad604e2..6e560cf17f 100644 --- a/plugins/catalog-backend-module-azure/src/module/index.ts +++ b/plugins/catalog-backend-module-azure/src/module/index.ts @@ -14,4 +14,4 @@ * limitations under the License. */ -export { catalogModuleAzureDevOpsEntityProvider as default } from './catalogModuleAzureDevOpsEntityProvider'; +export { catalogModuleAzureEntityProvider as default } from './catalogModuleAzureDevOpsEntityProvider'; diff --git a/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.test.ts b/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.test.ts new file mode 100644 index 0000000000..62b635d26e --- /dev/null +++ b/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.test.ts @@ -0,0 +1,238 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + SchedulerService, + SchedulerServiceTaskRunner, + SchedulerServiceTaskInvocationDefinition, +} from '@backstage/backend-plugin-api'; +import { ConfigReader } from '@backstage/config'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; +import { AzureBlobStorageEntityProvider } from './AzureBlobStorageEntityProvider'; +import { mockServices } from '@backstage/backend-test-utils'; + +class PersistingTaskRunner implements SchedulerServiceTaskRunner { + private tasks: SchedulerServiceTaskInvocationDefinition[] = []; + + getTasks() { + return this.tasks; + } + + run(task: SchedulerServiceTaskInvocationDefinition): Promise { + this.tasks.push(task); + return Promise.resolve(undefined); + } +} + +const blobs = ['key1.yaml', 'key2.yaml', 'key3.yaml', 'key4.yaml']; +const createBlobList = (blobsArray: string[]) => { + return blobsArray.map(blob => ({ + name: blob, + })); +}; +// Mocking Azure Storage Blob Library +jest.mock('@azure/storage-blob', () => { + return { + BlobServiceClient: jest.fn().mockImplementation(() => ({ + url: 'https://myaccount.blob.core.windows.net/', + getContainerClient: jest.fn().mockImplementation(() => ({ + // eslint-disable-next-line func-names + listBlobsFlat: jest.fn(async function* () { + yield* createBlobList(blobs); + }), + })), + })), + }; +}); + +const logger = mockServices.logger.mock(); + +describe('AzureBlobStorageEntityProvider', () => { + const containerName = 'container-1'; + const accountName = 'myaccount'; + const expectMutation = async ( + providerId: string, + providerConfig: object, + expectedBaseUrl: string, + integrationConfig?: object, + scheduleInConfig?: boolean, + ) => { + const config = new ConfigReader({ + integrations: { + azureBlobStorage: integrationConfig ? [integrationConfig] : [], + }, + catalog: { + providers: { + azureBlob: { + [providerId]: providerConfig, + }, + }, + }, + }); + + const schedulingConfig: Record = {}; + const normalizedExpectedBaseUrl = expectedBaseUrl.endsWith('/') + ? expectedBaseUrl + : `${expectedBaseUrl}/`; + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + if (scheduleInConfig) { + schedulingConfig.scheduler = { + createScheduledTaskRunner: (_: any) => schedule, + } as unknown as SchedulerService; + } else { + schedulingConfig.schedule = schedule; + } + + const provider = AzureBlobStorageEntityProvider.fromConfig(config, { + ...schedulingConfig, + logger, + })[0]; + + expect(provider.getProviderName()).toEqual( + `azureBlobStorage-provider:${providerId}`, + ); + + try { + await provider.connect(entityProviderConnection); + } catch (error) { + console.error('Error during provider connection:', error); + } + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual( + `azureBlobStorage-provider:${providerId}:refresh`, + ); + + await (taskDef.fn as () => Promise)(); + + const expectedEntities = blobs.map(blob => { + const url = encodeURI(`${normalizedExpectedBaseUrl}${blob}`); + return { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${url}`, + 'backstage.io/managed-by-origin-location': `url:${url}`, + }, + name: expect.stringMatching(/generated-[a-f0-9]{40}/), + }, + spec: { + presence: 'required', + target: `${url}`, + type: 'url', + }, + }, + locationKey: `azureBlobStorage-provider:${providerId}`, + }; + }); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: expectedEntities, + }); + }; + + // eslint-disable-next-line jest/expect-expect + it('apply full update on scheduled execution', async () => { + return expectMutation( + 'staticContainer', + { + containerName, + accountName, + }, + 'https://myaccount.blob.core.windows.net/container-1/', + { + accountName: 'myaccount', + }, + ); + }); + + // eslint-disable-next-line jest/expect-expect + it('apply full update no prefix', async () => { + return expectMutation( + 'staticContainerNoPrefix', + { + containerName, + accountName, + schedule: { + frequency: { minutes: 30 }, + timeout: { minutes: 3 }, + }, + }, + 'https://myaccount.blob.core.windows.net/container-1/', + { + host: 'blob.core.windows.net', + accountName: 'myaccount', + }, + ); + }); + + it('fail without schedule and scheduler', () => { + const config = new ConfigReader({ + catalog: { + providers: { + azureBlob: { + test: { + accountName: 'myaccount', + containerName: 'container-1', + prefix: 'sub/dir/', + }, + }, + }, + }, + }); + + expect(() => + AzureBlobStorageEntityProvider.fromConfig(config, { + logger, + }), + ).toThrow('Either schedule or scheduler must be provided'); + }); + + it('fail with scheduler but no schedule config', () => { + const scheduler = { + createScheduledTaskRunner: (_: any) => jest.fn(), + } as unknown as SchedulerService; + const config = new ConfigReader({ + catalog: { + providers: { + azureBlob: { + test: { + containerName: 'container-1', + prefix: 'sub/dir/', + accountName: 'myaccount', + }, + }, + }, + }, + }); + + expect(() => + AzureBlobStorageEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ).toThrow( + 'No schedule provided neither via code nor config for AzureBlobStorageEntityProvider:test.', + ); + }); +}); diff --git a/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.ts b/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.ts new file mode 100644 index 0000000000..ae2fbcb3ff --- /dev/null +++ b/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.ts @@ -0,0 +1,241 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + AnonymousCredential, + BlobServiceClient, + ContainerClient, + StorageSharedKeyCredential, +} from '@azure/storage-blob'; +import { Config } from '@backstage/config'; +import { + LoggerService, + SchedulerService, + SchedulerServiceTaskRunner, +} from '@backstage/backend-plugin-api'; +import { + EntityProvider, + EntityProviderConnection, + locationSpecToLocationEntity, +} from '@backstage/plugin-catalog-node'; +import { LocationSpec } from '@backstage/plugin-catalog-common'; +import * as uuid from 'uuid'; +import { readAzureBlobStorageConfigs } from './config'; +import { + AzureBlobStorageIntergation, + DefaultAzureCredentialsManager, + ScmIntegrations, +} from '@backstage/integration'; +import { TokenCredential } from '@azure/identity'; +import { AzureBlobStorageConfig } from './types'; + +/** + * Provider which discovers catalog files within an Azure Storage accounts. + * + * Use `AzureBlobStorageEntityProvider.fromConfig(...)` to create instances. + * + * @public + */ +export class AzureBlobStorageEntityProvider implements EntityProvider { + private readonly logger: LoggerService; + private connection?: EntityProviderConnection; + private blobServiceClient?: BlobServiceClient; + private readonly scheduleFn: () => Promise; + + static fromConfig( + configRoot: Config, + options: { + logger: LoggerService; + schedule?: SchedulerServiceTaskRunner; + scheduler?: SchedulerService; + }, + ): AzureBlobStorageEntityProvider[] { + const providerConfigs = readAzureBlobStorageConfigs(configRoot); + + const scmIntegration = ScmIntegrations.fromConfig(configRoot); + const credentialsProvider = + DefaultAzureCredentialsManager.fromIntegrations(scmIntegration); + if (!options.schedule && !options.scheduler) { + throw new Error('Either schedule or scheduler must be provided.'); + } + + return providerConfigs.map(providerConfig => { + const integration = scmIntegration.azureBlobStorage + .list() + .filter( + azureIntegration => + azureIntegration.config.accountName === providerConfig.accountName, + )[0]; + + if (!options.schedule && !providerConfig.schedule) { + throw new Error( + `No schedule provided neither via code nor config for AzureBlobStorageEntityProvider:${providerConfig.id}.`, + ); + } + + if (!integration) { + throw new Error( + `There is no Azure blob storage integration for account. Please add a configuration entry for it under integrations.azureBlobStorage`, + ); + } + + const taskRunner = + options.schedule ?? + options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!); + + return new AzureBlobStorageEntityProvider( + providerConfig, + integration, + credentialsProvider, + options.logger, + taskRunner, + ); + }); + } + private constructor( + private readonly config: AzureBlobStorageConfig, + private readonly integration: AzureBlobStorageIntergation, + private readonly credentialsProvider: DefaultAzureCredentialsManager, + logger: LoggerService, + schedule: SchedulerServiceTaskRunner, + ) { + this.logger = logger.child({ target: this.getProviderName() }); + this.scheduleFn = this.createScheduleFn(schedule); + } + + private createScheduleFn( + taskRunner: SchedulerServiceTaskRunner, + ): () => Promise { + return async () => { + const taskId = `${this.getProviderName()}:refresh`; + return taskRunner.run({ + id: taskId, + fn: async () => { + const logger = this.logger.child({ + class: AzureBlobStorageEntityProvider.prototype.constructor.name, + taskId, + taskInstanceId: uuid.v4(), + }); + + try { + await this.refresh(logger); + } catch (error) { + logger.error( + `${this.getProviderName()} refresh failed, ${error}`, + error, + ); + } + }, + }); + }; + } + + getProviderName(): string { + return `azureBlobStorage-provider:${this.config.id}`; + } + + async connect(connection: EntityProviderConnection): Promise { + this.connection = connection; + let credential: + | TokenCredential + | StorageSharedKeyCredential + | AnonymousCredential; + if (this.integration.config.accountKey) { + credential = new StorageSharedKeyCredential( + this.integration.config.accountName as string, + this.integration.config.accountKey as string, + ); // StorageSharedKeyCredential is only allowed in node.js runtime not in browser + } else { + credential = await this.credentialsProvider.getCredentials( + this.integration.config.accountName as string, + ); + } + let blobServiceClientUrl: string; + + if (this.integration.config.endpoint) { + if (this.integration.config.sasToken) { + blobServiceClientUrl = `${this.integration.config.endpoint}?${this.integration.config.sasToken}`; + } else { + blobServiceClientUrl = `${this.integration.config.endpoint}`; + } + } else { + blobServiceClientUrl = `https://${this.integration.config.accountName}.${this.integration.config.host}`; + } + + this.blobServiceClient = new BlobServiceClient( + blobServiceClientUrl, + credential, + ); + await this.scheduleFn(); + } + + async refresh(logger: LoggerService) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + logger.info('Discovering Azure Blob Storage blobs'); + + const keys = await this.listAllBlobKeys(); + logger.info(`Discovered ${keys.length} Azure Blob Storage blobs`); + + const locations = keys.map(key => this.createLocationSpec(key)); + + await this.connection.applyMutation({ + type: 'full', + entities: locations.map(location => { + return { + locationKey: this.getProviderName(), + entity: locationSpecToLocationEntity({ location }), + }; + }), + }); + + logger.info( + `Committed ${locations.length} Locations for Azure Blob Storage blobs`, + ); + } + + private async listAllBlobKeys(): Promise { + const keys: string[] = []; + const containerClient = this.blobServiceClient?.getContainerClient( + this.config.containerName, + ); + + for await (const blob of ( + containerClient as ContainerClient + ).listBlobsFlat()) { + if (blob.name) { + keys.push(blob.name); + } + } + + return keys; + } + + private createLocationSpec(key: string): LocationSpec { + return { + type: 'url', + target: this.createObjectUrl(key), + presence: 'required', + }; + } + + private createObjectUrl(key: string): string { + const endpoint = this.blobServiceClient?.url; + return `${endpoint}${this.config.containerName}/${key}`; + } +} diff --git a/plugins/catalog-backend-module-azure/src/providers/config.test.ts b/plugins/catalog-backend-module-azure/src/providers/config.test.ts index 5d5ae930c1..78ed559749 100644 --- a/plugins/catalog-backend-module-azure/src/providers/config.test.ts +++ b/plugins/catalog-backend-module-azure/src/providers/config.test.ts @@ -15,7 +15,7 @@ */ import { ConfigReader } from '@backstage/config'; -import { readAzureDevOpsConfigs } from './config'; +import { readAzureBlobStorageConfigs, readAzureDevOpsConfigs } from './config'; describe('readAzureDevOpsConfigs', () => { it('reads all provider configs and set default values', () => { @@ -106,3 +106,74 @@ describe('readAzureDevOpsConfigs', () => { }); }); }); + +describe('readAzureBlobStorageConfigs', () => { + it('reads single and multiple Azure Blob Storage provider configs', () => { + const provider1 = { + accountName: 'account-1', + containerName: 'container-1', + schedule: { + frequency: 'PT30M', + timeout: { + minutes: 3, + }, + }, + }; + const provider2 = { + accountName: 'account-1', + containerName: 'container-2', + }; + + const configSingle = { + catalog: { + providers: { + azureBlob: provider2, + }, + }, + }; + + const configMulti = { + catalog: { + providers: { + azureBlob: { + provider1, + provider2, + }, + }, + }, + }; + + // Single provider case + const actualSingle = readAzureBlobStorageConfigs( + new ConfigReader(configSingle), + ); + expect(actualSingle).toHaveLength(1); + expect(actualSingle[0]).toEqual({ + id: 'default', + accountName: 'account-1', + containerName: 'container-2', + schedule: undefined, // no schedule provided in this case + }); + + // Multiple providers case + const actualMulti = readAzureBlobStorageConfigs( + new ConfigReader(configMulti), + ); + expect(actualMulti).toHaveLength(2); + expect(actualMulti[0]).toEqual({ + id: 'provider1', + accountName: 'account-1', + containerName: 'container-1', + schedule: { + ...provider1.schedule, + frequency: { minutes: 30 }, + }, + }); + expect(actualMulti[1]).toEqual({ + id: 'provider2', + accountName: 'account-1', + containerName: 'container-2', + schedule: undefined, // no schedule provided + }); + }); +}); diff --git a/plugins/catalog-backend-module-azure/src/providers/config.ts b/plugins/catalog-backend-module-azure/src/providers/config.ts index 17616b06bc..8ec1cbfb1b 100644 --- a/plugins/catalog-backend-module-azure/src/providers/config.ts +++ b/plugins/catalog-backend-module-azure/src/providers/config.ts @@ -16,7 +16,9 @@ import { readSchedulerServiceTaskScheduleDefinitionFromConfig } from '@backstage/backend-plugin-api'; import { Config } from '@backstage/config'; -import { AzureDevOpsConfig } from './types'; +import { AzureDevOpsConfig, AzureBlobStorageConfig } from './types'; + +const DEFAULT_PROVIDER_ID = 'default'; export function readAzureDevOpsConfigs(config: Config): AzureDevOpsConfig[] { const configs: AzureDevOpsConfig[] = []; @@ -61,3 +63,52 @@ function readAzureDevOpsConfig(id: string, config: Config): AzureDevOpsConfig { schedule, }; } + +export function readAzureBlobStorageConfigs( + config: Config, +): AzureBlobStorageConfig[] { + const configs: AzureBlobStorageConfig[] = []; + + const providerConfigs = config.getOptionalConfig( + 'catalog.providers.azureBlob', + ); + + if (!providerConfigs) { + return configs; + } + + if (providerConfigs.has('containerName')) { + // simple/single config variant + configs.push( + readAzureBlobStorageConfig(DEFAULT_PROVIDER_ID, providerConfigs), + ); + + return configs; + } + + for (const id of providerConfigs.keys()) { + configs.push(readAzureBlobStorageConfig(id, providerConfigs.getConfig(id))); + } + + return configs; +} + +function readAzureBlobStorageConfig( + id: string, + config: Config, +): AzureBlobStorageConfig { + const containerName = config.getString('containerName'); + const accountName = config.getString('accountName'); + const schedule = config.has('schedule') + ? readSchedulerServiceTaskScheduleDefinitionFromConfig( + config.getConfig('schedule'), + ) + : undefined; + + return { + id, + containerName, + accountName, + schedule, + }; +} diff --git a/plugins/catalog-backend-module-azure/src/providers/index.ts b/plugins/catalog-backend-module-azure/src/providers/index.ts index 450fc0af27..5c9616e162 100644 --- a/plugins/catalog-backend-module-azure/src/providers/index.ts +++ b/plugins/catalog-backend-module-azure/src/providers/index.ts @@ -15,3 +15,4 @@ */ export { AzureDevOpsEntityProvider } from './AzureDevOpsEntityProvider'; +export { AzureBlobStorageEntityProvider } from './AzureBlobStorageEntityProvider'; diff --git a/plugins/catalog-backend-module-azure/src/providers/types.ts b/plugins/catalog-backend-module-azure/src/providers/types.ts index 5ca9dc94dc..e296d50ca7 100644 --- a/plugins/catalog-backend-module-azure/src/providers/types.ts +++ b/plugins/catalog-backend-module-azure/src/providers/types.ts @@ -26,3 +26,10 @@ export type AzureDevOpsConfig = { path: string; schedule?: SchedulerServiceTaskScheduleDefinition; }; + +export type AzureBlobStorageConfig = { + id: string; + containerName: string; + accountName: string; + schedule?: SchedulerServiceTaskScheduleDefinition; +}; diff --git a/yarn.lock b/yarn.lock index 98cead73b1..3f43ff15b8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3581,6 +3581,8 @@ __metadata: "@aws-sdk/credential-providers": ^3.350.0 "@aws-sdk/types": ^3.347.0 "@aws-sdk/util-stream-node": ^3.350.0 + "@azure/identity": ^4.0.0 + "@azure/storage-blob": ^12.5.0 "@backstage/backend-app-api": "workspace:^" "@backstage/backend-dev-utils": "workspace:^" "@backstage/backend-plugin-api": "workspace:^" @@ -4673,6 +4675,7 @@ __metadata: resolution: "@backstage/integration@workspace:packages/integration" dependencies: "@azure/identity": ^4.0.0 + "@azure/storage-blob": ^12.5.0 "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" "@backstage/config-loader": "workspace:^" @@ -5418,6 +5421,8 @@ __metadata: version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-backend-module-azure@workspace:plugins/catalog-backend-module-azure" dependencies: + "@azure/identity": ^4.0.0 + "@azure/storage-blob": ^12.5.0 "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-test-utils": "workspace:^" "@backstage/cli": "workspace:^"