From 277092a876943f96ce8d8ba9d83765919f87a11d Mon Sep 17 00:00:00 2001 From: NIKUNJ LALITKUMAR HUDKA Date: Fri, 18 Oct 2024 18:27:58 -0300 Subject: [PATCH] feat: azure blob storage entity provider for catalog implemented Signed-off-by: NIKUNJ LALITKUMAR HUDKA --- .changeset/cyan-rocks-divide.md | 5 + .changeset/fluffy-cobras-sort.md | 5 + .changeset/tall-comics-heal.md | 5 + app-config.yaml | 18 ++ packages/backend-defaults/package.json | 2 + .../lib/AzureBlobStorageUrlReader.ts | 256 +++++++++++++++++ .../entrypoints/urlReader/lib/UrlReaders.ts | 2 + .../src/entrypoints/urlReader/lib/index.ts | 1 + packages/integration/package.json | 1 + .../integration/src/ScmIntegrations.test.ts | 17 ++ packages/integration/src/ScmIntegrations.ts | 7 + .../AzureBlobStorageIntegration.test.ts | 78 ++++++ .../AzureBlobStorageIntegration.ts | 66 +++++ .../DefaultAzureCredentialsManager.test.ts | 151 ++++++++++ .../DefaultAzureCredentialsProvider.ts | 121 ++++++++ .../src/azureBlobStorage/config.test.ts | 264 ++++++++++++++++++ .../src/azureBlobStorage/config.ts | 170 +++++++++++ .../integration/src/azureBlobStorage/index.ts | 23 ++ .../integration/src/azureBlobStorage/types.ts | 29 ++ packages/integration/src/index.ts | 1 + packages/integration/src/registry.ts | 2 + .../catalog-backend-module-azure/package.json | 2 + .../catalog-backend-module-azure/src/index.ts | 1 + ...logModuleAzureDevOpsEntityProvider.test.ts | 59 +++- .../catalogModuleAzureDevOpsEntityProvider.ts | 34 ++- .../src/module/index.ts | 2 +- .../AzureBlobStorageEntityProvider.test.ts | 246 ++++++++++++++++ .../AzureBlobStorageEntityProvider.ts | 228 +++++++++++++++ .../src/providers/config.test.ts | 68 ++++- .../src/providers/config.ts | 51 +++- .../src/providers/index.ts | 1 + .../src/providers/types.ts | 6 + yarn.lock | 5 + 33 files changed, 1912 insertions(+), 15 deletions(-) create mode 100644 .changeset/cyan-rocks-divide.md create mode 100644 .changeset/fluffy-cobras-sort.md create mode 100644 .changeset/tall-comics-heal.md create mode 100644 packages/backend-defaults/src/entrypoints/urlReader/lib/AzureBlobStorageUrlReader.ts create mode 100644 packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.test.ts create mode 100644 packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.ts create mode 100644 packages/integration/src/azureBlobStorage/DefaultAzureCredentialsManager.test.ts create mode 100644 packages/integration/src/azureBlobStorage/DefaultAzureCredentialsProvider.ts create mode 100644 packages/integration/src/azureBlobStorage/config.test.ts create mode 100644 packages/integration/src/azureBlobStorage/config.ts create mode 100644 packages/integration/src/azureBlobStorage/index.ts create mode 100644 packages/integration/src/azureBlobStorage/types.ts create mode 100644 plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.test.ts create mode 100644 plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.ts diff --git a/.changeset/cyan-rocks-divide.md b/.changeset/cyan-rocks-divide.md new file mode 100644 index 0000000000..8fa76a493e --- /dev/null +++ b/.changeset/cyan-rocks-divide.md @@ -0,0 +1,5 @@ +--- +'@backstage/integration': major +--- + +Add the integration for Azure blob storage to read the credentials to access the storage account and provide the default credential provider. diff --git a/.changeset/fluffy-cobras-sort.md b/.changeset/fluffy-cobras-sort.md new file mode 100644 index 0000000000..363281a6c5 --- /dev/null +++ b/.changeset/fluffy-cobras-sort.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-defaults': minor +--- + +Implemented `AzureBlobStorageUrlReader` to read from the url of committed location from the entity provider diff --git a/.changeset/tall-comics-heal.md b/.changeset/tall-comics-heal.md new file mode 100644 index 0000000000..3d65caee67 --- /dev/null +++ b/.changeset/tall-comics-heal.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-azure': minor +--- + +Added the Azure Blob Storage as catalog entity provider to import all the desired entities from storage account provided in app-config.yaml diff --git a/app-config.yaml b/app-config.yaml index cb70170541..86737413fe 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -221,6 +221,17 @@ integrations: # apiBaseUrl: server.bitbucket.com # username: ${BITBUCKET_SERVER_USERNAME} # appPassword: ${BITBUCKET_SERVER_APP_PASSWORD} + + azureBlobStorage: + - accountName: ${ACCOUNT_NAME} # required + endpoint: ${CUSTOM_ENDPOINT} # custom endpoint will require either aadCredentials or sasToken + sasToken: ${SAS_TOKEN} + aadCredential: + clientId: ${CLIENT_ID} + tenantId: ${TENANT_ID} + clientSecret: ${CLIENT_SECRET} + accountKey: ${ACCOUNT_KEY} + azure: - host: dev.azure.com token: ${AZURE_TOKEN} @@ -245,6 +256,13 @@ catalog: - Domain - Location providers: + azureBlob: + containerName: ${CONTAINER_NAME} + schedule: # same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } backstageOpenapi: plugins: - catalog diff --git a/packages/backend-defaults/package.json b/packages/backend-defaults/package.json index 687d7379f1..754f16cad9 100644 --- a/packages/backend-defaults/package.json +++ b/packages/backend-defaults/package.json @@ -119,6 +119,8 @@ "@aws-sdk/client-s3": "^3.350.0", "@aws-sdk/credential-providers": "^3.350.0", "@aws-sdk/types": "^3.347.0", + "@azure/identity": "^4.0.0", + "@azure/storage-blob": "^12.5.0", "@backstage/backend-app-api": "workspace:^", "@backstage/backend-common": "^0.25.0", "@backstage/backend-dev-utils": "workspace:^", diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureBlobStorageUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureBlobStorageUrlReader.ts new file mode 100644 index 0000000000..f0c7e2570e --- /dev/null +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureBlobStorageUrlReader.ts @@ -0,0 +1,256 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + BlobServiceClient, + ContainerClient, + StorageSharedKeyCredential, +} from '@azure/storage-blob'; +import { ReaderFactory, ReadTreeResponseFactory } from './types'; +import { ForwardedError, NotModifiedError } from '@backstage/errors'; +import { Readable } from 'stream'; +import { relative } from 'path/posix'; +import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; +import { + AzureBlobStorageIntergation, + AzureCredentialsManager, + DefaultAzureCredentialsManager, + ScmIntegrations, +} from '@backstage/integration'; +import { + UrlReaderService, + UrlReaderServiceReadTreeOptions, + UrlReaderServiceReadTreeResponse, + UrlReaderServiceReadUrlOptions, + UrlReaderServiceReadUrlResponse, + UrlReaderServiceSearchResponse, +} from '@backstage/backend-plugin-api'; + +export function parseUrl( + url: string, + config: AzureBlobStorageIntergation, +): { path: string; container: string } { + const parsedUrl = new URL(url); + const pathSegments = parsedUrl.pathname.split('/').filter(Boolean); + + if (pathSegments.length < 2) { + throw new Error(`Invalid Azure Blob Storage URL format: ${url}`); + } + + // First segment is the container name, rest is the blob path + const container = pathSegments[0]; + const path = pathSegments.slice(1).join('/'); + + return { path, container }; +} +export class AzureBlobStorageUrlReader implements UrlReaderService { + static factory: ReaderFactory = ({ config, treeResponseFactory }) => { + const integrations = ScmIntegrations.fromConfig(config); + + const credsManager = + DefaultAzureCredentialsManager.fromIntegrations(integrations); + + return integrations.azureBlobStorage.list().map(integrationConfig => { + const reader = new AzureBlobStorageUrlReader( + credsManager, + integrationConfig, + { + treeResponseFactory, + }, + ); + + const predicate = (url: URL) => + url.host.endsWith(integrationConfig.config.host); + return { reader, predicate }; + }); + }; + + // private readonly blobServiceClient: BlobServiceClient; + + constructor( + private readonly credsManager: AzureCredentialsManager, + private readonly integration: AzureBlobStorageIntergation, + private readonly deps: { + treeResponseFactory: ReadTreeResponseFactory; + }, + ) {} + + private async createContainerClient( + containerName: string, + ): Promise { + const accountName = this.integration.config.accountName; // Use the account name from the integration config + const accountKey = this.integration.config.accountKey; // Get the account key if it exists + + if (accountKey && accountName) { + const creds = new StorageSharedKeyCredential(accountName, accountKey); + const blobServiceClient = new BlobServiceClient( + `https://${accountName}.${this.integration.config.host}`, + creds, + ); + return blobServiceClient.getContainerClient(containerName); + } + // Use the credentials manager to get the correct credentials + const credential = await this.credsManager.getCredentials( + accountName as string, + ); + + let blobServiceClientUrl: string; + + if (this.integration.config.endpoint) { + if (this.integration.config.sasToken) { + blobServiceClientUrl = `${this.integration.config.endpoint}?${this.integration.config.sasToken}`; + } else { + blobServiceClientUrl = `${this.integration.config.endpoint}`; + } + } else { + blobServiceClientUrl = `https://${this.integration.config.accountName}.${this.integration.config.host}`; + } + + const blobServiceClient = new BlobServiceClient( + blobServiceClientUrl, + credential, + ); + return blobServiceClient.getContainerClient(containerName); + } + + async read(url: string): Promise { + const response = await this.readUrl(url); + return response.buffer(); + } + + async readUrl( + url: string, + options?: UrlReaderServiceReadUrlOptions, + ): Promise { + const { etag, lastModifiedAfter } = options ?? {}; + + try { + const { path, container } = parseUrl(url, this.integration); + + const containerClient = await this.createContainerClient(container); + const blobClient = containerClient.getBlobClient(path); + + const abortController = new AbortController(); + + const getBlobOptions = { + conditions: { + ...(etag && { ifNoneMatch: etag }), + ...(lastModifiedAfter && { ifModifiedSince: lastModifiedAfter }), + }, + }; + options?.signal?.addEventListener('abort', () => abortController.abort()); + + const downloadBlockBlobResponse = await blobClient.download( + 0, + undefined, + getBlobOptions, + ); + + const data = await this.retrieveAzureBlobData( + downloadBlockBlobResponse.readableStreamBody as Readable, + ); + + return ReadUrlResponseFactory.fromReadable(data, { + etag: downloadBlockBlobResponse.etag, + lastModifiedAt: downloadBlockBlobResponse.lastModified, + }); + } catch (e) { + if (e.$metadata && e.$metadata.httpStatusCode === 304) { + throw new NotModifiedError(); + } + + throw new ForwardedError( + 'Could not retrieve file from Azure Blob Storage', + e, + ); + } + } + + async readTree( + url: string, + options?: UrlReaderServiceReadTreeOptions, + ): Promise { + try { + const { path, container } = parseUrl(url, this.integration); + + const containerClient = await this.createContainerClient(container); + + const blobs = containerClient.listBlobsFlat({ prefix: path }); + + const responses = []; + const abortController = new AbortController(); + for await (const blob of blobs) { + const blobClient = containerClient.getBlobClient(blob.name); + options?.signal?.addEventListener('abort', () => + abortController.abort(), + ); + const downloadBlockBlobResponse = await blobClient.download(); + const data = await this.retrieveAzureBlobData( + downloadBlockBlobResponse.readableStreamBody as Readable, + ); + + responses.push({ + data: Readable.from(data), + path: relative(path, blob.name), + lastModifiedAt: blob.properties.lastModified, + }); + } + + return this.deps.treeResponseFactory.fromReadableArray(responses); + } catch (e) { + throw new ForwardedError( + 'Could not retrieve file tree from Azure Blob Storage', + e, + ); + } + } + + async search(): Promise { + throw new Error('AzureBlobStorageUrlReader does not implement search'); + } + + toString() { + const accountName = this.integration.config.accountName; + const accountKey = this.integration.config.accountKey; + return `azureBlobStorage{accountName=${accountName},authed=${Boolean( + accountKey, + )}}`; + } + + private parseUrl(url: string): { path: string } { + const parsedUrl = new URL(url); + const path = parsedUrl.pathname.substring( + parsedUrl.pathname.lastIndexOf('/') + 1, + ); + + return { path }; + } + + private async retrieveAzureBlobData(stream: Readable): Promise { + return new Promise((resolve, reject) => { + try { + const chunks: any[] = []; + stream.on('data', chunk => chunks.push(chunk)); + stream.on('error', (e: Error) => + reject(new ForwardedError('Unable to read stream', e)), + ); + stream.on('end', () => resolve(Readable.from(Buffer.concat(chunks)))); + } catch (e) { + throw new ForwardedError('Unable to parse the response data', e); + } + }); + } +} diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/UrlReaders.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/UrlReaders.ts index 96e9ed6509..229bdd4903 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/UrlReaders.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/UrlReaders.ts @@ -35,6 +35,7 @@ import { AwsS3UrlReader } from './AwsS3UrlReader'; import { GiteaUrlReader } from './GiteaUrlReader'; import { AwsCodeCommitUrlReader } from './AwsCodeCommitUrlReader'; import { HarnessUrlReader } from './HarnessUrlReader'; +import { AzureBlobStorageUrlReader } from './AzureBlobStorageUrlReader'; /** * Creation options for {@link @backstage/backend-plugin-api#UrlReaderService}. @@ -99,6 +100,7 @@ export class UrlReaders { GoogleGcsUrlReader.factory, HarnessUrlReader.factory, AwsS3UrlReader.factory, + AzureBlobStorageUrlReader.factory, AwsCodeCommitUrlReader.factory, FetchUrlReader.factory, ]), diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/index.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/index.ts index 678c6ef64b..fc33efbc3a 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/index.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/index.ts @@ -24,6 +24,7 @@ export { GitlabUrlReader } from './GitlabUrlReader'; export { GiteaUrlReader } from './GiteaUrlReader'; export { HarnessUrlReader } from './HarnessUrlReader'; export { AwsS3UrlReader } from './AwsS3UrlReader'; +export { AzureBlobStorageUrlReader } from './AzureBlobStorageUrlReader'; export { FetchUrlReader } from './FetchUrlReader'; export { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; export type { diff --git a/packages/integration/package.json b/packages/integration/package.json index d41ffa501d..4039e94c34 100644 --- a/packages/integration/package.json +++ b/packages/integration/package.json @@ -38,6 +38,7 @@ }, "dependencies": { "@azure/identity": "^4.0.0", + "@azure/storage-blob": "^12.5.0", "@backstage/config": "workspace:^", "@backstage/errors": "workspace:^", "@octokit/auth-app": "^4.0.0", diff --git a/packages/integration/src/ScmIntegrations.test.ts b/packages/integration/src/ScmIntegrations.test.ts index c758816c03..0e0c181861 100644 --- a/packages/integration/src/ScmIntegrations.test.ts +++ b/packages/integration/src/ScmIntegrations.test.ts @@ -39,6 +39,10 @@ import { GiteaIntegration, GiteaIntegrationConfig } from './gitea'; import { AwsCodeCommitIntegration } from './awsCodeCommit/AwsCodeCommitIntegration'; import { AwsCodeCommitIntegrationConfig } from './awsCodeCommit'; import { HarnessIntegration, HarnessIntegrationConfig } from './harness'; +import { + AzureBlobStorageIntegrationConfig, + AzureBlobStorageIntergation, +} from './azureBlobStorage'; describe('ScmIntegrations', () => { const awsS3 = new AwsS3Integration({ @@ -53,6 +57,10 @@ describe('ScmIntegrations', () => { host: 'azure.local', } as AzureIntegrationConfig); + const azureBlob = new AzureBlobStorageIntergation({ + host: 'azureblobstorage.local', + } as AzureBlobStorageIntegrationConfig); + const bitbucket = new BitbucketIntegration({ host: 'bitbucket.local', } as BitbucketIntegrationConfig); @@ -89,6 +97,7 @@ describe('ScmIntegrations', () => { awsS3: basicIntegrations([awsS3], item => item.config.host), awsCodeCommit: basicIntegrations([awsCodeCommit], item => item.config.host), azure: basicIntegrations([azure], item => item.config.host), + azureBlobStorage: basicIntegrations([azureBlob], item => item.config.host), bitbucket: basicIntegrations([bitbucket], item => item.config.host), bitbucketCloud: basicIntegrations([bitbucketCloud], item => item.title), bitbucketServer: basicIntegrations( @@ -108,6 +117,9 @@ describe('ScmIntegrations', () => { awsCodeCommit, ); expect(i.azure.byUrl('https://azure.local')).toBe(azure); + expect(i.azureBlobStorage.byUrl('https://azureblobstorage.local')).toBe( + azureBlob, + ); expect(i.bitbucket.byUrl('https://bitbucket.local')).toBe(bitbucket); expect(i.bitbucketCloud.byUrl('https://bitbucket.org')).toBe( bitbucketCloud, @@ -128,6 +140,7 @@ describe('ScmIntegrations', () => { awsS3, awsCodeCommit, azure, + azureBlob, bitbucket, bitbucketCloud, bitbucketServer, @@ -144,6 +157,9 @@ describe('ScmIntegrations', () => { expect(i.byUrl('https://awss3.local')).toBe(awsS3); expect(i.byUrl('https://awscodecommit.local')).toBe(awsCodeCommit); expect(i.byUrl('https://azure.local')).toBe(azure); + expect(i.azureBlobStorage.byUrl('https://azureblobstorage.local')).toBe( + azureBlob, + ); expect(i.byUrl('https://bitbucket.local')).toBe(bitbucket); expect(i.byUrl('https://bitbucket.org')).toBe(bitbucketCloud); expect(i.byUrl('https://bitbucket-server.local')).toBe(bitbucketServer); @@ -156,6 +172,7 @@ describe('ScmIntegrations', () => { expect(i.byHost('awss3.local')).toBe(awsS3); expect(i.byHost('awscodecommit.local')).toBe(awsCodeCommit); expect(i.byHost('azure.local')).toBe(azure); + expect(i.byHost('azureblobstorage.local')).toBe(azureBlob); expect(i.byHost('bitbucket.local')).toBe(bitbucket); expect(i.byHost('bitbucket.org')).toBe(bitbucketCloud); expect(i.byHost('bitbucket-server.local')).toBe(bitbucketServer); diff --git a/packages/integration/src/ScmIntegrations.ts b/packages/integration/src/ScmIntegrations.ts index e0d430d834..f11cb3c17e 100644 --- a/packages/integration/src/ScmIntegrations.ts +++ b/packages/integration/src/ScmIntegrations.ts @@ -29,6 +29,7 @@ import { ScmIntegration, ScmIntegrationsGroup } from './types'; import { ScmIntegrationRegistry } from './registry'; import { GiteaIntegration } from './gitea'; import { HarnessIntegration } from './harness/HarnessIntegration'; +import { AzureBlobStorageIntergation } from './azureBlobStorage'; /** * The set of supported integrations. @@ -38,6 +39,7 @@ import { HarnessIntegration } from './harness/HarnessIntegration'; export interface IntegrationsByType { awsS3: ScmIntegrationsGroup; awsCodeCommit: ScmIntegrationsGroup; + azureBlobStorage: ScmIntegrationsGroup; azure: ScmIntegrationsGroup; /** * @deprecated in favor of `bitbucketCloud` and `bitbucketServer` @@ -64,6 +66,7 @@ export class ScmIntegrations implements ScmIntegrationRegistry { return new ScmIntegrations({ awsS3: AwsS3Integration.factory({ config }), awsCodeCommit: AwsCodeCommitIntegration.factory({ config }), + azureBlobStorage: AzureBlobStorageIntergation.factory({ config }), azure: AzureIntegration.factory({ config }), bitbucket: BitbucketIntegration.factory({ config }), bitbucketCloud: BitbucketCloudIntegration.factory({ config }), @@ -88,6 +91,10 @@ export class ScmIntegrations implements ScmIntegrationRegistry { return this.byType.awsCodeCommit; } + get azureBlobStorage(): ScmIntegrationsGroup { + return this.byType.azureBlobStorage; + } + get azure(): ScmIntegrationsGroup { return this.byType.azure; } diff --git a/packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.test.ts b/packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.test.ts new file mode 100644 index 0000000000..e7f7703be9 --- /dev/null +++ b/packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.test.ts @@ -0,0 +1,78 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { AzureBlobStorageIntergation } from './AzureBlobStorageIntegration'; + +describe('AzureBlobStorageIntegration', () => { + it('has a working factory', () => { + const integrations = AzureBlobStorageIntergation.factory({ + config: new ConfigReader({ + integrations: { + azureBlobStorage: [ + { + endpoint: 'https://myaccount.blob.core.windows.net', + accountName: 'myaccount', + accountKey: 'someAccountKey', + }, + ], + }, + }), + }); + expect(integrations.list().length).toBe(2); // including default + expect(integrations.list()[0].config.host).toBe( + 'myaccount.blob.core.windows.net', + ); + expect(integrations.list()[1].config.host).toBe('blob.core.windows.net'); // default integration + }); + + it('returns the basics', () => { + const integration = new AzureBlobStorageIntergation({ + host: 'myaccount.blob.core.windows.net', + } as any); + expect(integration.type).toBe('azureBlobStorage'); + expect(integration.title).toBe('myaccount.blob.core.windows.net'); + }); + + describe('resolveUrl', () => { + it('works for valid URLs', () => { + const integration = new AzureBlobStorageIntergation({ + host: 'blob.core.windows.net', + } as any); + + expect( + integration.resolveUrl({ + url: 'https://myaccount.blob.core.windows.net/container/file.yaml', + base: 'https://myaccount.blob.core.windows.net/container/file.yaml', + }), + ).toBe('https://myaccount.blob.core.windows.net/container/file.yaml'); + }); + }); + + it('resolve edit URL', () => { + const integration = new AzureBlobStorageIntergation({ + host: 'myaccount.blob.core.windows.net', + } as any); + + // TODO: The Azure Blob Storage integration doesn't support resolving an edit URL, + // instead we keep the input URL. + expect( + integration.resolveEditUrl( + 'https://myaccount.blob.core.windows.net/container/file.yaml', + ), + ).toBe('https://myaccount.blob.core.windows.net/container/file.yaml'); + }); +}); diff --git a/packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.ts b/packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.ts new file mode 100644 index 0000000000..01e6bbc317 --- /dev/null +++ b/packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.ts @@ -0,0 +1,66 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { basicIntegrations, defaultScmResolveUrl } from '../helpers'; +import { ScmIntegration, ScmIntegrationsFactory } from '../types'; +import { + AzureBlobStorageIntegrationConfig, + readAzureBlobStorageIntegrationConfigs, +} from './config'; + +export class AzureBlobStorageIntergation implements ScmIntegration { + static factory: ScmIntegrationsFactory = ({ + config, + }) => { + const configs = readAzureBlobStorageIntegrationConfigs( + config.getOptionalConfigArray('integrations.azureBlobStorage') ?? [], + ); + return basicIntegrations( + configs.map(c => new AzureBlobStorageIntergation(c)), + i => i.config.host, + ); + }; + + get type(): string { + return 'azureBlobStorage'; + } + + get title(): string { + return this.integrationConfig.host; + } + + get config(): AzureBlobStorageIntegrationConfig { + return this.integrationConfig; + } + + constructor( + private readonly integrationConfig: AzureBlobStorageIntegrationConfig, + ) {} + + resolveUrl(options: { + url: string; + base: string; + lineNumber?: number | undefined; + }): string { + const resolved = defaultScmResolveUrl(options); + return resolved; + } + + resolveEditUrl(url: string): string { + // TODO: Implement edit URL for azureBlobStorage + return url; + } +} diff --git a/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsManager.test.ts b/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsManager.test.ts new file mode 100644 index 0000000000..6272a68d13 --- /dev/null +++ b/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsManager.test.ts @@ -0,0 +1,151 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + AccessToken, + ClientSecretCredential, + DefaultAzureCredential, + TokenCredential, +} from '@azure/identity'; +import { AzureBlobStorageIntegrationConfig } from './config'; +import { ScmIntegrationRegistry } from '../registry'; +import { ConfigReader } from '@backstage/config'; +import { DefaultAzureCredentialsManager } from './DefaultAzureCredentialsProvider'; +import { ScmIntegrations } from '../ScmIntegrations'; +import { DateTime } from 'luxon'; + +const MockedClientSecretCredential = ClientSecretCredential as jest.MockedClass< + typeof ClientSecretCredential +>; + +jest.mock('@azure/identity'); + +describe('DefaultAzureCredentialsManager', () => { + let mockIntegration: ScmIntegrationRegistry; + + const buildProvider = (azureIntegrations: any[]) => + DefaultAzureCredentialsManager.fromIntegrations( + ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + azureBlobStorage: azureIntegrations, + }, + }), + ), + ); + + beforeEach(() => { + mockIntegration = { + azureBlobStorage: { + list: jest.fn().mockReturnValue([ + { + config: { + accountName: 'testaccount', + aadCredential: { + clientId: 'someClientId', + tenantId: 'someTenantId', + clientSecret: 'someClientSecret', + }, + }, + }, + ]), + }, + } as unknown as ScmIntegrationRegistry; + + MockedClientSecretCredential.prototype.getToken.mockImplementation(() => + Promise.resolve({ + expiresOnTimestamp: DateTime.local().plus({ days: 1 }).toSeconds(), + token: 'fake-client-secret-token', + } as AccessToken), + ); + }); + + it('should create an instance from ScmIntegrationRegistry', () => { + const manager = + DefaultAzureCredentialsManager.fromIntegrations(mockIntegration); + expect(manager).toBeInstanceOf(DefaultAzureCredentialsManager); + }); + + it('should return cached credentials if available', async () => { + const manager = buildProvider([ + { + accountName: 'testaccount', + aadCredential: { + clientId: 'someClientId', + tenantId: 'someTenantId', + clientSecret: 'someClientSecret', + }, + }, + ]); + + const mockCredential = new MockedClientSecretCredential( + 'someTenantId', + 'someClientId', + 'someClientSecret', + ); + + const credential = await manager.getCredentials('testaccount'); + + const scopes = ['https://storage.azure.com/.default']; + + const expectedToken = await mockCredential.getToken(scopes); + const receivedToken = await credential.getToken(scopes); + + expect(receivedToken?.token).toEqual(expectedToken.token); + }); + + it('should use Azure AD credentials if aadCredential is provided', async () => { + const manager = buildProvider([ + { + accountName: 'testaccount', + aadCredential: { + clientId: 'someClientId', + tenantId: 'someTenantId', + clientSecret: 'someClientSecret', + }, + }, + ]); + + const credential = await manager.getCredentials('testaccount'); + + expect(credential).toBeInstanceOf(ClientSecretCredential); + }); + + it('should use DefaultAzureCredential if no aadCredential is provided', async () => { + const manager = buildProvider([ + { + accountName: 'testaccount', + }, + ]); + + const credential = await manager.getCredentials('testaccount'); + + expect(credential).toBeInstanceOf(DefaultAzureCredential); + }); + + it('should cache credentials after first retrieval', async () => { + const manager = buildProvider([ + { + accountName: 'testaccount', + }, + ]); + + const credential = await manager.getCredentials('testaccount'); + + const cachedCredential = await manager.getCredentials('testaccount'); + expect(cachedCredential).toBe(credential); + }); +}); diff --git a/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsProvider.ts b/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsProvider.ts new file mode 100644 index 0000000000..9f209da61c --- /dev/null +++ b/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsProvider.ts @@ -0,0 +1,121 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + DefaultAzureCredential, + ClientSecretCredential, + TokenCredential, +} from '@azure/identity'; +import { AzureBlobStorageIntegrationConfig } from './config'; +import { AzureCredentialsManager } from './types'; +import { ScmIntegrationRegistry } from '../registry'; + +/** + * Default Azure Credentials Manager to dynamically select and manage Azure credentials. + * It supports Service Principal, Managed Identity, SAS Token, Connection String, Account Key, and Anonymous access. + */ +export class DefaultAzureCredentialsManager implements AzureCredentialsManager { + private config: AzureBlobStorageIntegrationConfig; + private cachedCredentials: Map; + + constructor(config: AzureBlobStorageIntegrationConfig) { + this.config = config; + this.cachedCredentials = new Map(); + } + + /** + * Creates an instance of DefaultAzureCredentialsManager from a Backstage Config. + */ + static fromIntegrations( + integration: ScmIntegrationRegistry, + ): DefaultAzureCredentialsManager { + const azureConfig = integration.azureBlobStorage.list().length + ? integration.azureBlobStorage.list()[0].config + : { host: 'blob.core.windows.net' }; // Default to Azure Blob Storage host if no config found + + return new DefaultAzureCredentialsManager(azureConfig); + } + + /** + * Determines the appropriate credential method and returns credentials for BlobServiceClient. + * Supports: + * - Service Principal + * - Managed Identity + * - SAS Token + * - Connection String + * - Account Key + * - Anonymous access + */ + async getCredentials(accountName: string): Promise { + // Check if the credentials are already cached + if (this.cachedCredentials.has(accountName)) { + return this.cachedCredentials.get(accountName)!; + } + + let credential: TokenCredential; + + // Check for SAS Token + // if (this.config.sasToken) { + // // console.log('Using SAS Token for Azure Blob Storage authentication'); + // // SAS Token does not return a credential but can be used directly in BlobServiceClient + // // Here we can simply return undefined or keep a placeholder if needed + // return this.config.sasToken; // Or return a string for the URL using the SAS token + // } + // // Check for Connection String + // else if (this.config.connectionString) { + // // console.log( + // // 'Using Connection String for Azure Blob Storage authentication', + // // ); + // // return undefined; // Connection string will also not return a specific credential object + // } + // Check for Account Key + // if (this.config.accountKey) { + // // console.log('Using Account Key for Azure Blob Storage authentication'); + // credential = new StorageSharedKeyCredential( + // accountName, + // this.config.accountKey, + // ); + // } + // Check for AAD credentials + + if ( + this.config.aadCredential && + this.config.aadCredential.clientId && + this.config.aadCredential.clientSecret && + this.config.aadCredential.tenantId + ) { + credential = new ClientSecretCredential( + this.config.aadCredential.tenantId, + this.config.aadCredential.clientId, + this.config.aadCredential.clientSecret, + ); + } + // Check for Anonymous access + // else if (this.config.anonymousAccess) { + // console.log('Using Anonymous Credential for Azure Blob Storage access'); + // credential = new AnonymousCredential(); + // } + // Fallback to Managed Identity + else { + credential = new DefaultAzureCredential(); + } + + // Cache the credentials for future use + this.cachedCredentials.set(accountName, credential); + + return credential; + } +} diff --git a/packages/integration/src/azureBlobStorage/config.test.ts b/packages/integration/src/azureBlobStorage/config.test.ts new file mode 100644 index 0000000000..f17bdf3efb --- /dev/null +++ b/packages/integration/src/azureBlobStorage/config.test.ts @@ -0,0 +1,264 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Config, ConfigReader } from '@backstage/config'; +import { + AzureBlobStorageIntegrationConfig, + readAzureBlobStorageIntegrationConfig, + readAzureBlobStorageIntegrationConfigs, +} from './config'; + +describe('readAzureBlobStorageIntegrationConfig', () => { + function buildConfig( + data: Partial, + ): Config { + return new ConfigReader(data); + } + + it('reads valid configuration with accountKey', () => { + const output = readAzureBlobStorageIntegrationConfig( + buildConfig({ + accountName: 'mystorageaccount', + accountKey: 'someAccountKey', + }), + ); + expect(output).toEqual({ + host: 'blob.core.windows.net', + endpoint: undefined, + accountName: 'mystorageaccount', + accountKey: 'someAccountKey', + sasToken: undefined, + connectionString: undefined, + endpointSuffix: undefined, + aadCredential: undefined, + }); + }); + + it('reads valid configuration with sasToken', () => { + const output = readAzureBlobStorageIntegrationConfig( + buildConfig({ + endpoint: 'https://blob.core.windows.net', + accountName: 'mystorageaccount', + sasToken: 'someSASToken', + }), + ); + expect(output).toEqual({ + host: 'blob.core.windows.net', + endpoint: 'https://blob.core.windows.net', + accountName: 'mystorageaccount', + accountKey: undefined, + sasToken: 'someSASToken', + connectionString: undefined, + endpointSuffix: undefined, + aadCredential: undefined, + }); + }); + + it('reads valid configuration with Azure AD credentials', () => { + const output = readAzureBlobStorageIntegrationConfig( + buildConfig({ + accountName: 'mystorageaccount', + aadCredential: { + clientId: 'someClientId', + tenantId: 'someTenantId', + clientSecret: 'someClientSecret', + }, + }), + ); + expect(output).toEqual({ + host: 'blob.core.windows.net', + endpoint: undefined, + accountName: 'mystorageaccount', + accountKey: undefined, + sasToken: undefined, + connectionString: undefined, + endpointSuffix: undefined, + aadCredential: { + clientId: 'someClientId', + tenantId: 'someTenantId', + clientSecret: 'someClientSecret', + }, + }); + }); + + it('reads valid configuration with a custom endpoint', () => { + const output = readAzureBlobStorageIntegrationConfig( + buildConfig({ + endpoint: 'https://custom.blob.core.windows.net', + accountName: 'customaccount', + }), + ); + expect(output).toEqual({ + host: 'custom.blob.core.windows.net', + endpoint: 'https://custom.blob.core.windows.net', + accountName: 'customaccount', + accountKey: undefined, + sasToken: undefined, + connectionString: undefined, + endpointSuffix: undefined, + aadCredential: undefined, + }); + }); + + it('throws error for invalid endpoint URL', () => { + const config = buildConfig({ + endpoint: 'invalid-url', + accountName: 'invalidaccount', + }); + + expect(() => readAzureBlobStorageIntegrationConfig(config)).toThrow( + `invalid azureBlobStorage integration config, endpoint 'invalid-url' is not a valid URL`, + ); + }); + + it('throws error if endpoint has a path', () => { + const config = buildConfig({ + endpoint: 'https://blob.core.windows.net/path', + accountName: 'accountwithpath', + }); + + expect(() => readAzureBlobStorageIntegrationConfig(config)).toThrow( + `invalid azureBlobStorage integration config, endpoints cannot contain path, got 'https://blob.core.windows.net/path'`, + ); + }); + + it('throws error if both accountKey and sasToken are provided', () => { + const config = buildConfig({ + accountName: 'mystorageaccount', + accountKey: 'someAccountKey', + sasToken: 'someSASToken', + }); + + expect(() => readAzureBlobStorageIntegrationConfig(config)).toThrow( + `Invalid Azure Blob Storage config for mystorageaccount: Both account key and SAS token cannot be used simultaneously.`, + ); + }); + + it('throws error if both aadCredential and accountKey/sasToken are provided', () => { + const config = buildConfig({ + accountName: 'mystorageaccount', + accountKey: 'someAccountKey', + aadCredential: { + clientId: 'someClientId', + tenantId: 'someTenantId', + clientSecret: 'someClientSecret', + }, + }); + + expect(() => readAzureBlobStorageIntegrationConfig(config)).toThrow( + `Invalid Azure Blob Storage config for mystorageaccount: Cannot use both Azure AD credentials and account keys/SAS tokens for the same account.`, + ); + }); +}); + +describe('readAzureBlobStorageIntegrationConfigs', () => { + function buildConfigs( + data: Partial[], + ): Config[] { + return data.map(item => new ConfigReader(item)); + } + + it('reads all provided configurations', () => { + const output = readAzureBlobStorageIntegrationConfigs( + buildConfigs([ + { + host: 'blob.core.windows.net', + accountName: 'account1', + accountKey: 'someAccountKey', + }, + { + endpoint: 'https://custom.blob.core.windows.net', + accountName: 'account2', + }, + ]), + ); + expect(output).toEqual([ + { + host: 'blob.core.windows.net', + endpoint: undefined, + accountName: 'account1', + accountKey: 'someAccountKey', + sasToken: undefined, + connectionString: undefined, + endpointSuffix: undefined, + aadCredential: undefined, + }, + { + host: 'custom.blob.core.windows.net', + endpoint: 'https://custom.blob.core.windows.net', + accountName: 'account2', + accountKey: undefined, + sasToken: undefined, + connectionString: undefined, + endpointSuffix: undefined, + aadCredential: undefined, + }, + ]); + }); + + it('adds default integration for blob.core.windows.net when missing', () => { + const output = readAzureBlobStorageIntegrationConfigs( + buildConfigs([ + { + endpoint: 'https://custom.blob.core.windows.net', + accountName: 'account2', + }, + ]), + ); + + expect(output).toEqual([ + { + host: 'custom.blob.core.windows.net', + endpoint: 'https://custom.blob.core.windows.net', + accountName: 'account2', + accountKey: undefined, + sasToken: undefined, + connectionString: undefined, + endpointSuffix: undefined, + aadCredential: undefined, + }, + { + host: 'blob.core.windows.net', + endpoint: undefined, + accountName: undefined, + accountKey: undefined, + sasToken: undefined, + connectionString: undefined, + endpointSuffix: undefined, + aadCredential: undefined, + }, + ]); + }); + + it('does not add default integration if blob.core.windows.net already exists', () => { + const output = readAzureBlobStorageIntegrationConfigs( + buildConfigs([ + { host: 'blob.core.windows.net', accountName: 'account1' }, + ]), + ); + expect(output).toEqual([ + { + host: 'blob.core.windows.net', + endpoint: undefined, + accountName: 'account1', + accountKey: undefined, + sasToken: undefined, + connectionString: undefined, + endpointSuffix: undefined, + aadCredential: undefined, + }, + ]); + }); +}); diff --git a/packages/integration/src/azureBlobStorage/config.ts b/packages/integration/src/azureBlobStorage/config.ts new file mode 100644 index 0000000000..10eccc252c --- /dev/null +++ b/packages/integration/src/azureBlobStorage/config.ts @@ -0,0 +1,170 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; + +const AZURE_HOST = 'blob.core.windows.net'; + +/** + * The configuration parameters for a single Azure Blob Storage account. + * + * @public + */ +export type AzureBlobStorageIntegrationConfig = { + /** + * The name of the Azure Storage Account, e.g., "mystorageaccount". + */ + accountName?: string; + + /** + * The primary or secondary key for the Azure Storage Account. + * Only required if connectionString or SAS token are not specified. + */ + accountKey?: string; + + /** + * A Shared Access Signature (SAS) token for limited access to resources. + */ + sasToken?: string; + + /** + * A full connection string for the Azure Storage Account. + * This includes the account name, key, and endpoint details. + */ + connectionString?: string; + + /** + * Optional endpoint suffix for custom domains or sovereign clouds. + * e.g., "core.windows.net" for public Azure or "core.usgovcloudapi.net" for US Government cloud. + */ + endpointSuffix?: string; + + /** + * The host of the target that this matches on, e.g., "blob.core.windows.net". + */ + host: string; + + endpoint?: string; + /** + * Optional credential to use for Azure Active Directory authentication. + */ + aadCredential?: { + /** + * The client ID of the Azure AD application. + */ + clientId: string; + + /** + * The tenant ID for Azure AD. + */ + tenantId: string; + + /** + * The client secret for the Azure AD application. + */ + clientSecret: string; + }; +}; + +/** + * Reads a single Azure Blob Storage integration config. + * + * @param config - The config object of a single integration. + * @public + */ +export function readAzureBlobStorageIntegrationConfig( + config: Config, +): AzureBlobStorageIntegrationConfig { + const endpoint = config.getOptionalString('endpoint'); + const accountName = config.getString('accountName'); + const accountKey = config.getOptionalString('accountKey')?.trim(); + const sasToken = config.getOptionalString('sasToken')?.trim(); + const connectionString = config.getOptionalString('connectionString')?.trim(); + const endpointSuffix = config.getOptionalString('endpointSuffix')?.trim(); + + let host; + let pathname; + if (endpoint) { + try { + const url = new URL(endpoint); + host = url.host; + pathname = url.pathname; + } catch { + throw new Error( + `invalid azureBlobStorage integration config, endpoint '${endpoint}' is not a valid URL`, + ); + } + if (pathname !== '/') { + throw new Error( + `invalid azureBlobStorage integration config, endpoints cannot contain path, got '${endpoint}'`, + ); + } + } else { + host = AZURE_HOST; + } + const aadCredential = config.has('aadCredential') + ? { + clientId: config.getString('aadCredential.clientId'), + tenantId: config.getString('aadCredential.tenantId'), + clientSecret: config.getString('aadCredential.clientSecret')?.trim(), + } + : undefined; + + if (accountKey && sasToken) { + throw new Error( + `Invalid Azure Blob Storage config for ${accountName}: Both account key and SAS token cannot be used simultaneously.`, + ); + } + + if (aadCredential && (accountKey || sasToken)) { + throw new Error( + `Invalid Azure Blob Storage config for ${accountName}: Cannot use both Azure AD credentials and account keys/SAS tokens for the same account.`, + ); + } + + return { + host, + endpoint, + accountName, + accountKey, + sasToken, + connectionString, + endpointSuffix, + aadCredential, + }; +} + +/** + * Reads a set of Azure Blob Storage integration configs. + * + * @param configs - All of the integration config objects. + * @public + */ +export function readAzureBlobStorageIntegrationConfigs( + configs: Config[], +): AzureBlobStorageIntegrationConfig[] { + // First read all the explicit integrations + const result = configs.map(readAzureBlobStorageIntegrationConfig); + + // If no explicit blob.core.windows.net integration was added, put one in the list as + // a convenience + if (!result.some(c => c.host === AZURE_HOST)) { + result.push({ + host: AZURE_HOST, + }); + } + return result; +} diff --git a/packages/integration/src/azureBlobStorage/index.ts b/packages/integration/src/azureBlobStorage/index.ts new file mode 100644 index 0000000000..f55e9aaa79 --- /dev/null +++ b/packages/integration/src/azureBlobStorage/index.ts @@ -0,0 +1,23 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export { AzureBlobStorageIntergation } from './AzureBlobStorageIntegration'; +export { + readAzureBlobStorageIntegrationConfig, + readAzureBlobStorageIntegrationConfigs, +} from './config'; +export type { AzureBlobStorageIntegrationConfig } from './config'; +export { DefaultAzureCredentialsManager } from './DefaultAzureCredentialsProvider'; +export type { AzureCredentialsManager } from './types'; diff --git a/packages/integration/src/azureBlobStorage/types.ts b/packages/integration/src/azureBlobStorage/types.ts new file mode 100644 index 0000000000..99bcbec57f --- /dev/null +++ b/packages/integration/src/azureBlobStorage/types.ts @@ -0,0 +1,29 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TokenCredential } from '@azure/identity'; +import { + StorageSharedKeyCredential, + AnonymousCredential, +} from '@azure/storage-blob'; + +export interface AzureCredentialsManager { + getCredentials( + accountName: string, + ): Promise< + TokenCredential | StorageSharedKeyCredential | AnonymousCredential + >; +} diff --git a/packages/integration/src/index.ts b/packages/integration/src/index.ts index 32c573abef..a625f5d83c 100644 --- a/packages/integration/src/index.ts +++ b/packages/integration/src/index.ts @@ -22,6 +22,7 @@ export * from './awsS3'; export * from './awsCodeCommit'; +export * from './azureBlobStorage'; export * from './azure'; export * from './bitbucket'; export * from './bitbucketCloud'; diff --git a/packages/integration/src/registry.ts b/packages/integration/src/registry.ts index 7e4b34cc34..2e8111903b 100644 --- a/packages/integration/src/registry.ts +++ b/packages/integration/src/registry.ts @@ -26,6 +26,7 @@ import { GithubIntegration } from './github/GithubIntegration'; import { GitLabIntegration } from './gitlab/GitLabIntegration'; import { GiteaIntegration } from './gitea/GiteaIntegration'; import { HarnessIntegration } from './harness/HarnessIntegration'; +import { AzureBlobStorageIntergation } from './azureBlobStorage'; /** * Holds all registered SCM integrations, of all types. @@ -36,6 +37,7 @@ export interface ScmIntegrationRegistry extends ScmIntegrationsGroup { awsS3: ScmIntegrationsGroup; awsCodeCommit: ScmIntegrationsGroup; + azureBlobStorage: ScmIntegrationsGroup; azure: ScmIntegrationsGroup; /** * @deprecated in favor of `bitbucketCloud` and `bitbucketServer` diff --git a/plugins/catalog-backend-module-azure/package.json b/plugins/catalog-backend-module-azure/package.json index 774db39280..c5ab759113 100644 --- a/plugins/catalog-backend-module-azure/package.json +++ b/plugins/catalog-backend-module-azure/package.json @@ -51,6 +51,8 @@ "test": "backstage-cli package test" }, "dependencies": { + "@azure/identity": "^4.0.0", + "@azure/storage-blob": "^12.5.0", "@backstage/backend-plugin-api": "workspace:^", "@backstage/config": "workspace:^", "@backstage/integration": "workspace:^", diff --git a/plugins/catalog-backend-module-azure/src/index.ts b/plugins/catalog-backend-module-azure/src/index.ts index dbbdff67ce..da1aaef30f 100644 --- a/plugins/catalog-backend-module-azure/src/index.ts +++ b/plugins/catalog-backend-module-azure/src/index.ts @@ -23,3 +23,4 @@ export { default } from './module'; export { AzureDevOpsDiscoveryProcessor } from './processors'; export { AzureDevOpsEntityProvider } from './providers'; +export { AzureBlobStorageEntityProvider } from './providers'; diff --git a/plugins/catalog-backend-module-azure/src/module/catalogModuleAzureDevOpsEntityProvider.test.ts b/plugins/catalog-backend-module-azure/src/module/catalogModuleAzureDevOpsEntityProvider.test.ts index 32eb190460..e47635de91 100644 --- a/plugins/catalog-backend-module-azure/src/module/catalogModuleAzureDevOpsEntityProvider.test.ts +++ b/plugins/catalog-backend-module-azure/src/module/catalogModuleAzureDevOpsEntityProvider.test.ts @@ -17,8 +17,11 @@ import { SchedulerServiceTaskScheduleDefinition } from '@backstage/backend-plugin-api'; import { mockServices, startTestBackend } from '@backstage/backend-test-utils'; import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; -import { catalogModuleAzureDevOpsEntityProvider } from './catalogModuleAzureDevOpsEntityProvider'; -import { AzureDevOpsEntityProvider } from '../providers'; +import { catalogModuleAzureEntityProvider } from './catalogModuleAzureDevOpsEntityProvider'; +import { + AzureBlobStorageEntityProvider, + AzureDevOpsEntityProvider, +} from '../providers'; describe('catalogModuleAzureDevOpsEntityProvider', () => { it('should register provider at the catalog extension point', async () => { @@ -58,7 +61,7 @@ describe('catalogModuleAzureDevOpsEntityProvider', () => { await startTestBackend({ extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]], features: [ - catalogModuleAzureDevOpsEntityProvider, + catalogModuleAzureEntityProvider, mockServices.rootConfig.factory({ data: config }), mockServices.logger.factory(), scheduler.factory, @@ -73,4 +76,54 @@ describe('catalogModuleAzureDevOpsEntityProvider', () => { ); expect(runner).not.toHaveBeenCalled(); }); + + it('should register azure blob storage provider at the catalog extension point', async () => { + let addedProviders: Array | undefined; + let usedSchedule: SchedulerServiceTaskScheduleDefinition | undefined; + + const extensionPoint = { + addEntityProvider: (providers: any) => { + addedProviders = providers; + }, + }; + const runner = jest.fn(); + const scheduler = mockServices.scheduler.mock({ + createScheduledTaskRunner(schedule) { + usedSchedule = schedule; + return { run: runner }; + }, + }); + + const config = { + catalog: { + providers: { + azureBlob: { + containerName: 'test', + schedule: { + frequency: 'P1M', + timeout: 'PT3M', + }, + }, + }, + }, + }; + + await startTestBackend({ + extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]], + features: [ + catalogModuleAzureEntityProvider, + mockServices.rootConfig.factory({ data: config }), + mockServices.logger.factory(), + scheduler.factory, + ], + }); + + expect(usedSchedule?.frequency).toEqual({ months: 1 }); + expect(usedSchedule?.timeout).toEqual({ minutes: 3 }); + expect(addedProviders?.length).toEqual(1); + expect(addedProviders?.pop()?.getProviderName()).toEqual( + 'azureBlobStorage-provider:default', + ); + expect(runner).not.toHaveBeenCalled(); + }); }); diff --git a/plugins/catalog-backend-module-azure/src/module/catalogModuleAzureDevOpsEntityProvider.ts b/plugins/catalog-backend-module-azure/src/module/catalogModuleAzureDevOpsEntityProvider.ts index 3aaa1b5b08..cc808292f3 100644 --- a/plugins/catalog-backend-module-azure/src/module/catalogModuleAzureDevOpsEntityProvider.ts +++ b/plugins/catalog-backend-module-azure/src/module/catalogModuleAzureDevOpsEntityProvider.ts @@ -19,16 +19,19 @@ import { createBackendModule, } from '@backstage/backend-plugin-api'; import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; -import { AzureDevOpsEntityProvider } from '../providers'; +import { + AzureBlobStorageEntityProvider, + AzureDevOpsEntityProvider, +} from '../providers'; /** * Registers the AzureDevOpsEntityProvider with the catalog processing extension point. * * @public */ -export const catalogModuleAzureDevOpsEntityProvider = createBackendModule({ +export const catalogModuleAzureEntityProvider = createBackendModule({ pluginId: 'catalog', - moduleId: 'azure-dev-ops-entity-provider', + moduleId: 'azure-providers', register(env) { env.registerInit({ deps: { @@ -38,12 +41,25 @@ export const catalogModuleAzureDevOpsEntityProvider = createBackendModule({ scheduler: coreServices.scheduler, }, async init({ config, catalog, logger, scheduler }) { - catalog.addEntityProvider( - AzureDevOpsEntityProvider.fromConfig(config, { - logger, - scheduler, - }), - ); + // Check for Azure Blob Storage provider configuration and register it + if (config.has('catalog.providers.azureBlob')) { + catalog.addEntityProvider( + AzureBlobStorageEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ); + } + + // Check for Azure DevOps provider configuration and register it + if (config.has('catalog.providers.azureDevOps')) { + catalog.addEntityProvider( + AzureDevOpsEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ); + } }, }); }, diff --git a/plugins/catalog-backend-module-azure/src/module/index.ts b/plugins/catalog-backend-module-azure/src/module/index.ts index 329ad604e2..6e560cf17f 100644 --- a/plugins/catalog-backend-module-azure/src/module/index.ts +++ b/plugins/catalog-backend-module-azure/src/module/index.ts @@ -14,4 +14,4 @@ * limitations under the License. */ -export { catalogModuleAzureDevOpsEntityProvider as default } from './catalogModuleAzureDevOpsEntityProvider'; +export { catalogModuleAzureEntityProvider as default } from './catalogModuleAzureDevOpsEntityProvider'; diff --git a/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.test.ts b/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.test.ts new file mode 100644 index 0000000000..5ec50754c0 --- /dev/null +++ b/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.test.ts @@ -0,0 +1,246 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + SchedulerService, + SchedulerServiceTaskRunner, + SchedulerServiceTaskInvocationDefinition, +} from '@backstage/backend-plugin-api'; +import { ConfigReader } from '@backstage/config'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; +import { AzureBlobStorageEntityProvider } from './AzureBlobStorageEntityProvider'; +import { mockServices } from '@backstage/backend-test-utils'; + +class PersistingTaskRunner implements SchedulerServiceTaskRunner { + private tasks: SchedulerServiceTaskInvocationDefinition[] = []; + + getTasks() { + return this.tasks; + } + + run(task: SchedulerServiceTaskInvocationDefinition): Promise { + this.tasks.push(task); + return Promise.resolve(undefined); + } +} + +const blobs = ['key1.yaml', 'key2.yaml', 'key3.yaml', 'key4.yaml']; +const createBlobList = (blobsArray: string[]) => { + return blobsArray.map(blob => ({ + name: blob, + })); +}; +// Mocking Azure Storage Blob Library +jest.mock('@azure/storage-blob', () => { + return { + BlobServiceClient: jest.fn().mockImplementation(() => ({ + url: 'https://myaccount.blob.core.windows.net/', + getContainerClient: jest.fn().mockImplementation(() => ({ + listBlobsFlat: jest.fn(async function* () { + yield* createBlobList(blobs); + }), + })), + })), + }; +}); + +const logger = mockServices.logger.mock(); + +describe('AzureBlobStorageEntityProvider', () => { + const containerName = 'container-1'; + + const expectMutation = async ( + providerId: string, + providerConfig: object, + expectedBaseUrl: string, + names: Record, + integrationConfig?: object, + scheduleInConfig?: boolean, + ) => { + const config = new ConfigReader({ + integrations: { + azureBlobStorage: integrationConfig ? [integrationConfig] : [], + }, + catalog: { + providers: { + azureBlob: { + [providerId]: providerConfig, + }, + }, + }, + }); + + const schedulingConfig: Record = {}; + const normalizedExpectedBaseUrl = expectedBaseUrl.endsWith('/') + ? expectedBaseUrl + : `${expectedBaseUrl}/`; + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + if (scheduleInConfig) { + schedulingConfig.scheduler = { + createScheduledTaskRunner: (_: any) => schedule, + } as unknown as SchedulerService; + } else { + schedulingConfig.schedule = schedule; + } + + const provider = AzureBlobStorageEntityProvider.fromConfig(config, { + ...schedulingConfig, + logger, + })[0]; + + expect(provider.getProviderName()).toEqual( + `azureBlobStorage-provider:${providerId}`, + ); + + try { + await provider.connect(entityProviderConnection); + } catch (error) { + console.error('Error during provider connection:', error); + } + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual( + `azureBlobStorage-provider:${providerId}:refresh`, + ); + + await (taskDef.fn as () => Promise)(); + + const expectedEntities = blobs.map(blob => { + const url = encodeURI(`${normalizedExpectedBaseUrl}${blob}`); + return { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${url}`, + 'backstage.io/managed-by-origin-location': `url:${url}`, + }, + name: expect.stringMatching(/generated-[a-f0-9]{40}/), + }, + spec: { + presence: 'required', + target: `${url}`, + type: 'url', + }, + }, + locationKey: `azureBlobStorage-provider:${providerId}`, + }; + }); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: expectedEntities, + }); + }; + + it('apply full update on scheduled execution', async () => { + return expectMutation( + 'staticContainer', + { + containerName, + }, + 'https://myaccount.blob.core.windows.net/container-1/', + { + 'key1.yaml': 'generated-8ece85ad90200c6577b99f553dcbedde05fa34bb', + 'key2.yaml': 'generated-6b54c6aaa44696f5e91ce0f54fb27bf837549d11', + 'key3.yaml': 'generated-88c703cf1aa66913db4033b029adc0b174574646', + 'key4.yaml': 'generated-2b7e068bb4ec818c14f179a1e721843fc2dbc5f9', + }, + { + accountName: 'myaccount', + }, + ); + }); + + it('apply full update no prefix', async () => { + return expectMutation( + 'staticContainerNoPrefix', + { + containerName, + schedule: { + frequency: { minutes: 30 }, + timeout: { minutes: 3 }, + }, + }, + 'https://myaccount.blob.core.windows.net/container-1/', + { + 'key1.yaml': 'generated-8ece85ad90200c6577b99f553dcbedde05fa34bb', + 'key2.yaml': 'generated-6b54c6aaa44696f5e91ce0f54fb27bf837549d11', + 'key3.yaml': 'generated-88c703cf1aa66913db4033b029adc0b174574646', + 'key4.yaml': 'generated-2b7e068bb4ec818c14f179a1e721843fc2dbc5f9', + }, + { + host: 'blob.core.windows.net', + accountName: 'myaccount', + }, + ); + }); + + it('fail without schedule and scheduler', () => { + const config = new ConfigReader({ + catalog: { + providers: { + azureBlob: { + test: { + containerName: 'container-1', + prefix: 'sub/dir/', + accountName: 'myaccount', + }, + }, + }, + }, + }); + + expect(() => + AzureBlobStorageEntityProvider.fromConfig(config, { + logger, + }), + ).toThrow('Either schedule or scheduler must be provided'); + }); + + it('fail with scheduler but no schedule config', () => { + const scheduler = { + createScheduledTaskRunner: (_: any) => jest.fn(), + } as unknown as SchedulerService; + const config = new ConfigReader({ + catalog: { + providers: { + azureBlob: { + test: { + containerName: 'container-1', + prefix: 'sub/dir/', + accountName: 'myaccount', + }, + }, + }, + }, + }); + + expect(() => + AzureBlobStorageEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ).toThrow( + 'No schedule provided neither via code nor config for AzureBlobStorageEntityProvider:test.', + ); + }); +}); diff --git a/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.ts b/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.ts new file mode 100644 index 0000000000..47154f5c89 --- /dev/null +++ b/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.ts @@ -0,0 +1,228 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + AnonymousCredential, + BlobServiceClient, + ContainerClient, + StorageSharedKeyCredential, +} from '@azure/storage-blob'; +import { Config } from '@backstage/config'; +import { + LoggerService, + SchedulerService, + SchedulerServiceTaskRunner, +} from '@backstage/backend-plugin-api'; +import { + EntityProvider, + EntityProviderConnection, + locationSpecToLocationEntity, +} from '@backstage/plugin-catalog-node'; +import { LocationSpec } from '@backstage/plugin-catalog-common'; +import * as uuid from 'uuid'; +import { readAzureBlobStorageConfigs } from './config'; +import { + AzureBlobStorageIntergation, + DefaultAzureCredentialsManager, + ScmIntegrations, +} from '@backstage/integration'; +import { TokenCredential } from '@azure/identity'; +import { AzureBlobStorageConfig } from './types'; + +export class AzureBlobStorageEntityProvider implements EntityProvider { + private readonly logger: LoggerService; + private connection?: EntityProviderConnection; + private blobServiceClient?: BlobServiceClient; + private readonly scheduleFn: () => Promise; + + static fromConfig( + configRoot: Config, + options: { + logger: LoggerService; + schedule?: SchedulerServiceTaskRunner; + scheduler?: SchedulerService; + }, + ): AzureBlobStorageEntityProvider[] { + const providerConfigs = readAzureBlobStorageConfigs(configRoot); + + const scmIntegration = ScmIntegrations.fromConfig(configRoot); + const credentialsProvider = + DefaultAzureCredentialsManager.fromIntegrations(scmIntegration); + if (!options.schedule && !options.scheduler) { + throw new Error('Either schedule or scheduler must be provided.'); + } + + return providerConfigs.map(providerConfig => { + const integration = scmIntegration.azureBlobStorage.list()[0]; + if (!integration) { + throw new Error( + `There is no Azure blob storage integration for host. Please add a configuration entry for it under integrations.azure`, + ); + } + + if (!options.schedule && !providerConfig.schedule) { + throw new Error( + `No schedule provided neither via code nor config for AzureBlobStorageEntityProvider:${providerConfig.id}.`, + ); + } + + const taskRunner = + options.schedule ?? + options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!); + + return new AzureBlobStorageEntityProvider( + providerConfig, + integration, + credentialsProvider, + options.logger, + taskRunner, + ); + }); + } + constructor( + private readonly config: AzureBlobStorageConfig, + private readonly integration: AzureBlobStorageIntergation, + private readonly credentialsProvider: DefaultAzureCredentialsManager, + logger: LoggerService, + schedule: SchedulerServiceTaskRunner, + ) { + this.logger = logger.child({ target: this.getProviderName() }); + this.scheduleFn = this.createScheduleFn(schedule); + } + + private createScheduleFn( + taskRunner: SchedulerServiceTaskRunner, + ): () => Promise { + return async () => { + const taskId = `${this.getProviderName()}:refresh`; + return taskRunner.run({ + id: taskId, + fn: async () => { + const logger = this.logger.child({ + class: AzureBlobStorageEntityProvider.prototype.constructor.name, + taskId, + taskInstanceId: uuid.v4(), + }); + + try { + await this.refresh(logger); + } catch (error) { + logger.error( + `${this.getProviderName()} refresh failed, ${error}`, + error, + ); + } + }, + }); + }; + } + + getProviderName(): string { + return `azureBlobStorage-provider:${this.config.id}`; + } + + async connect(connection: EntityProviderConnection): Promise { + this.connection = connection; + let credential: + | TokenCredential + | StorageSharedKeyCredential + | AnonymousCredential; + if (this.integration.config.accountKey) { + credential = new StorageSharedKeyCredential( + this.integration.config.accountName as string, + this.integration.config.accountKey as string, + ); // StorageSharedKeyCredential is only allowed in node.js runtime not in browser + } else { + credential = await this.credentialsProvider.getCredentials( + this.integration.config.accountName as string, + ); + } + let blobServiceClientUrl: string; + + if (this.integration.config.endpoint) { + if (this.integration.config.sasToken) { + blobServiceClientUrl = `${this.integration.config.endpoint}?${this.integration.config.sasToken}`; + } else { + blobServiceClientUrl = `${this.integration.config.endpoint}`; + } + } else { + blobServiceClientUrl = `https://${this.integration.config.accountName}.${this.integration.config.host}`; + } + + this.blobServiceClient = new BlobServiceClient( + blobServiceClientUrl, + credential, + ); + await this.scheduleFn(); + } + + async refresh(logger: LoggerService) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + logger.info('Discovering Azure Blob Storage blobs'); + + const keys = await this.listAllBlobKeys(); + logger.info(`Discovered ${keys.length} Azure Blob Storage blobs`); + + const locations = keys.map(key => this.createLocationSpec(key)); + + await this.connection.applyMutation({ + type: 'full', + entities: locations.map(location => { + return { + locationKey: this.getProviderName(), + entity: locationSpecToLocationEntity({ location }), + }; + }), + }); + + logger.info( + `Committed ${locations.length} Locations for Azure Blob Storage blobs`, + ); + } + + private async listAllBlobKeys(): Promise { + const keys: string[] = []; + const containerClient = this.blobServiceClient?.getContainerClient( + this.config.containerName, + ); + + for await (const blob of ( + containerClient as ContainerClient + ).listBlobsFlat()) { + if (blob.name) { + keys.push(blob.name); + } + } + + return keys; + } + + private createLocationSpec(key: string): LocationSpec { + return { + type: 'url', + target: this.createObjectUrl(key), + presence: 'required', + }; + } + + private createObjectUrl(key: string): string { + const endpoint = this.blobServiceClient?.url; + return `${endpoint}${this.config.containerName}/${key}`; + } +} diff --git a/plugins/catalog-backend-module-azure/src/providers/config.test.ts b/plugins/catalog-backend-module-azure/src/providers/config.test.ts index 5d5ae930c1..21c4ece78c 100644 --- a/plugins/catalog-backend-module-azure/src/providers/config.test.ts +++ b/plugins/catalog-backend-module-azure/src/providers/config.test.ts @@ -15,7 +15,7 @@ */ import { ConfigReader } from '@backstage/config'; -import { readAzureDevOpsConfigs } from './config'; +import { readAzureBlobStorageConfigs, readAzureDevOpsConfigs } from './config'; describe('readAzureDevOpsConfigs', () => { it('reads all provider configs and set default values', () => { @@ -106,3 +106,69 @@ describe('readAzureDevOpsConfigs', () => { }); }); }); + +describe('readAzureBlobStorageConfigs', () => { + it('reads single and multiple Azure Blob Storage provider configs', () => { + const provider1 = { + containerName: 'container-1', + schedule: { + frequency: 'PT30M', + timeout: { + minutes: 3, + }, + }, + }; + const provider2 = { + containerName: 'container-2', + }; + + const configSingle = { + catalog: { + providers: { + azureBlob: provider2, + }, + }, + }; + + const configMulti = { + catalog: { + providers: { + azureBlob: { + provider1, + provider2, + }, + }, + }, + }; + + // Single provider case + const actualSingle = readAzureBlobStorageConfigs( + new ConfigReader(configSingle), + ); + expect(actualSingle).toHaveLength(1); + expect(actualSingle[0]).toEqual({ + id: 'default', + containerName: 'container-2', + schedule: undefined, // no schedule provided in this case + }); + + // Multiple providers case + const actualMulti = readAzureBlobStorageConfigs( + new ConfigReader(configMulti), + ); + expect(actualMulti).toHaveLength(2); + expect(actualMulti[0]).toEqual({ + id: 'provider1', + containerName: 'container-1', + schedule: { + ...provider1.schedule, + frequency: { minutes: 30 }, + }, + }); + expect(actualMulti[1]).toEqual({ + id: 'provider2', + containerName: 'container-2', + schedule: undefined, // no schedule provided + }); + }); +}); diff --git a/plugins/catalog-backend-module-azure/src/providers/config.ts b/plugins/catalog-backend-module-azure/src/providers/config.ts index 17616b06bc..d082726b14 100644 --- a/plugins/catalog-backend-module-azure/src/providers/config.ts +++ b/plugins/catalog-backend-module-azure/src/providers/config.ts @@ -16,7 +16,9 @@ import { readSchedulerServiceTaskScheduleDefinitionFromConfig } from '@backstage/backend-plugin-api'; import { Config } from '@backstage/config'; -import { AzureDevOpsConfig } from './types'; +import { AzureDevOpsConfig, AzureBlobStorageConfig } from './types'; + +const DEFAULT_PROVIDER_ID = 'default'; export function readAzureDevOpsConfigs(config: Config): AzureDevOpsConfig[] { const configs: AzureDevOpsConfig[] = []; @@ -61,3 +63,50 @@ function readAzureDevOpsConfig(id: string, config: Config): AzureDevOpsConfig { schedule, }; } + +export function readAzureBlobStorageConfigs( + config: Config, +): AzureBlobStorageConfig[] { + const configs: AzureBlobStorageConfig[] = []; + + const providerConfigs = config.getOptionalConfig( + 'catalog.providers.azureBlob', + ); + + if (!providerConfigs) { + return configs; + } + + if (providerConfigs.has('containerName')) { + // simple/single config variant + configs.push( + readAzureBlobStorageConfig(DEFAULT_PROVIDER_ID, providerConfigs), + ); + + return configs; + } + + for (const id of providerConfigs.keys()) { + configs.push(readAzureBlobStorageConfig(id, providerConfigs.getConfig(id))); + } + + return configs; +} + +function readAzureBlobStorageConfig( + id: string, + config: Config, +): AzureBlobStorageConfig { + const containerName = config.getString('containerName'); + const schedule = config.has('schedule') + ? readSchedulerServiceTaskScheduleDefinitionFromConfig( + config.getConfig('schedule'), + ) + : undefined; + + return { + id, + containerName, + schedule, + }; +} diff --git a/plugins/catalog-backend-module-azure/src/providers/index.ts b/plugins/catalog-backend-module-azure/src/providers/index.ts index 450fc0af27..5c9616e162 100644 --- a/plugins/catalog-backend-module-azure/src/providers/index.ts +++ b/plugins/catalog-backend-module-azure/src/providers/index.ts @@ -15,3 +15,4 @@ */ export { AzureDevOpsEntityProvider } from './AzureDevOpsEntityProvider'; +export { AzureBlobStorageEntityProvider } from './AzureBlobStorageEntityProvider'; diff --git a/plugins/catalog-backend-module-azure/src/providers/types.ts b/plugins/catalog-backend-module-azure/src/providers/types.ts index 5ca9dc94dc..eb16dbd6ea 100644 --- a/plugins/catalog-backend-module-azure/src/providers/types.ts +++ b/plugins/catalog-backend-module-azure/src/providers/types.ts @@ -26,3 +26,9 @@ export type AzureDevOpsConfig = { path: string; schedule?: SchedulerServiceTaskScheduleDefinition; }; + +export type AzureBlobStorageConfig = { + id: string; + containerName: string; + schedule?: SchedulerServiceTaskScheduleDefinition; +}; diff --git a/yarn.lock b/yarn.lock index af909aa8c1..a76d05d9db 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3605,6 +3605,8 @@ __metadata: "@aws-sdk/credential-providers": ^3.350.0 "@aws-sdk/types": ^3.347.0 "@aws-sdk/util-stream-node": ^3.350.0 + "@azure/identity": ^4.0.0 + "@azure/storage-blob": ^12.5.0 "@backstage/backend-app-api": "workspace:^" "@backstage/backend-common": ^0.25.0 "@backstage/backend-dev-utils": "workspace:^" @@ -4649,6 +4651,7 @@ __metadata: resolution: "@backstage/integration@workspace:packages/integration" dependencies: "@azure/identity": ^4.0.0 + "@azure/storage-blob": ^12.5.0 "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" "@backstage/config-loader": "workspace:^" @@ -5396,6 +5399,8 @@ __metadata: version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-backend-module-azure@workspace:plugins/catalog-backend-module-azure" dependencies: + "@azure/identity": ^4.0.0 + "@azure/storage-blob": ^12.5.0 "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-test-utils": "workspace:^" "@backstage/cli": "workspace:^"