From 277092a876943f96ce8d8ba9d83765919f87a11d Mon Sep 17 00:00:00 2001 From: NIKUNJ LALITKUMAR HUDKA Date: Fri, 18 Oct 2024 18:27:58 -0300 Subject: [PATCH 01/13] feat: azure blob storage entity provider for catalog implemented Signed-off-by: NIKUNJ LALITKUMAR HUDKA --- .changeset/cyan-rocks-divide.md | 5 + .changeset/fluffy-cobras-sort.md | 5 + .changeset/tall-comics-heal.md | 5 + app-config.yaml | 18 ++ packages/backend-defaults/package.json | 2 + .../lib/AzureBlobStorageUrlReader.ts | 256 +++++++++++++++++ .../entrypoints/urlReader/lib/UrlReaders.ts | 2 + .../src/entrypoints/urlReader/lib/index.ts | 1 + packages/integration/package.json | 1 + .../integration/src/ScmIntegrations.test.ts | 17 ++ packages/integration/src/ScmIntegrations.ts | 7 + .../AzureBlobStorageIntegration.test.ts | 78 ++++++ .../AzureBlobStorageIntegration.ts | 66 +++++ .../DefaultAzureCredentialsManager.test.ts | 151 ++++++++++ .../DefaultAzureCredentialsProvider.ts | 121 ++++++++ .../src/azureBlobStorage/config.test.ts | 264 ++++++++++++++++++ .../src/azureBlobStorage/config.ts | 170 +++++++++++ .../integration/src/azureBlobStorage/index.ts | 23 ++ .../integration/src/azureBlobStorage/types.ts | 29 ++ packages/integration/src/index.ts | 1 + packages/integration/src/registry.ts | 2 + .../catalog-backend-module-azure/package.json | 2 + .../catalog-backend-module-azure/src/index.ts | 1 + ...logModuleAzureDevOpsEntityProvider.test.ts | 59 +++- .../catalogModuleAzureDevOpsEntityProvider.ts | 34 ++- .../src/module/index.ts | 2 +- .../AzureBlobStorageEntityProvider.test.ts | 246 ++++++++++++++++ .../AzureBlobStorageEntityProvider.ts | 228 +++++++++++++++ .../src/providers/config.test.ts | 68 ++++- .../src/providers/config.ts | 51 +++- .../src/providers/index.ts | 1 + .../src/providers/types.ts | 6 + yarn.lock | 5 + 33 files changed, 1912 insertions(+), 15 deletions(-) create mode 100644 .changeset/cyan-rocks-divide.md create mode 100644 .changeset/fluffy-cobras-sort.md create mode 100644 .changeset/tall-comics-heal.md create mode 100644 packages/backend-defaults/src/entrypoints/urlReader/lib/AzureBlobStorageUrlReader.ts create mode 100644 packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.test.ts create mode 100644 packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.ts create mode 100644 packages/integration/src/azureBlobStorage/DefaultAzureCredentialsManager.test.ts create mode 100644 packages/integration/src/azureBlobStorage/DefaultAzureCredentialsProvider.ts create mode 100644 packages/integration/src/azureBlobStorage/config.test.ts create mode 100644 packages/integration/src/azureBlobStorage/config.ts create mode 100644 packages/integration/src/azureBlobStorage/index.ts create mode 100644 packages/integration/src/azureBlobStorage/types.ts create mode 100644 plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.test.ts create mode 100644 plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.ts diff --git a/.changeset/cyan-rocks-divide.md b/.changeset/cyan-rocks-divide.md new file mode 100644 index 0000000000..8fa76a493e --- /dev/null +++ b/.changeset/cyan-rocks-divide.md @@ -0,0 +1,5 @@ +--- +'@backstage/integration': major +--- + +Add the integration for Azure blob storage to read the credentials to access the storage account and provide the default credential provider. diff --git a/.changeset/fluffy-cobras-sort.md b/.changeset/fluffy-cobras-sort.md new file mode 100644 index 0000000000..363281a6c5 --- /dev/null +++ b/.changeset/fluffy-cobras-sort.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-defaults': minor +--- + +Implemented `AzureBlobStorageUrlReader` to read from the url of committed location from the entity provider diff --git a/.changeset/tall-comics-heal.md b/.changeset/tall-comics-heal.md new file mode 100644 index 0000000000..3d65caee67 --- /dev/null +++ b/.changeset/tall-comics-heal.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-azure': minor +--- + +Added the Azure Blob Storage as catalog entity provider to import all the desired entities from storage account provided in app-config.yaml diff --git a/app-config.yaml b/app-config.yaml index cb70170541..86737413fe 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -221,6 +221,17 @@ integrations: # apiBaseUrl: server.bitbucket.com # username: ${BITBUCKET_SERVER_USERNAME} # appPassword: ${BITBUCKET_SERVER_APP_PASSWORD} + + azureBlobStorage: + - accountName: ${ACCOUNT_NAME} # required + endpoint: ${CUSTOM_ENDPOINT} # custom endpoint will require either aadCredentials or sasToken + sasToken: ${SAS_TOKEN} + aadCredential: + clientId: ${CLIENT_ID} + tenantId: ${TENANT_ID} + clientSecret: ${CLIENT_SECRET} + accountKey: ${ACCOUNT_KEY} + azure: - host: dev.azure.com token: ${AZURE_TOKEN} @@ -245,6 +256,13 @@ catalog: - Domain - Location providers: + azureBlob: + containerName: ${CONTAINER_NAME} + schedule: # same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } backstageOpenapi: plugins: - catalog diff --git a/packages/backend-defaults/package.json b/packages/backend-defaults/package.json index 687d7379f1..754f16cad9 100644 --- a/packages/backend-defaults/package.json +++ b/packages/backend-defaults/package.json @@ -119,6 +119,8 @@ "@aws-sdk/client-s3": "^3.350.0", "@aws-sdk/credential-providers": "^3.350.0", "@aws-sdk/types": "^3.347.0", + "@azure/identity": "^4.0.0", + "@azure/storage-blob": "^12.5.0", "@backstage/backend-app-api": "workspace:^", "@backstage/backend-common": "^0.25.0", "@backstage/backend-dev-utils": "workspace:^", diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureBlobStorageUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureBlobStorageUrlReader.ts new file mode 100644 index 0000000000..f0c7e2570e --- /dev/null +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureBlobStorageUrlReader.ts @@ -0,0 +1,256 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + BlobServiceClient, + ContainerClient, + StorageSharedKeyCredential, +} from '@azure/storage-blob'; +import { ReaderFactory, ReadTreeResponseFactory } from './types'; +import { ForwardedError, NotModifiedError } from '@backstage/errors'; +import { Readable } from 'stream'; +import { relative } from 'path/posix'; +import { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; +import { + AzureBlobStorageIntergation, + AzureCredentialsManager, + DefaultAzureCredentialsManager, + ScmIntegrations, +} from '@backstage/integration'; +import { + UrlReaderService, + UrlReaderServiceReadTreeOptions, + UrlReaderServiceReadTreeResponse, + UrlReaderServiceReadUrlOptions, + UrlReaderServiceReadUrlResponse, + UrlReaderServiceSearchResponse, +} from '@backstage/backend-plugin-api'; + +export function parseUrl( + url: string, + config: AzureBlobStorageIntergation, +): { path: string; container: string } { + const parsedUrl = new URL(url); + const pathSegments = parsedUrl.pathname.split('/').filter(Boolean); + + if (pathSegments.length < 2) { + throw new Error(`Invalid Azure Blob Storage URL format: ${url}`); + } + + // First segment is the container name, rest is the blob path + const container = pathSegments[0]; + const path = pathSegments.slice(1).join('/'); + + return { path, container }; +} +export class AzureBlobStorageUrlReader implements UrlReaderService { + static factory: ReaderFactory = ({ config, treeResponseFactory }) => { + const integrations = ScmIntegrations.fromConfig(config); + + const credsManager = + DefaultAzureCredentialsManager.fromIntegrations(integrations); + + return integrations.azureBlobStorage.list().map(integrationConfig => { + const reader = new AzureBlobStorageUrlReader( + credsManager, + integrationConfig, + { + treeResponseFactory, + }, + ); + + const predicate = (url: URL) => + url.host.endsWith(integrationConfig.config.host); + return { reader, predicate }; + }); + }; + + // private readonly blobServiceClient: BlobServiceClient; + + constructor( + private readonly credsManager: AzureCredentialsManager, + private readonly integration: AzureBlobStorageIntergation, + private readonly deps: { + treeResponseFactory: ReadTreeResponseFactory; + }, + ) {} + + private async createContainerClient( + containerName: string, + ): Promise { + const accountName = this.integration.config.accountName; // Use the account name from the integration config + const accountKey = this.integration.config.accountKey; // Get the account key if it exists + + if (accountKey && accountName) { + const creds = new StorageSharedKeyCredential(accountName, accountKey); + const blobServiceClient = new BlobServiceClient( + `https://${accountName}.${this.integration.config.host}`, + creds, + ); + return blobServiceClient.getContainerClient(containerName); + } + // Use the credentials manager to get the correct credentials + const credential = await this.credsManager.getCredentials( + accountName as string, + ); + + let blobServiceClientUrl: string; + + if (this.integration.config.endpoint) { + if (this.integration.config.sasToken) { + blobServiceClientUrl = `${this.integration.config.endpoint}?${this.integration.config.sasToken}`; + } else { + blobServiceClientUrl = `${this.integration.config.endpoint}`; + } + } else { + blobServiceClientUrl = `https://${this.integration.config.accountName}.${this.integration.config.host}`; + } + + const blobServiceClient = new BlobServiceClient( + blobServiceClientUrl, + credential, + ); + return blobServiceClient.getContainerClient(containerName); + } + + async read(url: string): Promise { + const response = await this.readUrl(url); + return response.buffer(); + } + + async readUrl( + url: string, + options?: UrlReaderServiceReadUrlOptions, + ): Promise { + const { etag, lastModifiedAfter } = options ?? {}; + + try { + const { path, container } = parseUrl(url, this.integration); + + const containerClient = await this.createContainerClient(container); + const blobClient = containerClient.getBlobClient(path); + + const abortController = new AbortController(); + + const getBlobOptions = { + conditions: { + ...(etag && { ifNoneMatch: etag }), + ...(lastModifiedAfter && { ifModifiedSince: lastModifiedAfter }), + }, + }; + options?.signal?.addEventListener('abort', () => abortController.abort()); + + const downloadBlockBlobResponse = await blobClient.download( + 0, + undefined, + getBlobOptions, + ); + + const data = await this.retrieveAzureBlobData( + downloadBlockBlobResponse.readableStreamBody as Readable, + ); + + return ReadUrlResponseFactory.fromReadable(data, { + etag: downloadBlockBlobResponse.etag, + lastModifiedAt: downloadBlockBlobResponse.lastModified, + }); + } catch (e) { + if (e.$metadata && e.$metadata.httpStatusCode === 304) { + throw new NotModifiedError(); + } + + throw new ForwardedError( + 'Could not retrieve file from Azure Blob Storage', + e, + ); + } + } + + async readTree( + url: string, + options?: UrlReaderServiceReadTreeOptions, + ): Promise { + try { + const { path, container } = parseUrl(url, this.integration); + + const containerClient = await this.createContainerClient(container); + + const blobs = containerClient.listBlobsFlat({ prefix: path }); + + const responses = []; + const abortController = new AbortController(); + for await (const blob of blobs) { + const blobClient = containerClient.getBlobClient(blob.name); + options?.signal?.addEventListener('abort', () => + abortController.abort(), + ); + const downloadBlockBlobResponse = await blobClient.download(); + const data = await this.retrieveAzureBlobData( + downloadBlockBlobResponse.readableStreamBody as Readable, + ); + + responses.push({ + data: Readable.from(data), + path: relative(path, blob.name), + lastModifiedAt: blob.properties.lastModified, + }); + } + + return this.deps.treeResponseFactory.fromReadableArray(responses); + } catch (e) { + throw new ForwardedError( + 'Could not retrieve file tree from Azure Blob Storage', + e, + ); + } + } + + async search(): Promise { + throw new Error('AzureBlobStorageUrlReader does not implement search'); + } + + toString() { + const accountName = this.integration.config.accountName; + const accountKey = this.integration.config.accountKey; + return `azureBlobStorage{accountName=${accountName},authed=${Boolean( + accountKey, + )}}`; + } + + private parseUrl(url: string): { path: string } { + const parsedUrl = new URL(url); + const path = parsedUrl.pathname.substring( + parsedUrl.pathname.lastIndexOf('/') + 1, + ); + + return { path }; + } + + private async retrieveAzureBlobData(stream: Readable): Promise { + return new Promise((resolve, reject) => { + try { + const chunks: any[] = []; + stream.on('data', chunk => chunks.push(chunk)); + stream.on('error', (e: Error) => + reject(new ForwardedError('Unable to read stream', e)), + ); + stream.on('end', () => resolve(Readable.from(Buffer.concat(chunks)))); + } catch (e) { + throw new ForwardedError('Unable to parse the response data', e); + } + }); + } +} diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/UrlReaders.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/UrlReaders.ts index 96e9ed6509..229bdd4903 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/UrlReaders.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/UrlReaders.ts @@ -35,6 +35,7 @@ import { AwsS3UrlReader } from './AwsS3UrlReader'; import { GiteaUrlReader } from './GiteaUrlReader'; import { AwsCodeCommitUrlReader } from './AwsCodeCommitUrlReader'; import { HarnessUrlReader } from './HarnessUrlReader'; +import { AzureBlobStorageUrlReader } from './AzureBlobStorageUrlReader'; /** * Creation options for {@link @backstage/backend-plugin-api#UrlReaderService}. @@ -99,6 +100,7 @@ export class UrlReaders { GoogleGcsUrlReader.factory, HarnessUrlReader.factory, AwsS3UrlReader.factory, + AzureBlobStorageUrlReader.factory, AwsCodeCommitUrlReader.factory, FetchUrlReader.factory, ]), diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/index.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/index.ts index 678c6ef64b..fc33efbc3a 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/index.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/index.ts @@ -24,6 +24,7 @@ export { GitlabUrlReader } from './GitlabUrlReader'; export { GiteaUrlReader } from './GiteaUrlReader'; export { HarnessUrlReader } from './HarnessUrlReader'; export { AwsS3UrlReader } from './AwsS3UrlReader'; +export { AzureBlobStorageUrlReader } from './AzureBlobStorageUrlReader'; export { FetchUrlReader } from './FetchUrlReader'; export { ReadUrlResponseFactory } from './ReadUrlResponseFactory'; export type { diff --git a/packages/integration/package.json b/packages/integration/package.json index d41ffa501d..4039e94c34 100644 --- a/packages/integration/package.json +++ b/packages/integration/package.json @@ -38,6 +38,7 @@ }, "dependencies": { "@azure/identity": "^4.0.0", + "@azure/storage-blob": "^12.5.0", "@backstage/config": "workspace:^", "@backstage/errors": "workspace:^", "@octokit/auth-app": "^4.0.0", diff --git a/packages/integration/src/ScmIntegrations.test.ts b/packages/integration/src/ScmIntegrations.test.ts index c758816c03..0e0c181861 100644 --- a/packages/integration/src/ScmIntegrations.test.ts +++ b/packages/integration/src/ScmIntegrations.test.ts @@ -39,6 +39,10 @@ import { GiteaIntegration, GiteaIntegrationConfig } from './gitea'; import { AwsCodeCommitIntegration } from './awsCodeCommit/AwsCodeCommitIntegration'; import { AwsCodeCommitIntegrationConfig } from './awsCodeCommit'; import { HarnessIntegration, HarnessIntegrationConfig } from './harness'; +import { + AzureBlobStorageIntegrationConfig, + AzureBlobStorageIntergation, +} from './azureBlobStorage'; describe('ScmIntegrations', () => { const awsS3 = new AwsS3Integration({ @@ -53,6 +57,10 @@ describe('ScmIntegrations', () => { host: 'azure.local', } as AzureIntegrationConfig); + const azureBlob = new AzureBlobStorageIntergation({ + host: 'azureblobstorage.local', + } as AzureBlobStorageIntegrationConfig); + const bitbucket = new BitbucketIntegration({ host: 'bitbucket.local', } as BitbucketIntegrationConfig); @@ -89,6 +97,7 @@ describe('ScmIntegrations', () => { awsS3: basicIntegrations([awsS3], item => item.config.host), awsCodeCommit: basicIntegrations([awsCodeCommit], item => item.config.host), azure: basicIntegrations([azure], item => item.config.host), + azureBlobStorage: basicIntegrations([azureBlob], item => item.config.host), bitbucket: basicIntegrations([bitbucket], item => item.config.host), bitbucketCloud: basicIntegrations([bitbucketCloud], item => item.title), bitbucketServer: basicIntegrations( @@ -108,6 +117,9 @@ describe('ScmIntegrations', () => { awsCodeCommit, ); expect(i.azure.byUrl('https://azure.local')).toBe(azure); + expect(i.azureBlobStorage.byUrl('https://azureblobstorage.local')).toBe( + azureBlob, + ); expect(i.bitbucket.byUrl('https://bitbucket.local')).toBe(bitbucket); expect(i.bitbucketCloud.byUrl('https://bitbucket.org')).toBe( bitbucketCloud, @@ -128,6 +140,7 @@ describe('ScmIntegrations', () => { awsS3, awsCodeCommit, azure, + azureBlob, bitbucket, bitbucketCloud, bitbucketServer, @@ -144,6 +157,9 @@ describe('ScmIntegrations', () => { expect(i.byUrl('https://awss3.local')).toBe(awsS3); expect(i.byUrl('https://awscodecommit.local')).toBe(awsCodeCommit); expect(i.byUrl('https://azure.local')).toBe(azure); + expect(i.azureBlobStorage.byUrl('https://azureblobstorage.local')).toBe( + azureBlob, + ); expect(i.byUrl('https://bitbucket.local')).toBe(bitbucket); expect(i.byUrl('https://bitbucket.org')).toBe(bitbucketCloud); expect(i.byUrl('https://bitbucket-server.local')).toBe(bitbucketServer); @@ -156,6 +172,7 @@ describe('ScmIntegrations', () => { expect(i.byHost('awss3.local')).toBe(awsS3); expect(i.byHost('awscodecommit.local')).toBe(awsCodeCommit); expect(i.byHost('azure.local')).toBe(azure); + expect(i.byHost('azureblobstorage.local')).toBe(azureBlob); expect(i.byHost('bitbucket.local')).toBe(bitbucket); expect(i.byHost('bitbucket.org')).toBe(bitbucketCloud); expect(i.byHost('bitbucket-server.local')).toBe(bitbucketServer); diff --git a/packages/integration/src/ScmIntegrations.ts b/packages/integration/src/ScmIntegrations.ts index e0d430d834..f11cb3c17e 100644 --- a/packages/integration/src/ScmIntegrations.ts +++ b/packages/integration/src/ScmIntegrations.ts @@ -29,6 +29,7 @@ import { ScmIntegration, ScmIntegrationsGroup } from './types'; import { ScmIntegrationRegistry } from './registry'; import { GiteaIntegration } from './gitea'; import { HarnessIntegration } from './harness/HarnessIntegration'; +import { AzureBlobStorageIntergation } from './azureBlobStorage'; /** * The set of supported integrations. @@ -38,6 +39,7 @@ import { HarnessIntegration } from './harness/HarnessIntegration'; export interface IntegrationsByType { awsS3: ScmIntegrationsGroup; awsCodeCommit: ScmIntegrationsGroup; + azureBlobStorage: ScmIntegrationsGroup; azure: ScmIntegrationsGroup; /** * @deprecated in favor of `bitbucketCloud` and `bitbucketServer` @@ -64,6 +66,7 @@ export class ScmIntegrations implements ScmIntegrationRegistry { return new ScmIntegrations({ awsS3: AwsS3Integration.factory({ config }), awsCodeCommit: AwsCodeCommitIntegration.factory({ config }), + azureBlobStorage: AzureBlobStorageIntergation.factory({ config }), azure: AzureIntegration.factory({ config }), bitbucket: BitbucketIntegration.factory({ config }), bitbucketCloud: BitbucketCloudIntegration.factory({ config }), @@ -88,6 +91,10 @@ export class ScmIntegrations implements ScmIntegrationRegistry { return this.byType.awsCodeCommit; } + get azureBlobStorage(): ScmIntegrationsGroup { + return this.byType.azureBlobStorage; + } + get azure(): ScmIntegrationsGroup { return this.byType.azure; } diff --git a/packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.test.ts b/packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.test.ts new file mode 100644 index 0000000000..e7f7703be9 --- /dev/null +++ b/packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.test.ts @@ -0,0 +1,78 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { AzureBlobStorageIntergation } from './AzureBlobStorageIntegration'; + +describe('AzureBlobStorageIntegration', () => { + it('has a working factory', () => { + const integrations = AzureBlobStorageIntergation.factory({ + config: new ConfigReader({ + integrations: { + azureBlobStorage: [ + { + endpoint: 'https://myaccount.blob.core.windows.net', + accountName: 'myaccount', + accountKey: 'someAccountKey', + }, + ], + }, + }), + }); + expect(integrations.list().length).toBe(2); // including default + expect(integrations.list()[0].config.host).toBe( + 'myaccount.blob.core.windows.net', + ); + expect(integrations.list()[1].config.host).toBe('blob.core.windows.net'); // default integration + }); + + it('returns the basics', () => { + const integration = new AzureBlobStorageIntergation({ + host: 'myaccount.blob.core.windows.net', + } as any); + expect(integration.type).toBe('azureBlobStorage'); + expect(integration.title).toBe('myaccount.blob.core.windows.net'); + }); + + describe('resolveUrl', () => { + it('works for valid URLs', () => { + const integration = new AzureBlobStorageIntergation({ + host: 'blob.core.windows.net', + } as any); + + expect( + integration.resolveUrl({ + url: 'https://myaccount.blob.core.windows.net/container/file.yaml', + base: 'https://myaccount.blob.core.windows.net/container/file.yaml', + }), + ).toBe('https://myaccount.blob.core.windows.net/container/file.yaml'); + }); + }); + + it('resolve edit URL', () => { + const integration = new AzureBlobStorageIntergation({ + host: 'myaccount.blob.core.windows.net', + } as any); + + // TODO: The Azure Blob Storage integration doesn't support resolving an edit URL, + // instead we keep the input URL. + expect( + integration.resolveEditUrl( + 'https://myaccount.blob.core.windows.net/container/file.yaml', + ), + ).toBe('https://myaccount.blob.core.windows.net/container/file.yaml'); + }); +}); diff --git a/packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.ts b/packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.ts new file mode 100644 index 0000000000..01e6bbc317 --- /dev/null +++ b/packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.ts @@ -0,0 +1,66 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { basicIntegrations, defaultScmResolveUrl } from '../helpers'; +import { ScmIntegration, ScmIntegrationsFactory } from '../types'; +import { + AzureBlobStorageIntegrationConfig, + readAzureBlobStorageIntegrationConfigs, +} from './config'; + +export class AzureBlobStorageIntergation implements ScmIntegration { + static factory: ScmIntegrationsFactory = ({ + config, + }) => { + const configs = readAzureBlobStorageIntegrationConfigs( + config.getOptionalConfigArray('integrations.azureBlobStorage') ?? [], + ); + return basicIntegrations( + configs.map(c => new AzureBlobStorageIntergation(c)), + i => i.config.host, + ); + }; + + get type(): string { + return 'azureBlobStorage'; + } + + get title(): string { + return this.integrationConfig.host; + } + + get config(): AzureBlobStorageIntegrationConfig { + return this.integrationConfig; + } + + constructor( + private readonly integrationConfig: AzureBlobStorageIntegrationConfig, + ) {} + + resolveUrl(options: { + url: string; + base: string; + lineNumber?: number | undefined; + }): string { + const resolved = defaultScmResolveUrl(options); + return resolved; + } + + resolveEditUrl(url: string): string { + // TODO: Implement edit URL for azureBlobStorage + return url; + } +} diff --git a/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsManager.test.ts b/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsManager.test.ts new file mode 100644 index 0000000000..6272a68d13 --- /dev/null +++ b/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsManager.test.ts @@ -0,0 +1,151 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + AccessToken, + ClientSecretCredential, + DefaultAzureCredential, + TokenCredential, +} from '@azure/identity'; +import { AzureBlobStorageIntegrationConfig } from './config'; +import { ScmIntegrationRegistry } from '../registry'; +import { ConfigReader } from '@backstage/config'; +import { DefaultAzureCredentialsManager } from './DefaultAzureCredentialsProvider'; +import { ScmIntegrations } from '../ScmIntegrations'; +import { DateTime } from 'luxon'; + +const MockedClientSecretCredential = ClientSecretCredential as jest.MockedClass< + typeof ClientSecretCredential +>; + +jest.mock('@azure/identity'); + +describe('DefaultAzureCredentialsManager', () => { + let mockIntegration: ScmIntegrationRegistry; + + const buildProvider = (azureIntegrations: any[]) => + DefaultAzureCredentialsManager.fromIntegrations( + ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + azureBlobStorage: azureIntegrations, + }, + }), + ), + ); + + beforeEach(() => { + mockIntegration = { + azureBlobStorage: { + list: jest.fn().mockReturnValue([ + { + config: { + accountName: 'testaccount', + aadCredential: { + clientId: 'someClientId', + tenantId: 'someTenantId', + clientSecret: 'someClientSecret', + }, + }, + }, + ]), + }, + } as unknown as ScmIntegrationRegistry; + + MockedClientSecretCredential.prototype.getToken.mockImplementation(() => + Promise.resolve({ + expiresOnTimestamp: DateTime.local().plus({ days: 1 }).toSeconds(), + token: 'fake-client-secret-token', + } as AccessToken), + ); + }); + + it('should create an instance from ScmIntegrationRegistry', () => { + const manager = + DefaultAzureCredentialsManager.fromIntegrations(mockIntegration); + expect(manager).toBeInstanceOf(DefaultAzureCredentialsManager); + }); + + it('should return cached credentials if available', async () => { + const manager = buildProvider([ + { + accountName: 'testaccount', + aadCredential: { + clientId: 'someClientId', + tenantId: 'someTenantId', + clientSecret: 'someClientSecret', + }, + }, + ]); + + const mockCredential = new MockedClientSecretCredential( + 'someTenantId', + 'someClientId', + 'someClientSecret', + ); + + const credential = await manager.getCredentials('testaccount'); + + const scopes = ['https://storage.azure.com/.default']; + + const expectedToken = await mockCredential.getToken(scopes); + const receivedToken = await credential.getToken(scopes); + + expect(receivedToken?.token).toEqual(expectedToken.token); + }); + + it('should use Azure AD credentials if aadCredential is provided', async () => { + const manager = buildProvider([ + { + accountName: 'testaccount', + aadCredential: { + clientId: 'someClientId', + tenantId: 'someTenantId', + clientSecret: 'someClientSecret', + }, + }, + ]); + + const credential = await manager.getCredentials('testaccount'); + + expect(credential).toBeInstanceOf(ClientSecretCredential); + }); + + it('should use DefaultAzureCredential if no aadCredential is provided', async () => { + const manager = buildProvider([ + { + accountName: 'testaccount', + }, + ]); + + const credential = await manager.getCredentials('testaccount'); + + expect(credential).toBeInstanceOf(DefaultAzureCredential); + }); + + it('should cache credentials after first retrieval', async () => { + const manager = buildProvider([ + { + accountName: 'testaccount', + }, + ]); + + const credential = await manager.getCredentials('testaccount'); + + const cachedCredential = await manager.getCredentials('testaccount'); + expect(cachedCredential).toBe(credential); + }); +}); diff --git a/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsProvider.ts b/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsProvider.ts new file mode 100644 index 0000000000..9f209da61c --- /dev/null +++ b/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsProvider.ts @@ -0,0 +1,121 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + DefaultAzureCredential, + ClientSecretCredential, + TokenCredential, +} from '@azure/identity'; +import { AzureBlobStorageIntegrationConfig } from './config'; +import { AzureCredentialsManager } from './types'; +import { ScmIntegrationRegistry } from '../registry'; + +/** + * Default Azure Credentials Manager to dynamically select and manage Azure credentials. + * It supports Service Principal, Managed Identity, SAS Token, Connection String, Account Key, and Anonymous access. + */ +export class DefaultAzureCredentialsManager implements AzureCredentialsManager { + private config: AzureBlobStorageIntegrationConfig; + private cachedCredentials: Map; + + constructor(config: AzureBlobStorageIntegrationConfig) { + this.config = config; + this.cachedCredentials = new Map(); + } + + /** + * Creates an instance of DefaultAzureCredentialsManager from a Backstage Config. + */ + static fromIntegrations( + integration: ScmIntegrationRegistry, + ): DefaultAzureCredentialsManager { + const azureConfig = integration.azureBlobStorage.list().length + ? integration.azureBlobStorage.list()[0].config + : { host: 'blob.core.windows.net' }; // Default to Azure Blob Storage host if no config found + + return new DefaultAzureCredentialsManager(azureConfig); + } + + /** + * Determines the appropriate credential method and returns credentials for BlobServiceClient. + * Supports: + * - Service Principal + * - Managed Identity + * - SAS Token + * - Connection String + * - Account Key + * - Anonymous access + */ + async getCredentials(accountName: string): Promise { + // Check if the credentials are already cached + if (this.cachedCredentials.has(accountName)) { + return this.cachedCredentials.get(accountName)!; + } + + let credential: TokenCredential; + + // Check for SAS Token + // if (this.config.sasToken) { + // // console.log('Using SAS Token for Azure Blob Storage authentication'); + // // SAS Token does not return a credential but can be used directly in BlobServiceClient + // // Here we can simply return undefined or keep a placeholder if needed + // return this.config.sasToken; // Or return a string for the URL using the SAS token + // } + // // Check for Connection String + // else if (this.config.connectionString) { + // // console.log( + // // 'Using Connection String for Azure Blob Storage authentication', + // // ); + // // return undefined; // Connection string will also not return a specific credential object + // } + // Check for Account Key + // if (this.config.accountKey) { + // // console.log('Using Account Key for Azure Blob Storage authentication'); + // credential = new StorageSharedKeyCredential( + // accountName, + // this.config.accountKey, + // ); + // } + // Check for AAD credentials + + if ( + this.config.aadCredential && + this.config.aadCredential.clientId && + this.config.aadCredential.clientSecret && + this.config.aadCredential.tenantId + ) { + credential = new ClientSecretCredential( + this.config.aadCredential.tenantId, + this.config.aadCredential.clientId, + this.config.aadCredential.clientSecret, + ); + } + // Check for Anonymous access + // else if (this.config.anonymousAccess) { + // console.log('Using Anonymous Credential for Azure Blob Storage access'); + // credential = new AnonymousCredential(); + // } + // Fallback to Managed Identity + else { + credential = new DefaultAzureCredential(); + } + + // Cache the credentials for future use + this.cachedCredentials.set(accountName, credential); + + return credential; + } +} diff --git a/packages/integration/src/azureBlobStorage/config.test.ts b/packages/integration/src/azureBlobStorage/config.test.ts new file mode 100644 index 0000000000..f17bdf3efb --- /dev/null +++ b/packages/integration/src/azureBlobStorage/config.test.ts @@ -0,0 +1,264 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Config, ConfigReader } from '@backstage/config'; +import { + AzureBlobStorageIntegrationConfig, + readAzureBlobStorageIntegrationConfig, + readAzureBlobStorageIntegrationConfigs, +} from './config'; + +describe('readAzureBlobStorageIntegrationConfig', () => { + function buildConfig( + data: Partial, + ): Config { + return new ConfigReader(data); + } + + it('reads valid configuration with accountKey', () => { + const output = readAzureBlobStorageIntegrationConfig( + buildConfig({ + accountName: 'mystorageaccount', + accountKey: 'someAccountKey', + }), + ); + expect(output).toEqual({ + host: 'blob.core.windows.net', + endpoint: undefined, + accountName: 'mystorageaccount', + accountKey: 'someAccountKey', + sasToken: undefined, + connectionString: undefined, + endpointSuffix: undefined, + aadCredential: undefined, + }); + }); + + it('reads valid configuration with sasToken', () => { + const output = readAzureBlobStorageIntegrationConfig( + buildConfig({ + endpoint: 'https://blob.core.windows.net', + accountName: 'mystorageaccount', + sasToken: 'someSASToken', + }), + ); + expect(output).toEqual({ + host: 'blob.core.windows.net', + endpoint: 'https://blob.core.windows.net', + accountName: 'mystorageaccount', + accountKey: undefined, + sasToken: 'someSASToken', + connectionString: undefined, + endpointSuffix: undefined, + aadCredential: undefined, + }); + }); + + it('reads valid configuration with Azure AD credentials', () => { + const output = readAzureBlobStorageIntegrationConfig( + buildConfig({ + accountName: 'mystorageaccount', + aadCredential: { + clientId: 'someClientId', + tenantId: 'someTenantId', + clientSecret: 'someClientSecret', + }, + }), + ); + expect(output).toEqual({ + host: 'blob.core.windows.net', + endpoint: undefined, + accountName: 'mystorageaccount', + accountKey: undefined, + sasToken: undefined, + connectionString: undefined, + endpointSuffix: undefined, + aadCredential: { + clientId: 'someClientId', + tenantId: 'someTenantId', + clientSecret: 'someClientSecret', + }, + }); + }); + + it('reads valid configuration with a custom endpoint', () => { + const output = readAzureBlobStorageIntegrationConfig( + buildConfig({ + endpoint: 'https://custom.blob.core.windows.net', + accountName: 'customaccount', + }), + ); + expect(output).toEqual({ + host: 'custom.blob.core.windows.net', + endpoint: 'https://custom.blob.core.windows.net', + accountName: 'customaccount', + accountKey: undefined, + sasToken: undefined, + connectionString: undefined, + endpointSuffix: undefined, + aadCredential: undefined, + }); + }); + + it('throws error for invalid endpoint URL', () => { + const config = buildConfig({ + endpoint: 'invalid-url', + accountName: 'invalidaccount', + }); + + expect(() => readAzureBlobStorageIntegrationConfig(config)).toThrow( + `invalid azureBlobStorage integration config, endpoint 'invalid-url' is not a valid URL`, + ); + }); + + it('throws error if endpoint has a path', () => { + const config = buildConfig({ + endpoint: 'https://blob.core.windows.net/path', + accountName: 'accountwithpath', + }); + + expect(() => readAzureBlobStorageIntegrationConfig(config)).toThrow( + `invalid azureBlobStorage integration config, endpoints cannot contain path, got 'https://blob.core.windows.net/path'`, + ); + }); + + it('throws error if both accountKey and sasToken are provided', () => { + const config = buildConfig({ + accountName: 'mystorageaccount', + accountKey: 'someAccountKey', + sasToken: 'someSASToken', + }); + + expect(() => readAzureBlobStorageIntegrationConfig(config)).toThrow( + `Invalid Azure Blob Storage config for mystorageaccount: Both account key and SAS token cannot be used simultaneously.`, + ); + }); + + it('throws error if both aadCredential and accountKey/sasToken are provided', () => { + const config = buildConfig({ + accountName: 'mystorageaccount', + accountKey: 'someAccountKey', + aadCredential: { + clientId: 'someClientId', + tenantId: 'someTenantId', + clientSecret: 'someClientSecret', + }, + }); + + expect(() => readAzureBlobStorageIntegrationConfig(config)).toThrow( + `Invalid Azure Blob Storage config for mystorageaccount: Cannot use both Azure AD credentials and account keys/SAS tokens for the same account.`, + ); + }); +}); + +describe('readAzureBlobStorageIntegrationConfigs', () => { + function buildConfigs( + data: Partial[], + ): Config[] { + return data.map(item => new ConfigReader(item)); + } + + it('reads all provided configurations', () => { + const output = readAzureBlobStorageIntegrationConfigs( + buildConfigs([ + { + host: 'blob.core.windows.net', + accountName: 'account1', + accountKey: 'someAccountKey', + }, + { + endpoint: 'https://custom.blob.core.windows.net', + accountName: 'account2', + }, + ]), + ); + expect(output).toEqual([ + { + host: 'blob.core.windows.net', + endpoint: undefined, + accountName: 'account1', + accountKey: 'someAccountKey', + sasToken: undefined, + connectionString: undefined, + endpointSuffix: undefined, + aadCredential: undefined, + }, + { + host: 'custom.blob.core.windows.net', + endpoint: 'https://custom.blob.core.windows.net', + accountName: 'account2', + accountKey: undefined, + sasToken: undefined, + connectionString: undefined, + endpointSuffix: undefined, + aadCredential: undefined, + }, + ]); + }); + + it('adds default integration for blob.core.windows.net when missing', () => { + const output = readAzureBlobStorageIntegrationConfigs( + buildConfigs([ + { + endpoint: 'https://custom.blob.core.windows.net', + accountName: 'account2', + }, + ]), + ); + + expect(output).toEqual([ + { + host: 'custom.blob.core.windows.net', + endpoint: 'https://custom.blob.core.windows.net', + accountName: 'account2', + accountKey: undefined, + sasToken: undefined, + connectionString: undefined, + endpointSuffix: undefined, + aadCredential: undefined, + }, + { + host: 'blob.core.windows.net', + endpoint: undefined, + accountName: undefined, + accountKey: undefined, + sasToken: undefined, + connectionString: undefined, + endpointSuffix: undefined, + aadCredential: undefined, + }, + ]); + }); + + it('does not add default integration if blob.core.windows.net already exists', () => { + const output = readAzureBlobStorageIntegrationConfigs( + buildConfigs([ + { host: 'blob.core.windows.net', accountName: 'account1' }, + ]), + ); + expect(output).toEqual([ + { + host: 'blob.core.windows.net', + endpoint: undefined, + accountName: 'account1', + accountKey: undefined, + sasToken: undefined, + connectionString: undefined, + endpointSuffix: undefined, + aadCredential: undefined, + }, + ]); + }); +}); diff --git a/packages/integration/src/azureBlobStorage/config.ts b/packages/integration/src/azureBlobStorage/config.ts new file mode 100644 index 0000000000..10eccc252c --- /dev/null +++ b/packages/integration/src/azureBlobStorage/config.ts @@ -0,0 +1,170 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Config } from '@backstage/config'; + +const AZURE_HOST = 'blob.core.windows.net'; + +/** + * The configuration parameters for a single Azure Blob Storage account. + * + * @public + */ +export type AzureBlobStorageIntegrationConfig = { + /** + * The name of the Azure Storage Account, e.g., "mystorageaccount". + */ + accountName?: string; + + /** + * The primary or secondary key for the Azure Storage Account. + * Only required if connectionString or SAS token are not specified. + */ + accountKey?: string; + + /** + * A Shared Access Signature (SAS) token for limited access to resources. + */ + sasToken?: string; + + /** + * A full connection string for the Azure Storage Account. + * This includes the account name, key, and endpoint details. + */ + connectionString?: string; + + /** + * Optional endpoint suffix for custom domains or sovereign clouds. + * e.g., "core.windows.net" for public Azure or "core.usgovcloudapi.net" for US Government cloud. + */ + endpointSuffix?: string; + + /** + * The host of the target that this matches on, e.g., "blob.core.windows.net". + */ + host: string; + + endpoint?: string; + /** + * Optional credential to use for Azure Active Directory authentication. + */ + aadCredential?: { + /** + * The client ID of the Azure AD application. + */ + clientId: string; + + /** + * The tenant ID for Azure AD. + */ + tenantId: string; + + /** + * The client secret for the Azure AD application. + */ + clientSecret: string; + }; +}; + +/** + * Reads a single Azure Blob Storage integration config. + * + * @param config - The config object of a single integration. + * @public + */ +export function readAzureBlobStorageIntegrationConfig( + config: Config, +): AzureBlobStorageIntegrationConfig { + const endpoint = config.getOptionalString('endpoint'); + const accountName = config.getString('accountName'); + const accountKey = config.getOptionalString('accountKey')?.trim(); + const sasToken = config.getOptionalString('sasToken')?.trim(); + const connectionString = config.getOptionalString('connectionString')?.trim(); + const endpointSuffix = config.getOptionalString('endpointSuffix')?.trim(); + + let host; + let pathname; + if (endpoint) { + try { + const url = new URL(endpoint); + host = url.host; + pathname = url.pathname; + } catch { + throw new Error( + `invalid azureBlobStorage integration config, endpoint '${endpoint}' is not a valid URL`, + ); + } + if (pathname !== '/') { + throw new Error( + `invalid azureBlobStorage integration config, endpoints cannot contain path, got '${endpoint}'`, + ); + } + } else { + host = AZURE_HOST; + } + const aadCredential = config.has('aadCredential') + ? { + clientId: config.getString('aadCredential.clientId'), + tenantId: config.getString('aadCredential.tenantId'), + clientSecret: config.getString('aadCredential.clientSecret')?.trim(), + } + : undefined; + + if (accountKey && sasToken) { + throw new Error( + `Invalid Azure Blob Storage config for ${accountName}: Both account key and SAS token cannot be used simultaneously.`, + ); + } + + if (aadCredential && (accountKey || sasToken)) { + throw new Error( + `Invalid Azure Blob Storage config for ${accountName}: Cannot use both Azure AD credentials and account keys/SAS tokens for the same account.`, + ); + } + + return { + host, + endpoint, + accountName, + accountKey, + sasToken, + connectionString, + endpointSuffix, + aadCredential, + }; +} + +/** + * Reads a set of Azure Blob Storage integration configs. + * + * @param configs - All of the integration config objects. + * @public + */ +export function readAzureBlobStorageIntegrationConfigs( + configs: Config[], +): AzureBlobStorageIntegrationConfig[] { + // First read all the explicit integrations + const result = configs.map(readAzureBlobStorageIntegrationConfig); + + // If no explicit blob.core.windows.net integration was added, put one in the list as + // a convenience + if (!result.some(c => c.host === AZURE_HOST)) { + result.push({ + host: AZURE_HOST, + }); + } + return result; +} diff --git a/packages/integration/src/azureBlobStorage/index.ts b/packages/integration/src/azureBlobStorage/index.ts new file mode 100644 index 0000000000..f55e9aaa79 --- /dev/null +++ b/packages/integration/src/azureBlobStorage/index.ts @@ -0,0 +1,23 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export { AzureBlobStorageIntergation } from './AzureBlobStorageIntegration'; +export { + readAzureBlobStorageIntegrationConfig, + readAzureBlobStorageIntegrationConfigs, +} from './config'; +export type { AzureBlobStorageIntegrationConfig } from './config'; +export { DefaultAzureCredentialsManager } from './DefaultAzureCredentialsProvider'; +export type { AzureCredentialsManager } from './types'; diff --git a/packages/integration/src/azureBlobStorage/types.ts b/packages/integration/src/azureBlobStorage/types.ts new file mode 100644 index 0000000000..99bcbec57f --- /dev/null +++ b/packages/integration/src/azureBlobStorage/types.ts @@ -0,0 +1,29 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TokenCredential } from '@azure/identity'; +import { + StorageSharedKeyCredential, + AnonymousCredential, +} from '@azure/storage-blob'; + +export interface AzureCredentialsManager { + getCredentials( + accountName: string, + ): Promise< + TokenCredential | StorageSharedKeyCredential | AnonymousCredential + >; +} diff --git a/packages/integration/src/index.ts b/packages/integration/src/index.ts index 32c573abef..a625f5d83c 100644 --- a/packages/integration/src/index.ts +++ b/packages/integration/src/index.ts @@ -22,6 +22,7 @@ export * from './awsS3'; export * from './awsCodeCommit'; +export * from './azureBlobStorage'; export * from './azure'; export * from './bitbucket'; export * from './bitbucketCloud'; diff --git a/packages/integration/src/registry.ts b/packages/integration/src/registry.ts index 7e4b34cc34..2e8111903b 100644 --- a/packages/integration/src/registry.ts +++ b/packages/integration/src/registry.ts @@ -26,6 +26,7 @@ import { GithubIntegration } from './github/GithubIntegration'; import { GitLabIntegration } from './gitlab/GitLabIntegration'; import { GiteaIntegration } from './gitea/GiteaIntegration'; import { HarnessIntegration } from './harness/HarnessIntegration'; +import { AzureBlobStorageIntergation } from './azureBlobStorage'; /** * Holds all registered SCM integrations, of all types. @@ -36,6 +37,7 @@ export interface ScmIntegrationRegistry extends ScmIntegrationsGroup { awsS3: ScmIntegrationsGroup; awsCodeCommit: ScmIntegrationsGroup; + azureBlobStorage: ScmIntegrationsGroup; azure: ScmIntegrationsGroup; /** * @deprecated in favor of `bitbucketCloud` and `bitbucketServer` diff --git a/plugins/catalog-backend-module-azure/package.json b/plugins/catalog-backend-module-azure/package.json index 774db39280..c5ab759113 100644 --- a/plugins/catalog-backend-module-azure/package.json +++ b/plugins/catalog-backend-module-azure/package.json @@ -51,6 +51,8 @@ "test": "backstage-cli package test" }, "dependencies": { + "@azure/identity": "^4.0.0", + "@azure/storage-blob": "^12.5.0", "@backstage/backend-plugin-api": "workspace:^", "@backstage/config": "workspace:^", "@backstage/integration": "workspace:^", diff --git a/plugins/catalog-backend-module-azure/src/index.ts b/plugins/catalog-backend-module-azure/src/index.ts index dbbdff67ce..da1aaef30f 100644 --- a/plugins/catalog-backend-module-azure/src/index.ts +++ b/plugins/catalog-backend-module-azure/src/index.ts @@ -23,3 +23,4 @@ export { default } from './module'; export { AzureDevOpsDiscoveryProcessor } from './processors'; export { AzureDevOpsEntityProvider } from './providers'; +export { AzureBlobStorageEntityProvider } from './providers'; diff --git a/plugins/catalog-backend-module-azure/src/module/catalogModuleAzureDevOpsEntityProvider.test.ts b/plugins/catalog-backend-module-azure/src/module/catalogModuleAzureDevOpsEntityProvider.test.ts index 32eb190460..e47635de91 100644 --- a/plugins/catalog-backend-module-azure/src/module/catalogModuleAzureDevOpsEntityProvider.test.ts +++ b/plugins/catalog-backend-module-azure/src/module/catalogModuleAzureDevOpsEntityProvider.test.ts @@ -17,8 +17,11 @@ import { SchedulerServiceTaskScheduleDefinition } from '@backstage/backend-plugin-api'; import { mockServices, startTestBackend } from '@backstage/backend-test-utils'; import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; -import { catalogModuleAzureDevOpsEntityProvider } from './catalogModuleAzureDevOpsEntityProvider'; -import { AzureDevOpsEntityProvider } from '../providers'; +import { catalogModuleAzureEntityProvider } from './catalogModuleAzureDevOpsEntityProvider'; +import { + AzureBlobStorageEntityProvider, + AzureDevOpsEntityProvider, +} from '../providers'; describe('catalogModuleAzureDevOpsEntityProvider', () => { it('should register provider at the catalog extension point', async () => { @@ -58,7 +61,7 @@ describe('catalogModuleAzureDevOpsEntityProvider', () => { await startTestBackend({ extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]], features: [ - catalogModuleAzureDevOpsEntityProvider, + catalogModuleAzureEntityProvider, mockServices.rootConfig.factory({ data: config }), mockServices.logger.factory(), scheduler.factory, @@ -73,4 +76,54 @@ describe('catalogModuleAzureDevOpsEntityProvider', () => { ); expect(runner).not.toHaveBeenCalled(); }); + + it('should register azure blob storage provider at the catalog extension point', async () => { + let addedProviders: Array | undefined; + let usedSchedule: SchedulerServiceTaskScheduleDefinition | undefined; + + const extensionPoint = { + addEntityProvider: (providers: any) => { + addedProviders = providers; + }, + }; + const runner = jest.fn(); + const scheduler = mockServices.scheduler.mock({ + createScheduledTaskRunner(schedule) { + usedSchedule = schedule; + return { run: runner }; + }, + }); + + const config = { + catalog: { + providers: { + azureBlob: { + containerName: 'test', + schedule: { + frequency: 'P1M', + timeout: 'PT3M', + }, + }, + }, + }, + }; + + await startTestBackend({ + extensionPoints: [[catalogProcessingExtensionPoint, extensionPoint]], + features: [ + catalogModuleAzureEntityProvider, + mockServices.rootConfig.factory({ data: config }), + mockServices.logger.factory(), + scheduler.factory, + ], + }); + + expect(usedSchedule?.frequency).toEqual({ months: 1 }); + expect(usedSchedule?.timeout).toEqual({ minutes: 3 }); + expect(addedProviders?.length).toEqual(1); + expect(addedProviders?.pop()?.getProviderName()).toEqual( + 'azureBlobStorage-provider:default', + ); + expect(runner).not.toHaveBeenCalled(); + }); }); diff --git a/plugins/catalog-backend-module-azure/src/module/catalogModuleAzureDevOpsEntityProvider.ts b/plugins/catalog-backend-module-azure/src/module/catalogModuleAzureDevOpsEntityProvider.ts index 3aaa1b5b08..cc808292f3 100644 --- a/plugins/catalog-backend-module-azure/src/module/catalogModuleAzureDevOpsEntityProvider.ts +++ b/plugins/catalog-backend-module-azure/src/module/catalogModuleAzureDevOpsEntityProvider.ts @@ -19,16 +19,19 @@ import { createBackendModule, } from '@backstage/backend-plugin-api'; import { catalogProcessingExtensionPoint } from '@backstage/plugin-catalog-node/alpha'; -import { AzureDevOpsEntityProvider } from '../providers'; +import { + AzureBlobStorageEntityProvider, + AzureDevOpsEntityProvider, +} from '../providers'; /** * Registers the AzureDevOpsEntityProvider with the catalog processing extension point. * * @public */ -export const catalogModuleAzureDevOpsEntityProvider = createBackendModule({ +export const catalogModuleAzureEntityProvider = createBackendModule({ pluginId: 'catalog', - moduleId: 'azure-dev-ops-entity-provider', + moduleId: 'azure-providers', register(env) { env.registerInit({ deps: { @@ -38,12 +41,25 @@ export const catalogModuleAzureDevOpsEntityProvider = createBackendModule({ scheduler: coreServices.scheduler, }, async init({ config, catalog, logger, scheduler }) { - catalog.addEntityProvider( - AzureDevOpsEntityProvider.fromConfig(config, { - logger, - scheduler, - }), - ); + // Check for Azure Blob Storage provider configuration and register it + if (config.has('catalog.providers.azureBlob')) { + catalog.addEntityProvider( + AzureBlobStorageEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ); + } + + // Check for Azure DevOps provider configuration and register it + if (config.has('catalog.providers.azureDevOps')) { + catalog.addEntityProvider( + AzureDevOpsEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ); + } }, }); }, diff --git a/plugins/catalog-backend-module-azure/src/module/index.ts b/plugins/catalog-backend-module-azure/src/module/index.ts index 329ad604e2..6e560cf17f 100644 --- a/plugins/catalog-backend-module-azure/src/module/index.ts +++ b/plugins/catalog-backend-module-azure/src/module/index.ts @@ -14,4 +14,4 @@ * limitations under the License. */ -export { catalogModuleAzureDevOpsEntityProvider as default } from './catalogModuleAzureDevOpsEntityProvider'; +export { catalogModuleAzureEntityProvider as default } from './catalogModuleAzureDevOpsEntityProvider'; diff --git a/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.test.ts b/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.test.ts new file mode 100644 index 0000000000..5ec50754c0 --- /dev/null +++ b/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.test.ts @@ -0,0 +1,246 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + SchedulerService, + SchedulerServiceTaskRunner, + SchedulerServiceTaskInvocationDefinition, +} from '@backstage/backend-plugin-api'; +import { ConfigReader } from '@backstage/config'; +import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; +import { AzureBlobStorageEntityProvider } from './AzureBlobStorageEntityProvider'; +import { mockServices } from '@backstage/backend-test-utils'; + +class PersistingTaskRunner implements SchedulerServiceTaskRunner { + private tasks: SchedulerServiceTaskInvocationDefinition[] = []; + + getTasks() { + return this.tasks; + } + + run(task: SchedulerServiceTaskInvocationDefinition): Promise { + this.tasks.push(task); + return Promise.resolve(undefined); + } +} + +const blobs = ['key1.yaml', 'key2.yaml', 'key3.yaml', 'key4.yaml']; +const createBlobList = (blobsArray: string[]) => { + return blobsArray.map(blob => ({ + name: blob, + })); +}; +// Mocking Azure Storage Blob Library +jest.mock('@azure/storage-blob', () => { + return { + BlobServiceClient: jest.fn().mockImplementation(() => ({ + url: 'https://myaccount.blob.core.windows.net/', + getContainerClient: jest.fn().mockImplementation(() => ({ + listBlobsFlat: jest.fn(async function* () { + yield* createBlobList(blobs); + }), + })), + })), + }; +}); + +const logger = mockServices.logger.mock(); + +describe('AzureBlobStorageEntityProvider', () => { + const containerName = 'container-1'; + + const expectMutation = async ( + providerId: string, + providerConfig: object, + expectedBaseUrl: string, + names: Record, + integrationConfig?: object, + scheduleInConfig?: boolean, + ) => { + const config = new ConfigReader({ + integrations: { + azureBlobStorage: integrationConfig ? [integrationConfig] : [], + }, + catalog: { + providers: { + azureBlob: { + [providerId]: providerConfig, + }, + }, + }, + }); + + const schedulingConfig: Record = {}; + const normalizedExpectedBaseUrl = expectedBaseUrl.endsWith('/') + ? expectedBaseUrl + : `${expectedBaseUrl}/`; + const schedule = new PersistingTaskRunner(); + const entityProviderConnection: EntityProviderConnection = { + applyMutation: jest.fn(), + refresh: jest.fn(), + }; + + if (scheduleInConfig) { + schedulingConfig.scheduler = { + createScheduledTaskRunner: (_: any) => schedule, + } as unknown as SchedulerService; + } else { + schedulingConfig.schedule = schedule; + } + + const provider = AzureBlobStorageEntityProvider.fromConfig(config, { + ...schedulingConfig, + logger, + })[0]; + + expect(provider.getProviderName()).toEqual( + `azureBlobStorage-provider:${providerId}`, + ); + + try { + await provider.connect(entityProviderConnection); + } catch (error) { + console.error('Error during provider connection:', error); + } + + const taskDef = schedule.getTasks()[0]; + expect(taskDef.id).toEqual( + `azureBlobStorage-provider:${providerId}:refresh`, + ); + + await (taskDef.fn as () => Promise)(); + + const expectedEntities = blobs.map(blob => { + const url = encodeURI(`${normalizedExpectedBaseUrl}${blob}`); + return { + entity: { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Location', + metadata: { + annotations: { + 'backstage.io/managed-by-location': `url:${url}`, + 'backstage.io/managed-by-origin-location': `url:${url}`, + }, + name: expect.stringMatching(/generated-[a-f0-9]{40}/), + }, + spec: { + presence: 'required', + target: `${url}`, + type: 'url', + }, + }, + locationKey: `azureBlobStorage-provider:${providerId}`, + }; + }); + + expect(entityProviderConnection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: expectedEntities, + }); + }; + + it('apply full update on scheduled execution', async () => { + return expectMutation( + 'staticContainer', + { + containerName, + }, + 'https://myaccount.blob.core.windows.net/container-1/', + { + 'key1.yaml': 'generated-8ece85ad90200c6577b99f553dcbedde05fa34bb', + 'key2.yaml': 'generated-6b54c6aaa44696f5e91ce0f54fb27bf837549d11', + 'key3.yaml': 'generated-88c703cf1aa66913db4033b029adc0b174574646', + 'key4.yaml': 'generated-2b7e068bb4ec818c14f179a1e721843fc2dbc5f9', + }, + { + accountName: 'myaccount', + }, + ); + }); + + it('apply full update no prefix', async () => { + return expectMutation( + 'staticContainerNoPrefix', + { + containerName, + schedule: { + frequency: { minutes: 30 }, + timeout: { minutes: 3 }, + }, + }, + 'https://myaccount.blob.core.windows.net/container-1/', + { + 'key1.yaml': 'generated-8ece85ad90200c6577b99f553dcbedde05fa34bb', + 'key2.yaml': 'generated-6b54c6aaa44696f5e91ce0f54fb27bf837549d11', + 'key3.yaml': 'generated-88c703cf1aa66913db4033b029adc0b174574646', + 'key4.yaml': 'generated-2b7e068bb4ec818c14f179a1e721843fc2dbc5f9', + }, + { + host: 'blob.core.windows.net', + accountName: 'myaccount', + }, + ); + }); + + it('fail without schedule and scheduler', () => { + const config = new ConfigReader({ + catalog: { + providers: { + azureBlob: { + test: { + containerName: 'container-1', + prefix: 'sub/dir/', + accountName: 'myaccount', + }, + }, + }, + }, + }); + + expect(() => + AzureBlobStorageEntityProvider.fromConfig(config, { + logger, + }), + ).toThrow('Either schedule or scheduler must be provided'); + }); + + it('fail with scheduler but no schedule config', () => { + const scheduler = { + createScheduledTaskRunner: (_: any) => jest.fn(), + } as unknown as SchedulerService; + const config = new ConfigReader({ + catalog: { + providers: { + azureBlob: { + test: { + containerName: 'container-1', + prefix: 'sub/dir/', + accountName: 'myaccount', + }, + }, + }, + }, + }); + + expect(() => + AzureBlobStorageEntityProvider.fromConfig(config, { + logger, + scheduler, + }), + ).toThrow( + 'No schedule provided neither via code nor config for AzureBlobStorageEntityProvider:test.', + ); + }); +}); diff --git a/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.ts b/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.ts new file mode 100644 index 0000000000..47154f5c89 --- /dev/null +++ b/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.ts @@ -0,0 +1,228 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + AnonymousCredential, + BlobServiceClient, + ContainerClient, + StorageSharedKeyCredential, +} from '@azure/storage-blob'; +import { Config } from '@backstage/config'; +import { + LoggerService, + SchedulerService, + SchedulerServiceTaskRunner, +} from '@backstage/backend-plugin-api'; +import { + EntityProvider, + EntityProviderConnection, + locationSpecToLocationEntity, +} from '@backstage/plugin-catalog-node'; +import { LocationSpec } from '@backstage/plugin-catalog-common'; +import * as uuid from 'uuid'; +import { readAzureBlobStorageConfigs } from './config'; +import { + AzureBlobStorageIntergation, + DefaultAzureCredentialsManager, + ScmIntegrations, +} from '@backstage/integration'; +import { TokenCredential } from '@azure/identity'; +import { AzureBlobStorageConfig } from './types'; + +export class AzureBlobStorageEntityProvider implements EntityProvider { + private readonly logger: LoggerService; + private connection?: EntityProviderConnection; + private blobServiceClient?: BlobServiceClient; + private readonly scheduleFn: () => Promise; + + static fromConfig( + configRoot: Config, + options: { + logger: LoggerService; + schedule?: SchedulerServiceTaskRunner; + scheduler?: SchedulerService; + }, + ): AzureBlobStorageEntityProvider[] { + const providerConfigs = readAzureBlobStorageConfigs(configRoot); + + const scmIntegration = ScmIntegrations.fromConfig(configRoot); + const credentialsProvider = + DefaultAzureCredentialsManager.fromIntegrations(scmIntegration); + if (!options.schedule && !options.scheduler) { + throw new Error('Either schedule or scheduler must be provided.'); + } + + return providerConfigs.map(providerConfig => { + const integration = scmIntegration.azureBlobStorage.list()[0]; + if (!integration) { + throw new Error( + `There is no Azure blob storage integration for host. Please add a configuration entry for it under integrations.azure`, + ); + } + + if (!options.schedule && !providerConfig.schedule) { + throw new Error( + `No schedule provided neither via code nor config for AzureBlobStorageEntityProvider:${providerConfig.id}.`, + ); + } + + const taskRunner = + options.schedule ?? + options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!); + + return new AzureBlobStorageEntityProvider( + providerConfig, + integration, + credentialsProvider, + options.logger, + taskRunner, + ); + }); + } + constructor( + private readonly config: AzureBlobStorageConfig, + private readonly integration: AzureBlobStorageIntergation, + private readonly credentialsProvider: DefaultAzureCredentialsManager, + logger: LoggerService, + schedule: SchedulerServiceTaskRunner, + ) { + this.logger = logger.child({ target: this.getProviderName() }); + this.scheduleFn = this.createScheduleFn(schedule); + } + + private createScheduleFn( + taskRunner: SchedulerServiceTaskRunner, + ): () => Promise { + return async () => { + const taskId = `${this.getProviderName()}:refresh`; + return taskRunner.run({ + id: taskId, + fn: async () => { + const logger = this.logger.child({ + class: AzureBlobStorageEntityProvider.prototype.constructor.name, + taskId, + taskInstanceId: uuid.v4(), + }); + + try { + await this.refresh(logger); + } catch (error) { + logger.error( + `${this.getProviderName()} refresh failed, ${error}`, + error, + ); + } + }, + }); + }; + } + + getProviderName(): string { + return `azureBlobStorage-provider:${this.config.id}`; + } + + async connect(connection: EntityProviderConnection): Promise { + this.connection = connection; + let credential: + | TokenCredential + | StorageSharedKeyCredential + | AnonymousCredential; + if (this.integration.config.accountKey) { + credential = new StorageSharedKeyCredential( + this.integration.config.accountName as string, + this.integration.config.accountKey as string, + ); // StorageSharedKeyCredential is only allowed in node.js runtime not in browser + } else { + credential = await this.credentialsProvider.getCredentials( + this.integration.config.accountName as string, + ); + } + let blobServiceClientUrl: string; + + if (this.integration.config.endpoint) { + if (this.integration.config.sasToken) { + blobServiceClientUrl = `${this.integration.config.endpoint}?${this.integration.config.sasToken}`; + } else { + blobServiceClientUrl = `${this.integration.config.endpoint}`; + } + } else { + blobServiceClientUrl = `https://${this.integration.config.accountName}.${this.integration.config.host}`; + } + + this.blobServiceClient = new BlobServiceClient( + blobServiceClientUrl, + credential, + ); + await this.scheduleFn(); + } + + async refresh(logger: LoggerService) { + if (!this.connection) { + throw new Error('Not initialized'); + } + + logger.info('Discovering Azure Blob Storage blobs'); + + const keys = await this.listAllBlobKeys(); + logger.info(`Discovered ${keys.length} Azure Blob Storage blobs`); + + const locations = keys.map(key => this.createLocationSpec(key)); + + await this.connection.applyMutation({ + type: 'full', + entities: locations.map(location => { + return { + locationKey: this.getProviderName(), + entity: locationSpecToLocationEntity({ location }), + }; + }), + }); + + logger.info( + `Committed ${locations.length} Locations for Azure Blob Storage blobs`, + ); + } + + private async listAllBlobKeys(): Promise { + const keys: string[] = []; + const containerClient = this.blobServiceClient?.getContainerClient( + this.config.containerName, + ); + + for await (const blob of ( + containerClient as ContainerClient + ).listBlobsFlat()) { + if (blob.name) { + keys.push(blob.name); + } + } + + return keys; + } + + private createLocationSpec(key: string): LocationSpec { + return { + type: 'url', + target: this.createObjectUrl(key), + presence: 'required', + }; + } + + private createObjectUrl(key: string): string { + const endpoint = this.blobServiceClient?.url; + return `${endpoint}${this.config.containerName}/${key}`; + } +} diff --git a/plugins/catalog-backend-module-azure/src/providers/config.test.ts b/plugins/catalog-backend-module-azure/src/providers/config.test.ts index 5d5ae930c1..21c4ece78c 100644 --- a/plugins/catalog-backend-module-azure/src/providers/config.test.ts +++ b/plugins/catalog-backend-module-azure/src/providers/config.test.ts @@ -15,7 +15,7 @@ */ import { ConfigReader } from '@backstage/config'; -import { readAzureDevOpsConfigs } from './config'; +import { readAzureBlobStorageConfigs, readAzureDevOpsConfigs } from './config'; describe('readAzureDevOpsConfigs', () => { it('reads all provider configs and set default values', () => { @@ -106,3 +106,69 @@ describe('readAzureDevOpsConfigs', () => { }); }); }); + +describe('readAzureBlobStorageConfigs', () => { + it('reads single and multiple Azure Blob Storage provider configs', () => { + const provider1 = { + containerName: 'container-1', + schedule: { + frequency: 'PT30M', + timeout: { + minutes: 3, + }, + }, + }; + const provider2 = { + containerName: 'container-2', + }; + + const configSingle = { + catalog: { + providers: { + azureBlob: provider2, + }, + }, + }; + + const configMulti = { + catalog: { + providers: { + azureBlob: { + provider1, + provider2, + }, + }, + }, + }; + + // Single provider case + const actualSingle = readAzureBlobStorageConfigs( + new ConfigReader(configSingle), + ); + expect(actualSingle).toHaveLength(1); + expect(actualSingle[0]).toEqual({ + id: 'default', + containerName: 'container-2', + schedule: undefined, // no schedule provided in this case + }); + + // Multiple providers case + const actualMulti = readAzureBlobStorageConfigs( + new ConfigReader(configMulti), + ); + expect(actualMulti).toHaveLength(2); + expect(actualMulti[0]).toEqual({ + id: 'provider1', + containerName: 'container-1', + schedule: { + ...provider1.schedule, + frequency: { minutes: 30 }, + }, + }); + expect(actualMulti[1]).toEqual({ + id: 'provider2', + containerName: 'container-2', + schedule: undefined, // no schedule provided + }); + }); +}); diff --git a/plugins/catalog-backend-module-azure/src/providers/config.ts b/plugins/catalog-backend-module-azure/src/providers/config.ts index 17616b06bc..d082726b14 100644 --- a/plugins/catalog-backend-module-azure/src/providers/config.ts +++ b/plugins/catalog-backend-module-azure/src/providers/config.ts @@ -16,7 +16,9 @@ import { readSchedulerServiceTaskScheduleDefinitionFromConfig } from '@backstage/backend-plugin-api'; import { Config } from '@backstage/config'; -import { AzureDevOpsConfig } from './types'; +import { AzureDevOpsConfig, AzureBlobStorageConfig } from './types'; + +const DEFAULT_PROVIDER_ID = 'default'; export function readAzureDevOpsConfigs(config: Config): AzureDevOpsConfig[] { const configs: AzureDevOpsConfig[] = []; @@ -61,3 +63,50 @@ function readAzureDevOpsConfig(id: string, config: Config): AzureDevOpsConfig { schedule, }; } + +export function readAzureBlobStorageConfigs( + config: Config, +): AzureBlobStorageConfig[] { + const configs: AzureBlobStorageConfig[] = []; + + const providerConfigs = config.getOptionalConfig( + 'catalog.providers.azureBlob', + ); + + if (!providerConfigs) { + return configs; + } + + if (providerConfigs.has('containerName')) { + // simple/single config variant + configs.push( + readAzureBlobStorageConfig(DEFAULT_PROVIDER_ID, providerConfigs), + ); + + return configs; + } + + for (const id of providerConfigs.keys()) { + configs.push(readAzureBlobStorageConfig(id, providerConfigs.getConfig(id))); + } + + return configs; +} + +function readAzureBlobStorageConfig( + id: string, + config: Config, +): AzureBlobStorageConfig { + const containerName = config.getString('containerName'); + const schedule = config.has('schedule') + ? readSchedulerServiceTaskScheduleDefinitionFromConfig( + config.getConfig('schedule'), + ) + : undefined; + + return { + id, + containerName, + schedule, + }; +} diff --git a/plugins/catalog-backend-module-azure/src/providers/index.ts b/plugins/catalog-backend-module-azure/src/providers/index.ts index 450fc0af27..5c9616e162 100644 --- a/plugins/catalog-backend-module-azure/src/providers/index.ts +++ b/plugins/catalog-backend-module-azure/src/providers/index.ts @@ -15,3 +15,4 @@ */ export { AzureDevOpsEntityProvider } from './AzureDevOpsEntityProvider'; +export { AzureBlobStorageEntityProvider } from './AzureBlobStorageEntityProvider'; diff --git a/plugins/catalog-backend-module-azure/src/providers/types.ts b/plugins/catalog-backend-module-azure/src/providers/types.ts index 5ca9dc94dc..eb16dbd6ea 100644 --- a/plugins/catalog-backend-module-azure/src/providers/types.ts +++ b/plugins/catalog-backend-module-azure/src/providers/types.ts @@ -26,3 +26,9 @@ export type AzureDevOpsConfig = { path: string; schedule?: SchedulerServiceTaskScheduleDefinition; }; + +export type AzureBlobStorageConfig = { + id: string; + containerName: string; + schedule?: SchedulerServiceTaskScheduleDefinition; +}; diff --git a/yarn.lock b/yarn.lock index af909aa8c1..a76d05d9db 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3605,6 +3605,8 @@ __metadata: "@aws-sdk/credential-providers": ^3.350.0 "@aws-sdk/types": ^3.347.0 "@aws-sdk/util-stream-node": ^3.350.0 + "@azure/identity": ^4.0.0 + "@azure/storage-blob": ^12.5.0 "@backstage/backend-app-api": "workspace:^" "@backstage/backend-common": ^0.25.0 "@backstage/backend-dev-utils": "workspace:^" @@ -4649,6 +4651,7 @@ __metadata: resolution: "@backstage/integration@workspace:packages/integration" dependencies: "@azure/identity": ^4.0.0 + "@azure/storage-blob": ^12.5.0 "@backstage/cli": "workspace:^" "@backstage/config": "workspace:^" "@backstage/config-loader": "workspace:^" @@ -5396,6 +5399,8 @@ __metadata: version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-backend-module-azure@workspace:plugins/catalog-backend-module-azure" dependencies: + "@azure/identity": ^4.0.0 + "@azure/storage-blob": ^12.5.0 "@backstage/backend-plugin-api": "workspace:^" "@backstage/backend-test-utils": "workspace:^" "@backstage/cli": "workspace:^" From c40cb951a96d0740d820f73df6f9804990b4dc35 Mon Sep 17 00:00:00 2001 From: NIKUNJ LALITKUMAR HUDKA Date: Fri, 18 Oct 2024 18:57:42 -0300 Subject: [PATCH 02/13] chore: remove used code and solve eslint error Signed-off-by: NIKUNJ LALITKUMAR HUDKA --- .../urlReader/lib/AzureBlobStorageUrlReader.ts | 18 +++--------------- .../DefaultAzureCredentialsManager.test.ts | 2 -- .../AzureBlobStorageEntityProvider.test.ts | 16 +++------------- 3 files changed, 6 insertions(+), 30 deletions(-) diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureBlobStorageUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureBlobStorageUrlReader.ts index f0c7e2570e..039af6d3ed 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureBlobStorageUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureBlobStorageUrlReader.ts @@ -39,10 +39,7 @@ import { UrlReaderServiceSearchResponse, } from '@backstage/backend-plugin-api'; -export function parseUrl( - url: string, - config: AzureBlobStorageIntergation, -): { path: string; container: string } { +export function parseUrl(url: string): { path: string; container: string } { const parsedUrl = new URL(url); const pathSegments = parsedUrl.pathname.split('/').filter(Boolean); @@ -138,7 +135,7 @@ export class AzureBlobStorageUrlReader implements UrlReaderService { const { etag, lastModifiedAfter } = options ?? {}; try { - const { path, container } = parseUrl(url, this.integration); + const { path, container } = parseUrl(url); const containerClient = await this.createContainerClient(container); const blobClient = containerClient.getBlobClient(path); @@ -184,7 +181,7 @@ export class AzureBlobStorageUrlReader implements UrlReaderService { options?: UrlReaderServiceReadTreeOptions, ): Promise { try { - const { path, container } = parseUrl(url, this.integration); + const { path, container } = parseUrl(url); const containerClient = await this.createContainerClient(container); @@ -230,15 +227,6 @@ export class AzureBlobStorageUrlReader implements UrlReaderService { )}}`; } - private parseUrl(url: string): { path: string } { - const parsedUrl = new URL(url); - const path = parsedUrl.pathname.substring( - parsedUrl.pathname.lastIndexOf('/') + 1, - ); - - return { path }; - } - private async retrieveAzureBlobData(stream: Readable): Promise { return new Promise((resolve, reject) => { try { diff --git a/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsManager.test.ts b/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsManager.test.ts index 6272a68d13..d8e8be2919 100644 --- a/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsManager.test.ts +++ b/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsManager.test.ts @@ -18,9 +18,7 @@ import { AccessToken, ClientSecretCredential, DefaultAzureCredential, - TokenCredential, } from '@azure/identity'; -import { AzureBlobStorageIntegrationConfig } from './config'; import { ScmIntegrationRegistry } from '../registry'; import { ConfigReader } from '@backstage/config'; import { DefaultAzureCredentialsManager } from './DefaultAzureCredentialsProvider'; diff --git a/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.test.ts b/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.test.ts index 5ec50754c0..515d2ee7d0 100644 --- a/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.test.ts +++ b/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.test.ts @@ -48,6 +48,7 @@ jest.mock('@azure/storage-blob', () => { BlobServiceClient: jest.fn().mockImplementation(() => ({ url: 'https://myaccount.blob.core.windows.net/', getContainerClient: jest.fn().mockImplementation(() => ({ + // eslint-disable-next-line func-names listBlobsFlat: jest.fn(async function* () { yield* createBlobList(blobs); }), @@ -65,7 +66,6 @@ describe('AzureBlobStorageEntityProvider', () => { providerId: string, providerConfig: object, expectedBaseUrl: string, - names: Record, integrationConfig?: object, scheduleInConfig?: boolean, ) => { @@ -151,6 +151,7 @@ describe('AzureBlobStorageEntityProvider', () => { }); }; + // eslint-disable-next-line jest/expect-expect it('apply full update on scheduled execution', async () => { return expectMutation( 'staticContainer', @@ -158,18 +159,13 @@ describe('AzureBlobStorageEntityProvider', () => { containerName, }, 'https://myaccount.blob.core.windows.net/container-1/', - { - 'key1.yaml': 'generated-8ece85ad90200c6577b99f553dcbedde05fa34bb', - 'key2.yaml': 'generated-6b54c6aaa44696f5e91ce0f54fb27bf837549d11', - 'key3.yaml': 'generated-88c703cf1aa66913db4033b029adc0b174574646', - 'key4.yaml': 'generated-2b7e068bb4ec818c14f179a1e721843fc2dbc5f9', - }, { accountName: 'myaccount', }, ); }); + // eslint-disable-next-line jest/expect-expect it('apply full update no prefix', async () => { return expectMutation( 'staticContainerNoPrefix', @@ -181,12 +177,6 @@ describe('AzureBlobStorageEntityProvider', () => { }, }, 'https://myaccount.blob.core.windows.net/container-1/', - { - 'key1.yaml': 'generated-8ece85ad90200c6577b99f553dcbedde05fa34bb', - 'key2.yaml': 'generated-6b54c6aaa44696f5e91ce0f54fb27bf837549d11', - 'key3.yaml': 'generated-88c703cf1aa66913db4033b029adc0b174574646', - 'key4.yaml': 'generated-2b7e068bb4ec818c14f179a1e721843fc2dbc5f9', - }, { host: 'blob.core.windows.net', accountName: 'myaccount', From 634f6b72ca71e11c934088fa904776d794c68c6a Mon Sep 17 00:00:00 2001 From: NIKUNJ LALITKUMAR HUDKA Date: Sun, 20 Oct 2024 22:58:08 -0300 Subject: [PATCH 03/13] prettier app-config.yaml file changes Signed-off-by: NIKUNJ LALITKUMAR HUDKA --- app-config.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index 86737413fe..1486e8b3ec 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -225,8 +225,8 @@ integrations: azureBlobStorage: - accountName: ${ACCOUNT_NAME} # required endpoint: ${CUSTOM_ENDPOINT} # custom endpoint will require either aadCredentials or sasToken - sasToken: ${SAS_TOKEN} - aadCredential: + sasToken: ${SAS_TOKEN} + aadCredential: clientId: ${CLIENT_ID} tenantId: ${TENANT_ID} clientSecret: ${CLIENT_SECRET} @@ -259,10 +259,10 @@ catalog: azureBlob: containerName: ${CONTAINER_NAME} schedule: # same options as in TaskScheduleDefinition - # supports cron, ISO duration, "human duration" as used in code - frequency: { minutes: 30 } - # supports ISO duration, "human duration" as used in code - timeout: { minutes: 3 } + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } backstageOpenapi: plugins: - catalog From 05ceb85a47a90e146f5b4d0bcae632e166b71217 Mon Sep 17 00:00:00 2001 From: NIKUNJ LALITKUMAR HUDKA Date: Wed, 23 Oct 2024 15:26:19 -0300 Subject: [PATCH 04/13] Test file change add new integration of azure blob storage and review changes Signed-off-by: NIKUNJ LALITKUMAR HUDKA --- .../lib/AzureBlobStorageUrlReader.ts | 37 ++++++------------- .../src/api/ScmIntegrationsApi.test.ts | 2 +- 2 files changed, 12 insertions(+), 27 deletions(-) diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureBlobStorageUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureBlobStorageUrlReader.ts index 039af6d3ed..4de5f79f4e 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureBlobStorageUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureBlobStorageUrlReader.ts @@ -15,6 +15,7 @@ */ import { + BlobDownloadOptions, BlobServiceClient, ContainerClient, StorageSharedKeyCredential, @@ -142,7 +143,8 @@ export class AzureBlobStorageUrlReader implements UrlReaderService { const abortController = new AbortController(); - const getBlobOptions = { + const getBlobOptions: BlobDownloadOptions = { + abortSignal: abortController.signal, conditions: { ...(etag && { ifNoneMatch: etag }), ...(lastModifiedAfter && { ifModifiedSince: lastModifiedAfter }), @@ -156,14 +158,13 @@ export class AzureBlobStorageUrlReader implements UrlReaderService { getBlobOptions, ); - const data = await this.retrieveAzureBlobData( + return ReadUrlResponseFactory.fromReadable( downloadBlockBlobResponse.readableStreamBody as Readable, + { + etag: downloadBlockBlobResponse.etag, + lastModifiedAt: downloadBlockBlobResponse.lastModified, + }, ); - - return ReadUrlResponseFactory.fromReadable(data, { - etag: downloadBlockBlobResponse.etag, - lastModifiedAt: downloadBlockBlobResponse.lastModified, - }); } catch (e) { if (e.$metadata && e.$metadata.httpStatusCode === 304) { throw new NotModifiedError(); @@ -195,12 +196,11 @@ export class AzureBlobStorageUrlReader implements UrlReaderService { abortController.abort(), ); const downloadBlockBlobResponse = await blobClient.download(); - const data = await this.retrieveAzureBlobData( - downloadBlockBlobResponse.readableStreamBody as Readable, - ); responses.push({ - data: Readable.from(data), + data: Readable.from( + downloadBlockBlobResponse.readableStreamBody as Readable, + ), path: relative(path, blob.name), lastModifiedAt: blob.properties.lastModified, }); @@ -226,19 +226,4 @@ export class AzureBlobStorageUrlReader implements UrlReaderService { accountKey, )}}`; } - - private async retrieveAzureBlobData(stream: Readable): Promise { - return new Promise((resolve, reject) => { - try { - const chunks: any[] = []; - stream.on('data', chunk => chunks.push(chunk)); - stream.on('error', (e: Error) => - reject(new ForwardedError('Unable to read stream', e)), - ); - stream.on('end', () => resolve(Readable.from(Buffer.concat(chunks)))); - } catch (e) { - throw new ForwardedError('Unable to parse the response data', e); - } - }); - } } diff --git a/packages/integration-react/src/api/ScmIntegrationsApi.test.ts b/packages/integration-react/src/api/ScmIntegrationsApi.test.ts index fb53bbdb12..0a1f1b1b8f 100644 --- a/packages/integration-react/src/api/ScmIntegrationsApi.test.ts +++ b/packages/integration-react/src/api/ScmIntegrationsApi.test.ts @@ -26,6 +26,6 @@ describe('scmIntegrationsApiRef', () => { it('should be instantiated', () => { const i = ScmIntegrationsApi.fromConfig(new ConfigReader({})); - expect(i.list().length).toBe(6); // The default ones + expect(i.list().length).toBe(7); // The default ones }); }); From 8d0f80e5ab33b3c272b95a7bee8c73393b4730ef Mon Sep 17 00:00:00 2001 From: NIKUNJ LALITKUMAR HUDKA Date: Wed, 23 Oct 2024 15:33:50 -0300 Subject: [PATCH 05/13] changeset changes from major to minor Signed-off-by: NIKUNJ LALITKUMAR HUDKA --- .changeset/cyan-rocks-divide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/cyan-rocks-divide.md b/.changeset/cyan-rocks-divide.md index 8fa76a493e..8576e515d8 100644 --- a/.changeset/cyan-rocks-divide.md +++ b/.changeset/cyan-rocks-divide.md @@ -1,5 +1,5 @@ --- -'@backstage/integration': major +'@backstage/integration': minor --- Add the integration for Azure blob storage to read the credentials to access the storage account and provide the default credential provider. From 6997a53fa7f2c6fc162d447824d06346b13e25d7 Mon Sep 17 00:00:00 2001 From: Nikunj Hudka Date: Mon, 4 Nov 2024 11:11:37 -0400 Subject: [PATCH 06/13] chore: api report generated and removed comments Signed-off-by: Nikunj Hudka --- .../DefaultAzureCredentialsProvider.ts | 48 +------------------ .../report.api.md | 36 +++++++++++++- 2 files changed, 35 insertions(+), 49 deletions(-) diff --git a/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsProvider.ts b/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsProvider.ts index 9f209da61c..8971e560ad 100644 --- a/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsProvider.ts +++ b/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsProvider.ts @@ -23,10 +23,6 @@ import { AzureBlobStorageIntegrationConfig } from './config'; import { AzureCredentialsManager } from './types'; import { ScmIntegrationRegistry } from '../registry'; -/** - * Default Azure Credentials Manager to dynamically select and manage Azure credentials. - * It supports Service Principal, Managed Identity, SAS Token, Connection String, Account Key, and Anonymous access. - */ export class DefaultAzureCredentialsManager implements AzureCredentialsManager { private config: AzureBlobStorageIntegrationConfig; private cachedCredentials: Map; @@ -49,48 +45,13 @@ export class DefaultAzureCredentialsManager implements AzureCredentialsManager { return new DefaultAzureCredentialsManager(azureConfig); } - /** - * Determines the appropriate credential method and returns credentials for BlobServiceClient. - * Supports: - * - Service Principal - * - Managed Identity - * - SAS Token - * - Connection String - * - Account Key - * - Anonymous access - */ async getCredentials(accountName: string): Promise { - // Check if the credentials are already cached if (this.cachedCredentials.has(accountName)) { return this.cachedCredentials.get(accountName)!; } let credential: TokenCredential; - // Check for SAS Token - // if (this.config.sasToken) { - // // console.log('Using SAS Token for Azure Blob Storage authentication'); - // // SAS Token does not return a credential but can be used directly in BlobServiceClient - // // Here we can simply return undefined or keep a placeholder if needed - // return this.config.sasToken; // Or return a string for the URL using the SAS token - // } - // // Check for Connection String - // else if (this.config.connectionString) { - // // console.log( - // // 'Using Connection String for Azure Blob Storage authentication', - // // ); - // // return undefined; // Connection string will also not return a specific credential object - // } - // Check for Account Key - // if (this.config.accountKey) { - // // console.log('Using Account Key for Azure Blob Storage authentication'); - // credential = new StorageSharedKeyCredential( - // accountName, - // this.config.accountKey, - // ); - // } - // Check for AAD credentials - if ( this.config.aadCredential && this.config.aadCredential.clientId && @@ -102,14 +63,7 @@ export class DefaultAzureCredentialsManager implements AzureCredentialsManager { this.config.aadCredential.clientId, this.config.aadCredential.clientSecret, ); - } - // Check for Anonymous access - // else if (this.config.anonymousAccess) { - // console.log('Using Anonymous Credential for Azure Blob Storage access'); - // credential = new AnonymousCredential(); - // } - // Fallback to Managed Identity - else { + } else { credential = new DefaultAzureCredential(); } diff --git a/plugins/catalog-backend-module-azure/report.api.md b/plugins/catalog-backend-module-azure/report.api.md index d11197d7f2..f515349e94 100644 --- a/plugins/catalog-backend-module-azure/report.api.md +++ b/plugins/catalog-backend-module-azure/report.api.md @@ -3,18 +3,50 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { AzureBlobStorageIntergation } from '@backstage/integration'; import { BackendFeature } from '@backstage/backend-plugin-api'; import { CatalogProcessor } from '@backstage/plugin-catalog-node'; import { CatalogProcessorEmit } from '@backstage/plugin-catalog-node'; import { Config } from '@backstage/config'; +import { DefaultAzureCredentialsManager } from '@backstage/integration'; import { EntityProvider } from '@backstage/plugin-catalog-node'; import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; import { LocationSpec } from '@backstage/plugin-catalog-common'; import { LoggerService } from '@backstage/backend-plugin-api'; import { SchedulerService } from '@backstage/backend-plugin-api'; import { SchedulerServiceTaskRunner } from '@backstage/backend-plugin-api'; +import { SchedulerServiceTaskScheduleDefinition } from '@backstage/backend-plugin-api'; import { ScmIntegrationRegistry } from '@backstage/integration'; +// Warning: (ae-missing-release-tag) "AzureBlobStorageEntityProvider" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export class AzureBlobStorageEntityProvider implements EntityProvider { + // Warning: (ae-forgotten-export) The symbol "AzureBlobStorageConfig" needs to be exported by the entry point index.d.ts + constructor( + config: AzureBlobStorageConfig, + integration: AzureBlobStorageIntergation, + credentialsProvider: DefaultAzureCredentialsManager, + logger: LoggerService, + schedule: SchedulerServiceTaskRunner, + ); + // (undocumented) + connect(connection: EntityProviderConnection): Promise; + // (undocumented) + static fromConfig( + configRoot: Config, + options: { + logger: LoggerService; + schedule?: SchedulerServiceTaskRunner; + scheduler?: SchedulerService; + }, + ): AzureBlobStorageEntityProvider[]; + // (undocumented) + getProviderName(): string; + // (undocumented) + refresh(logger: LoggerService): Promise; +} + // @public export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { constructor(options: { @@ -56,6 +88,6 @@ export class AzureDevOpsEntityProvider implements EntityProvider { } // @public -const catalogModuleAzureDevOpsEntityProvider: BackendFeature; -export default catalogModuleAzureDevOpsEntityProvider; +const catalogModuleAzureEntityProvider: BackendFeature; +export default catalogModuleAzureEntityProvider; ``` From 68464c85be1f1fbe8460246194e3229dbe4a5723 Mon Sep 17 00:00:00 2001 From: Nikunj Hudka Date: Wed, 6 Nov 2024 13:10:30 -0400 Subject: [PATCH 07/13] constructor needs to be private and changed the API report Signed-off-by: Nikunj Hudka --- plugins/catalog-backend-module-azure/report.api.md | 11 ----------- .../src/providers/AzureBlobStorageEntityProvider.ts | 2 +- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/plugins/catalog-backend-module-azure/report.api.md b/plugins/catalog-backend-module-azure/report.api.md index f515349e94..19b79a6d0d 100644 --- a/plugins/catalog-backend-module-azure/report.api.md +++ b/plugins/catalog-backend-module-azure/report.api.md @@ -3,33 +3,22 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts -import { AzureBlobStorageIntergation } from '@backstage/integration'; import { BackendFeature } from '@backstage/backend-plugin-api'; import { CatalogProcessor } from '@backstage/plugin-catalog-node'; import { CatalogProcessorEmit } from '@backstage/plugin-catalog-node'; import { Config } from '@backstage/config'; -import { DefaultAzureCredentialsManager } from '@backstage/integration'; import { EntityProvider } from '@backstage/plugin-catalog-node'; import { EntityProviderConnection } from '@backstage/plugin-catalog-node'; import { LocationSpec } from '@backstage/plugin-catalog-common'; import { LoggerService } from '@backstage/backend-plugin-api'; import { SchedulerService } from '@backstage/backend-plugin-api'; import { SchedulerServiceTaskRunner } from '@backstage/backend-plugin-api'; -import { SchedulerServiceTaskScheduleDefinition } from '@backstage/backend-plugin-api'; import { ScmIntegrationRegistry } from '@backstage/integration'; // Warning: (ae-missing-release-tag) "AzureBlobStorageEntityProvider" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) export class AzureBlobStorageEntityProvider implements EntityProvider { - // Warning: (ae-forgotten-export) The symbol "AzureBlobStorageConfig" needs to be exported by the entry point index.d.ts - constructor( - config: AzureBlobStorageConfig, - integration: AzureBlobStorageIntergation, - credentialsProvider: DefaultAzureCredentialsManager, - logger: LoggerService, - schedule: SchedulerServiceTaskRunner, - ); // (undocumented) connect(connection: EntityProviderConnection): Promise; // (undocumented) diff --git a/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.ts b/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.ts index 47154f5c89..f30675cc9e 100644 --- a/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.ts +++ b/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.ts @@ -92,7 +92,7 @@ export class AzureBlobStorageEntityProvider implements EntityProvider { ); }); } - constructor( + private constructor( private readonly config: AzureBlobStorageConfig, private readonly integration: AzureBlobStorageIntergation, private readonly credentialsProvider: DefaultAzureCredentialsManager, From c105e55582dca4c1db3b580ef74e2a2ae495bd1b Mon Sep 17 00:00:00 2001 From: Nikunj Hudka Date: Wed, 13 Nov 2024 00:55:46 -0400 Subject: [PATCH 08/13] added handling logic for multiple integrations and added docs on how to configure the plugin; added api report Signed-off-by: Nikunj Hudka --- app-config.yaml | 1 + .../azure-blobStorage/discovery.md | 76 +++++++++++++++++++ .../azure-blobStorage/locations.md | 51 +++++++++++++ .../backend-defaults/report-urlReader.api.md | 31 ++++++++ .../lib/AzureBlobStorageUrlReader.ts | 24 +++--- packages/backend/package.json | 1 + packages/backend/src/index.ts | 1 + packages/integration/report.api.md | 75 ++++++++++++++++++ .../AzureBlobStorageIntegration.ts | 5 ++ .../DefaultAzureCredentialsProvider.ts | 70 +++++++++++------ .../integration/src/azureBlobStorage/types.ts | 6 ++ .../report.api.md | 4 +- .../AzureBlobStorageEntityProvider.ts | 14 +++- .../src/providers/config.ts | 2 + .../src/providers/types.ts | 1 + yarn.lock | 3 +- 16 files changed, 327 insertions(+), 38 deletions(-) create mode 100644 docs/integrations/azure-blobStorage/discovery.md create mode 100644 docs/integrations/azure-blobStorage/locations.md diff --git a/app-config.yaml b/app-config.yaml index 1486e8b3ec..456fd17974 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -257,6 +257,7 @@ catalog: - Location providers: azureBlob: + accountName: ${ACCOUNT_NAME} containerName: ${CONTAINER_NAME} schedule: # same options as in TaskScheduleDefinition # supports cron, ISO duration, "human duration" as used in code diff --git a/docs/integrations/azure-blobStorage/discovery.md b/docs/integrations/azure-blobStorage/discovery.md new file mode 100644 index 0000000000..a62333ff69 --- /dev/null +++ b/docs/integrations/azure-blobStorage/discovery.md @@ -0,0 +1,76 @@ +--- +id: discovery +title: Azure Blob Storage Discovery +sidebar_label: Discovery +# prettier-ignore +description: Automatically discovering catalog entities from an Azure Blob Storage account +--- + +:::info +This documentation is written for [the new backend system](../../backend-system/index.md) which is the default since Backstage [version 1.24](../../releases/v1.24.0.md). If you are still on the old backend system, you may want to read [its own article](./discovery--old.md) instead, and [consider migrating](../../backend-system/building-backends/08-migrating.md)! +::: + +The Azure Blob Storage account integration has a special entity provider for discovering catalog +entities located in a stroage account container. If you have a conatiner that contains multiple +catalog files, and you want to automatically discover them, you can use this +provider. The provider will crawl your Blob Storage account container and register entities +matching the configured path. This can be useful as an alternative to static +locations or manually adding things to the catalog. + +To use the entity provider, you'll need an Azure Blob Storage account integration +[set up](locations.md) with `accountName` and either `aadCredential`, `sasToken`, or `accountKey` + +At production deployments, you likely manage these with the permissions attached +to your instance. + +In your configuration, you add a provider config per bucket: + +```yaml +# app-config.yaml + +catalog: + providers: + azureBlob: + providerId: + accountName: ${ACCOUNT_NAME} + containerName: ${CONTAINER_NAME} + schedule: # same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } +``` + +For simple setups, you can omit the provider ID at the config +which has the same effect as using `default` for it. + +```yaml +# app-config.yaml + +catalog: + providers: + azureBlob: + accountName: ${ACCOUNT_NAME} + containerName: ${CONTAINER_NAME} + schedule: # same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } +``` + +As this provider is not one of the default providers, you will first need to install +the Azure catalog plugin: + +```bash title="From your Backstage root directory" +yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-azure +``` + +Then updated your backend by adding the following line: + +```ts title="packages/backend/src/index.ts" +backend.add(import('@backstage/plugin-catalog-backend')); +/* highlight-add-start */ +backend.add(import('@backstage/plugin-catalog-backend-module-azure')); +/* highlight-add-end */ +``` diff --git a/docs/integrations/azure-blobStorage/locations.md b/docs/integrations/azure-blobStorage/locations.md new file mode 100644 index 0000000000..38d8555c2e --- /dev/null +++ b/docs/integrations/azure-blobStorage/locations.md @@ -0,0 +1,51 @@ +--- +id: locations +sidebar_label: Locations +title: Azure Blob Storage account Locations +# prettier-ignore +description: Setting up an integration with Azure Blob Storage account +--- + +The Azure Blob Storage account integration supports loading catalog entities from an blob storage account container. +Entities can be added to +[static catalog configuration](../../features/software-catalog/configuration.md), +or registered with the +[catalog-import](https://github.com/backstage/backstage/tree/master/plugins/catalog-import) +plugin. + +## Configuration + +To use this integration, add configuration to your `app-config.yaml`: + +Using Azure active directory credentials: + +```yaml +integrations: + azureBlobStorage: + - accountName: ${ACCOUNT_NAME} # required + endpoint: ${CUSTOM_ENDPOINT} # custom endpoint will require either aadCredentials or sasToken + aadCredential: + clientId: ${CLIENT_ID} + tenantId: ${TENANT_ID} + clientSecret: ${CLIENT_SECRET} +``` + +Using Azure storage account SAS token: + +```yaml +integrations: + azureBlobStorage: + - accountName: ${ACCOUNT_NAME} # required + endpoint: ${CUSTOM_ENDPOINT} # custom endpoint will require either aadCredentials or sasToken + sasToken: ${SAS_TOKEN} +``` + +Using Azure storage account access key: + +```yaml +integrations: + azureBlobStorage: + - accountName: ${ACCOUNT_NAME} # required + endpoint: ${CUSTOM_ENDPOINT} # custom endpoint will require either aadCredentials or sasToken + accountKey: ${ACCOUNT_KEY} +``` diff --git a/packages/backend-defaults/report-urlReader.api.md b/packages/backend-defaults/report-urlReader.api.md index f8b8db2954..4813643d68 100644 --- a/packages/backend-defaults/report-urlReader.api.md +++ b/packages/backend-defaults/report-urlReader.api.md @@ -7,6 +7,8 @@ import { AwsCredentialsManager } from '@backstage/integration-aws-node'; import { AwsS3Integration } from '@backstage/integration'; +import { AzureBlobStorageIntergation } from '@backstage/integration'; +import { AzureCredentialsManager } from '@backstage/integration'; import { AzureDevOpsCredentialsProvider } from '@backstage/integration'; import { AzureIntegration } from '@backstage/integration'; import { BitbucketCloudIntegration } from '@backstage/integration'; @@ -60,6 +62,35 @@ export class AwsS3UrlReader implements UrlReaderService { toString(): string; } +// @public +export class AzureBlobStorageUrlReader implements UrlReaderService { + constructor( + credsManager: AzureCredentialsManager, + integration: AzureBlobStorageIntergation, + deps: { + treeResponseFactory: ReadTreeResponseFactory; + }, + ); + // (undocumented) + static factory: ReaderFactory; + // (undocumented) + read(url: string): Promise; + // (undocumented) + readTree( + url: string, + options?: UrlReaderServiceReadTreeOptions, + ): Promise; + // (undocumented) + readUrl( + url: string, + options?: UrlReaderServiceReadUrlOptions, + ): Promise; + // (undocumented) + search(): Promise; + // (undocumented) + toString(): string; +} + // @public export class AzureUrlReader implements UrlReaderService { constructor( diff --git a/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureBlobStorageUrlReader.ts b/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureBlobStorageUrlReader.ts index 4de5f79f4e..069b772a15 100644 --- a/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureBlobStorageUrlReader.ts +++ b/packages/backend-defaults/src/entrypoints/urlReader/lib/AzureBlobStorageUrlReader.ts @@ -54,6 +54,12 @@ export function parseUrl(url: string): { path: string; container: string } { return { path, container }; } + +/** + * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for Azure storage accounts urls. + * + * @public + */ export class AzureBlobStorageUrlReader implements UrlReaderService { static factory: ReaderFactory = ({ config, treeResponseFactory }) => { const integrations = ScmIntegrations.fromConfig(config); @@ -71,7 +77,9 @@ export class AzureBlobStorageUrlReader implements UrlReaderService { ); const predicate = (url: URL) => - url.host.endsWith(integrationConfig.config.host); + url.host.endsWith( + `${integrationConfig.config.accountName}.${integrationConfig.config.host}`, + ); return { reader, predicate }; }); }; @@ -141,16 +149,13 @@ export class AzureBlobStorageUrlReader implements UrlReaderService { const containerClient = await this.createContainerClient(container); const blobClient = containerClient.getBlobClient(path); - const abortController = new AbortController(); - const getBlobOptions: BlobDownloadOptions = { - abortSignal: abortController.signal, + abortSignal: options?.signal, conditions: { ...(etag && { ifNoneMatch: etag }), ...(lastModifiedAfter && { ifModifiedSince: lastModifiedAfter }), }, }; - options?.signal?.addEventListener('abort', () => abortController.abort()); const downloadBlockBlobResponse = await blobClient.download( 0, @@ -189,13 +194,14 @@ export class AzureBlobStorageUrlReader implements UrlReaderService { const blobs = containerClient.listBlobsFlat({ prefix: path }); const responses = []; - const abortController = new AbortController(); + for await (const blob of blobs) { const blobClient = containerClient.getBlobClient(blob.name); - options?.signal?.addEventListener('abort', () => - abortController.abort(), + const downloadBlockBlobResponse = await blobClient.download( + undefined, + undefined, + { abortSignal: options?.signal }, ); - const downloadBlockBlobResponse = await blobClient.download(); responses.push({ data: Readable.from( diff --git a/packages/backend/package.json b/packages/backend/package.json index 34de57574d..6b0ca47f98 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -39,6 +39,7 @@ "@backstage/plugin-auth-backend-module-guest-provider": "workspace:^", "@backstage/plugin-auth-node": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", + "@backstage/plugin-catalog-backend-module-azure": "workspace:^", "@backstage/plugin-catalog-backend-module-backstage-openapi": "workspace:^", "@backstage/plugin-catalog-backend-module-openapi": "workspace:^", "@backstage/plugin-catalog-backend-module-scaffolder-entity-model": "workspace:^", diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index b9130d452f..80537f1751 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -56,5 +56,6 @@ backend.add(searchLoader); backend.add(import('@backstage/plugin-techdocs-backend')); backend.add(import('@backstage/plugin-signals-backend')); backend.add(import('@backstage/plugin-notifications-backend')); +backend.add(import('@backstage/plugin-catalog-backend-module-azure')); backend.start(); diff --git a/packages/integration/report.api.md b/packages/integration/report.api.md index e9b67bb6c1..5129dbb7c2 100644 --- a/packages/integration/report.api.md +++ b/packages/integration/report.api.md @@ -3,9 +3,12 @@ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/). ```ts +import { AnonymousCredential } from '@azure/storage-blob'; import { Config } from '@backstage/config'; import { ConsumedResponse } from '@backstage/errors'; import { RestEndpointMethodTypes } from '@octokit/rest'; +import { StorageSharedKeyCredential } from '@azure/storage-blob'; +import { TokenCredential } from '@azure/identity'; // @public export class AwsCodeCommitIntegration implements ScmIntegration { @@ -70,6 +73,43 @@ export type AwsS3IntegrationConfig = { externalId?: string; }; +// @public +export type AzureBlobStorageIntegrationConfig = { + accountName?: string; + accountKey?: string; + sasToken?: string; + connectionString?: string; + endpointSuffix?: string; + host: string; + endpoint?: string; + aadCredential?: { + clientId: string; + tenantId: string; + clientSecret: string; + }; +}; + +// @public +export class AzureBlobStorageIntergation implements ScmIntegration { + constructor(integrationConfig: AzureBlobStorageIntegrationConfig); + // (undocumented) + get config(): AzureBlobStorageIntegrationConfig; + // (undocumented) + static factory: ScmIntegrationsFactory; + // (undocumented) + resolveEditUrl(url: string): string; + // (undocumented) + resolveUrl(options: { + url: string; + base: string; + lineNumber?: number | undefined; + }): string; + // (undocumented) + get title(): string; + // (undocumented) + get type(): string; +} + // @public export type AzureClientSecretCredential = AzureCredentialBase & { kind: 'ClientSecret'; @@ -84,6 +124,16 @@ export type AzureCredentialBase = { organizations?: string[]; }; +// @public +export interface AzureCredentialsManager { + // (undocumented) + getCredentials( + accountName: string, + ): Promise< + TokenCredential | StorageSharedKeyCredential | AnonymousCredential + >; +} + // @public export type AzureDevOpsCredential = | AzureClientSecretCredential @@ -257,6 +307,15 @@ export function buildGerritGitilesArchiveUrl( filePath: string, ): string; +// @public +export class DefaultAzureCredentialsManager implements AzureCredentialsManager { + static fromIntegrations( + integrations: ScmIntegrationRegistry, + ): DefaultAzureCredentialsManager; + // (undocumented) + getCredentials(accountName: string): Promise; +} + // @public export class DefaultAzureDevOpsCredentialsProvider implements AzureDevOpsCredentialsProvider @@ -723,6 +782,8 @@ export interface IntegrationsByType { awsS3: ScmIntegrationsGroup; // (undocumented) azure: ScmIntegrationsGroup; + // (undocumented) + azureBlobStorage: ScmIntegrationsGroup; // @deprecated (undocumented) bitbucket: ScmIntegrationsGroup; // (undocumented) @@ -826,6 +887,16 @@ export function readAwsS3IntegrationConfigs( configs: Config[], ): AwsS3IntegrationConfig[]; +// @public +export function readAzureBlobStorageIntegrationConfig( + config: Config, +): AzureBlobStorageIntegrationConfig; + +// @public +export function readAzureBlobStorageIntegrationConfigs( + configs: Config[], +): AzureBlobStorageIntegrationConfig[]; + // @public export function readAzureIntegrationConfig( config: Config, @@ -940,6 +1011,8 @@ export interface ScmIntegrationRegistry awsS3: ScmIntegrationsGroup; // (undocumented) azure: ScmIntegrationsGroup; + // (undocumented) + azureBlobStorage: ScmIntegrationsGroup; // @deprecated (undocumented) bitbucket: ScmIntegrationsGroup; // (undocumented) @@ -973,6 +1046,8 @@ export class ScmIntegrations implements ScmIntegrationRegistry { get awsS3(): ScmIntegrationsGroup; // (undocumented) get azure(): ScmIntegrationsGroup; + // (undocumented) + get azureBlobStorage(): ScmIntegrationsGroup; // @deprecated (undocumented) get bitbucket(): ScmIntegrationsGroup; // (undocumented) diff --git a/packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.ts b/packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.ts index 01e6bbc317..27061f7d9d 100644 --- a/packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.ts +++ b/packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.ts @@ -21,6 +21,11 @@ import { readAzureBlobStorageIntegrationConfigs, } from './config'; +/** + * Microsoft Azure Blob storage based integration. + * + * @public + */ export class AzureBlobStorageIntergation implements ScmIntegration { static factory: ScmIntegrationsFactory = ({ config, diff --git a/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsProvider.ts b/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsProvider.ts index 8971e560ad..3861329300 100644 --- a/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsProvider.ts +++ b/packages/integration/src/azureBlobStorage/DefaultAzureCredentialsProvider.ts @@ -23,26 +23,58 @@ import { AzureBlobStorageIntegrationConfig } from './config'; import { AzureCredentialsManager } from './types'; import { ScmIntegrationRegistry } from '../registry'; +/** + * Default implementation of AzureCredentialsManager that supports multiple Azure Blob Storage integrations. + * @public + */ export class DefaultAzureCredentialsManager implements AzureCredentialsManager { - private config: AzureBlobStorageIntegrationConfig; private cachedCredentials: Map; - constructor(config: AzureBlobStorageIntegrationConfig) { - this.config = config; + private constructor( + private readonly configProviders: Map< + string, + AzureBlobStorageIntegrationConfig + >, + ) { this.cachedCredentials = new Map(); } /** - * Creates an instance of DefaultAzureCredentialsManager from a Backstage Config. + * Creates an instance of DefaultAzureCredentialsManager from a Backstage integration registry. */ static fromIntegrations( - integration: ScmIntegrationRegistry, + integrations: ScmIntegrationRegistry, ): DefaultAzureCredentialsManager { - const azureConfig = integration.azureBlobStorage.list().length - ? integration.azureBlobStorage.list()[0].config - : { host: 'blob.core.windows.net' }; // Default to Azure Blob Storage host if no config found + const configProviders = integrations.azureBlobStorage + .list() + .reduce((acc, integration) => { + acc.set( + integration.config.accountName || 'default', + integration.config, + ); + return acc; + }, new Map()); - return new DefaultAzureCredentialsManager(azureConfig); + return new DefaultAzureCredentialsManager(configProviders); + } + + private createCredential( + config: AzureBlobStorageIntegrationConfig, + ): TokenCredential { + if ( + config.aadCredential && + config.aadCredential.clientId && + config.aadCredential.clientSecret && + config.aadCredential.tenantId + ) { + return new ClientSecretCredential( + config.aadCredential.tenantId, + config.aadCredential.clientId, + config.aadCredential.clientSecret, + ); + } + + return new DefaultAzureCredential(); } async getCredentials(accountName: string): Promise { @@ -50,23 +82,13 @@ export class DefaultAzureCredentialsManager implements AzureCredentialsManager { return this.cachedCredentials.get(accountName)!; } - let credential: TokenCredential; - - if ( - this.config.aadCredential && - this.config.aadCredential.clientId && - this.config.aadCredential.clientSecret && - this.config.aadCredential.tenantId - ) { - credential = new ClientSecretCredential( - this.config.aadCredential.tenantId, - this.config.aadCredential.clientId, - this.config.aadCredential.clientSecret, - ); - } else { - credential = new DefaultAzureCredential(); + const config = this.configProviders.get(accountName); + if (!config) { + throw new Error(`No configuration found for account: ${accountName}`); } + const credential = this.createCredential(config); + // Cache the credentials for future use this.cachedCredentials.set(accountName, credential); diff --git a/packages/integration/src/azureBlobStorage/types.ts b/packages/integration/src/azureBlobStorage/types.ts index 99bcbec57f..90c2725d05 100644 --- a/packages/integration/src/azureBlobStorage/types.ts +++ b/packages/integration/src/azureBlobStorage/types.ts @@ -20,6 +20,12 @@ import { AnonymousCredential, } from '@azure/storage-blob'; +/** + * This allows implementations to be provided to retrieve Azure Storage accounts credentials. + * + * @public + * + */ export interface AzureCredentialsManager { getCredentials( accountName: string, diff --git a/plugins/catalog-backend-module-azure/report.api.md b/plugins/catalog-backend-module-azure/report.api.md index 19b79a6d0d..252513aa0a 100644 --- a/plugins/catalog-backend-module-azure/report.api.md +++ b/plugins/catalog-backend-module-azure/report.api.md @@ -15,9 +15,7 @@ import { SchedulerService } from '@backstage/backend-plugin-api'; import { SchedulerServiceTaskRunner } from '@backstage/backend-plugin-api'; import { ScmIntegrationRegistry } from '@backstage/integration'; -// Warning: (ae-missing-release-tag) "AzureBlobStorageEntityProvider" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal) -// -// @public (undocumented) +// @public export class AzureBlobStorageEntityProvider implements EntityProvider { // (undocumented) connect(connection: EntityProviderConnection): Promise; diff --git a/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.ts b/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.ts index f30675cc9e..3e2d023e76 100644 --- a/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.ts +++ b/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.ts @@ -42,6 +42,13 @@ import { import { TokenCredential } from '@azure/identity'; import { AzureBlobStorageConfig } from './types'; +/** + * Provider which discovers catalog files within an Azure Storage accounts. + * + * Use `AzureBlobStorageEntityProvider.fromConfig(...)` to create instances. + * + * @public + */ export class AzureBlobStorageEntityProvider implements EntityProvider { private readonly logger: LoggerService; private connection?: EntityProviderConnection; @@ -66,7 +73,12 @@ export class AzureBlobStorageEntityProvider implements EntityProvider { } return providerConfigs.map(providerConfig => { - const integration = scmIntegration.azureBlobStorage.list()[0]; + const integration = scmIntegration.azureBlobStorage + .list() + .filter( + azureIntegration => + azureIntegration.config.accountName === providerConfig.accountName, + )[0]; if (!integration) { throw new Error( `There is no Azure blob storage integration for host. Please add a configuration entry for it under integrations.azure`, diff --git a/plugins/catalog-backend-module-azure/src/providers/config.ts b/plugins/catalog-backend-module-azure/src/providers/config.ts index d082726b14..8ec1cbfb1b 100644 --- a/plugins/catalog-backend-module-azure/src/providers/config.ts +++ b/plugins/catalog-backend-module-azure/src/providers/config.ts @@ -98,6 +98,7 @@ function readAzureBlobStorageConfig( config: Config, ): AzureBlobStorageConfig { const containerName = config.getString('containerName'); + const accountName = config.getString('accountName'); const schedule = config.has('schedule') ? readSchedulerServiceTaskScheduleDefinitionFromConfig( config.getConfig('schedule'), @@ -107,6 +108,7 @@ function readAzureBlobStorageConfig( return { id, containerName, + accountName, schedule, }; } diff --git a/plugins/catalog-backend-module-azure/src/providers/types.ts b/plugins/catalog-backend-module-azure/src/providers/types.ts index eb16dbd6ea..e296d50ca7 100644 --- a/plugins/catalog-backend-module-azure/src/providers/types.ts +++ b/plugins/catalog-backend-module-azure/src/providers/types.ts @@ -30,5 +30,6 @@ export type AzureDevOpsConfig = { export type AzureBlobStorageConfig = { id: string; containerName: string; + accountName: string; schedule?: SchedulerServiceTaskScheduleDefinition; }; diff --git a/yarn.lock b/yarn.lock index a76d05d9db..7a68420f41 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5395,7 +5395,7 @@ __metadata: languageName: unknown linkType: soft -"@backstage/plugin-catalog-backend-module-azure@workspace:plugins/catalog-backend-module-azure": +"@backstage/plugin-catalog-backend-module-azure@workspace:^, @backstage/plugin-catalog-backend-module-azure@workspace:plugins/catalog-backend-module-azure": version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-backend-module-azure@workspace:plugins/catalog-backend-module-azure" dependencies: @@ -27070,6 +27070,7 @@ __metadata: "@backstage/plugin-auth-backend-module-guest-provider": "workspace:^" "@backstage/plugin-auth-node": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" + "@backstage/plugin-catalog-backend-module-azure": "workspace:^" "@backstage/plugin-catalog-backend-module-backstage-openapi": "workspace:^" "@backstage/plugin-catalog-backend-module-openapi": "workspace:^" "@backstage/plugin-catalog-backend-module-scaffolder-entity-model": "workspace:^" From bb17475348d641344f00cfbf6e20ba58f574a126 Mon Sep 17 00:00:00 2001 From: Nikunj Hudka Date: Wed, 13 Nov 2024 01:13:13 -0400 Subject: [PATCH 09/13] chore: add azure blob storage discovery docs for old backend system Signed-off-by: Nikunj Hudka --- .../azure-blobStorage/discovery--old.md | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 docs/integrations/azure-blobStorage/discovery--old.md diff --git a/docs/integrations/azure-blobStorage/discovery--old.md b/docs/integrations/azure-blobStorage/discovery--old.md new file mode 100644 index 0000000000..bd8b663dc9 --- /dev/null +++ b/docs/integrations/azure-blobStorage/discovery--old.md @@ -0,0 +1,91 @@ +--- +id: discovery--old +title: Azure Blob Storage Discovery +sidebar_label: Discovery +# prettier-ignore +description: Automatically discovering catalog entities from an Azure Blob Storage account +--- + +:::info +This documentation is written for [the new backend system](../../backend-system/index.md) which is the default since Backstage [version 1.24](../../releases/v1.24.0.md). If have migrated to the new backend system, you may want to read [its own article](./discovery.md) instead. Otherwise, [consider migrating](../../backend-system/building-backends/08-migrating.md)! +::: + +The Azure Blob Storage account integration has a special entity provider for discovering catalog +entities located in a stroage account container. If you have a conatiner that contains multiple +catalog files, and you want to automatically discover them, you can use this +provider. The provider will crawl your Blob Storage account container and register entities +matching the configured path. This can be useful as an alternative to static +locations or manually adding things to the catalog. + +To use the entity provider, you'll need an Azure Blob Storage account integration +[set up](locations.md) with `accountName` and either `aadCredential`, `sasToken`, or `accountKey` + +At production deployments, you likely manage these with the permissions attached +to your instance. + +In your configuration, you add a provider config per bucket: + +```yaml +# app-config.yaml + +catalog: + providers: + azureBlob: + providerId: + accountName: ${ACCOUNT_NAME} + containerName: ${CONTAINER_NAME} + schedule: # same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } +``` + +For simple setups, you can omit the provider ID at the config +which has the same effect as using `default` for it. + +```yaml +# app-config.yaml + +catalog: + providers: + azureBlob: + accountName: ${ACCOUNT_NAME} + containerName: ${CONTAINER_NAME} + schedule: # same options as in TaskScheduleDefinition + # supports cron, ISO duration, "human duration" as used in code + frequency: { minutes: 30 } + # supports ISO duration, "human duration" as used in code + timeout: { minutes: 3 } +``` + +As this provider is not one of the default providers, you will first need to install +the Azure catalog plugin: + +```bash title="From your Backstage root directory" +yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-azure +``` + +Once you've done that, you'll also need to add the segment below to `packages/backend/src/plugins/catalog.ts`: + +```ts title="packages/backend/src/plugins/catalog.ts" +/* highlight-add-next-line */ +import { AzureDevOpsEntityProvider } from '@backstage/plugin-catalog-backend-module-azure'; + +const builder = await CatalogBuilder.create(env); +/** ... other processors and/or providers ... */ +/* highlight-add-start */ +builder.addEntityProvider( + AzureBlobStorageEntityProvider.fromConfig(env.config, { + logger: env.logger, + // optional: alternatively, use scheduler with schedule defined in app-config.yaml + schedule: env.scheduler.createScheduledTaskRunner({ + frequency: { minutes: 30 }, + timeout: { minutes: 3 }, + }), + // optional: alternatively, use schedule + scheduler: env.scheduler, + }), +); +/* highlight-add-end */ +``` From 38a1506733a213350903a361981269999698d9b8 Mon Sep 17 00:00:00 2001 From: Nikunj Hudka Date: Wed, 13 Nov 2024 02:10:59 -0400 Subject: [PATCH 10/13] fix: changes in testcase to accomodate accountName in config provider Signed-off-by: Nikunj Hudka --- .../catalogModuleAzureDevOpsEntityProvider.test.ts | 9 +++++++++ .../providers/AzureBlobStorageEntityProvider.test.ts | 6 ++++-- .../src/providers/AzureBlobStorageEntityProvider.ts | 11 ++++++----- .../src/providers/config.test.ts | 5 +++++ 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/plugins/catalog-backend-module-azure/src/module/catalogModuleAzureDevOpsEntityProvider.test.ts b/plugins/catalog-backend-module-azure/src/module/catalogModuleAzureDevOpsEntityProvider.test.ts index e47635de91..9c5d1baef7 100644 --- a/plugins/catalog-backend-module-azure/src/module/catalogModuleAzureDevOpsEntityProvider.test.ts +++ b/plugins/catalog-backend-module-azure/src/module/catalogModuleAzureDevOpsEntityProvider.test.ts @@ -95,10 +95,19 @@ describe('catalogModuleAzureDevOpsEntityProvider', () => { }); const config = { + integrations: { + azureBlobStorage: [ + { + accountName: 'test', + accountKey: 'test', + }, + ], + }, catalog: { providers: { azureBlob: { containerName: 'test', + accountName: 'test', schedule: { frequency: 'P1M', timeout: 'PT3M', diff --git a/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.test.ts b/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.test.ts index 515d2ee7d0..62b635d26e 100644 --- a/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.test.ts +++ b/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.test.ts @@ -61,7 +61,7 @@ const logger = mockServices.logger.mock(); describe('AzureBlobStorageEntityProvider', () => { const containerName = 'container-1'; - + const accountName = 'myaccount'; const expectMutation = async ( providerId: string, providerConfig: object, @@ -157,6 +157,7 @@ describe('AzureBlobStorageEntityProvider', () => { 'staticContainer', { containerName, + accountName, }, 'https://myaccount.blob.core.windows.net/container-1/', { @@ -171,6 +172,7 @@ describe('AzureBlobStorageEntityProvider', () => { 'staticContainerNoPrefix', { containerName, + accountName, schedule: { frequency: { minutes: 30 }, timeout: { minutes: 3 }, @@ -190,9 +192,9 @@ describe('AzureBlobStorageEntityProvider', () => { providers: { azureBlob: { test: { + accountName: 'myaccount', containerName: 'container-1', prefix: 'sub/dir/', - accountName: 'myaccount', }, }, }, diff --git a/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.ts b/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.ts index 3e2d023e76..ae2fbcb3ff 100644 --- a/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.ts +++ b/plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.ts @@ -79,11 +79,6 @@ export class AzureBlobStorageEntityProvider implements EntityProvider { azureIntegration => azureIntegration.config.accountName === providerConfig.accountName, )[0]; - if (!integration) { - throw new Error( - `There is no Azure blob storage integration for host. Please add a configuration entry for it under integrations.azure`, - ); - } if (!options.schedule && !providerConfig.schedule) { throw new Error( @@ -91,6 +86,12 @@ export class AzureBlobStorageEntityProvider implements EntityProvider { ); } + if (!integration) { + throw new Error( + `There is no Azure blob storage integration for account. Please add a configuration entry for it under integrations.azureBlobStorage`, + ); + } + const taskRunner = options.schedule ?? options.scheduler!.createScheduledTaskRunner(providerConfig.schedule!); diff --git a/plugins/catalog-backend-module-azure/src/providers/config.test.ts b/plugins/catalog-backend-module-azure/src/providers/config.test.ts index 21c4ece78c..78ed559749 100644 --- a/plugins/catalog-backend-module-azure/src/providers/config.test.ts +++ b/plugins/catalog-backend-module-azure/src/providers/config.test.ts @@ -110,6 +110,7 @@ describe('readAzureDevOpsConfigs', () => { describe('readAzureBlobStorageConfigs', () => { it('reads single and multiple Azure Blob Storage provider configs', () => { const provider1 = { + accountName: 'account-1', containerName: 'container-1', schedule: { frequency: 'PT30M', @@ -119,6 +120,7 @@ describe('readAzureBlobStorageConfigs', () => { }, }; const provider2 = { + accountName: 'account-1', containerName: 'container-2', }; @@ -148,6 +150,7 @@ describe('readAzureBlobStorageConfigs', () => { expect(actualSingle).toHaveLength(1); expect(actualSingle[0]).toEqual({ id: 'default', + accountName: 'account-1', containerName: 'container-2', schedule: undefined, // no schedule provided in this case }); @@ -159,6 +162,7 @@ describe('readAzureBlobStorageConfigs', () => { expect(actualMulti).toHaveLength(2); expect(actualMulti[0]).toEqual({ id: 'provider1', + accountName: 'account-1', containerName: 'container-1', schedule: { ...provider1.schedule, @@ -167,6 +171,7 @@ describe('readAzureBlobStorageConfigs', () => { }); expect(actualMulti[1]).toEqual({ id: 'provider2', + accountName: 'account-1', containerName: 'container-2', schedule: undefined, // no schedule provided }); From acba6c5f4f8c855d1c840c79ace0d8a595397b14 Mon Sep 17 00:00:00 2001 From: Nikunj Hudka Date: Thu, 21 Nov 2024 22:28:30 -0400 Subject: [PATCH 11/13] chore: docs spelling mistake correction Signed-off-by: Nikunj Hudka --- docs/integrations/azure-blobStorage/discovery--old.md | 2 +- docs/integrations/azure-blobStorage/discovery.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/integrations/azure-blobStorage/discovery--old.md b/docs/integrations/azure-blobStorage/discovery--old.md index bd8b663dc9..5ff0ca5bdd 100644 --- a/docs/integrations/azure-blobStorage/discovery--old.md +++ b/docs/integrations/azure-blobStorage/discovery--old.md @@ -11,7 +11,7 @@ This documentation is written for [the new backend system](../../backend-system/ ::: The Azure Blob Storage account integration has a special entity provider for discovering catalog -entities located in a stroage account container. If you have a conatiner that contains multiple +entities located in a storage account container. If you have a container that contains multiple catalog files, and you want to automatically discover them, you can use this provider. The provider will crawl your Blob Storage account container and register entities matching the configured path. This can be useful as an alternative to static diff --git a/docs/integrations/azure-blobStorage/discovery.md b/docs/integrations/azure-blobStorage/discovery.md index a62333ff69..db6b7c49f1 100644 --- a/docs/integrations/azure-blobStorage/discovery.md +++ b/docs/integrations/azure-blobStorage/discovery.md @@ -11,7 +11,7 @@ This documentation is written for [the new backend system](../../backend-system/ ::: The Azure Blob Storage account integration has a special entity provider for discovering catalog -entities located in a stroage account container. If you have a conatiner that contains multiple +entities located in a storage account container. If you have a container that contains multiple catalog files, and you want to automatically discover them, you can use this provider. The provider will crawl your Blob Storage account container and register entities matching the configured path. This can be useful as an alternative to static From 019e0189d6620830a48eead81e9da7f73df75e44 Mon Sep 17 00:00:00 2001 From: Nikunj Hudka Date: Fri, 22 Nov 2024 17:17:00 -0400 Subject: [PATCH 12/13] add docs to microsite and do not export entity provider Signed-off-by: Nikunj Hudka --- .../azure-blobStorage/discovery--old.md | 91 ------------------- .../azure-blobStorage/discovery.md | 2 +- microsite/sidebars.js | 8 ++ packages/backend/package.json | 1 - packages/backend/src/index.ts | 1 - .../report.api.md | 19 ---- .../catalog-backend-module-azure/src/index.ts | 1 - 7 files changed, 9 insertions(+), 114 deletions(-) delete mode 100644 docs/integrations/azure-blobStorage/discovery--old.md diff --git a/docs/integrations/azure-blobStorage/discovery--old.md b/docs/integrations/azure-blobStorage/discovery--old.md deleted file mode 100644 index 5ff0ca5bdd..0000000000 --- a/docs/integrations/azure-blobStorage/discovery--old.md +++ /dev/null @@ -1,91 +0,0 @@ ---- -id: discovery--old -title: Azure Blob Storage Discovery -sidebar_label: Discovery -# prettier-ignore -description: Automatically discovering catalog entities from an Azure Blob Storage account ---- - -:::info -This documentation is written for [the new backend system](../../backend-system/index.md) which is the default since Backstage [version 1.24](../../releases/v1.24.0.md). If have migrated to the new backend system, you may want to read [its own article](./discovery.md) instead. Otherwise, [consider migrating](../../backend-system/building-backends/08-migrating.md)! -::: - -The Azure Blob Storage account integration has a special entity provider for discovering catalog -entities located in a storage account container. If you have a container that contains multiple -catalog files, and you want to automatically discover them, you can use this -provider. The provider will crawl your Blob Storage account container and register entities -matching the configured path. This can be useful as an alternative to static -locations or manually adding things to the catalog. - -To use the entity provider, you'll need an Azure Blob Storage account integration -[set up](locations.md) with `accountName` and either `aadCredential`, `sasToken`, or `accountKey` - -At production deployments, you likely manage these with the permissions attached -to your instance. - -In your configuration, you add a provider config per bucket: - -```yaml -# app-config.yaml - -catalog: - providers: - azureBlob: - providerId: - accountName: ${ACCOUNT_NAME} - containerName: ${CONTAINER_NAME} - schedule: # same options as in TaskScheduleDefinition - # supports cron, ISO duration, "human duration" as used in code - frequency: { minutes: 30 } - # supports ISO duration, "human duration" as used in code - timeout: { minutes: 3 } -``` - -For simple setups, you can omit the provider ID at the config -which has the same effect as using `default` for it. - -```yaml -# app-config.yaml - -catalog: - providers: - azureBlob: - accountName: ${ACCOUNT_NAME} - containerName: ${CONTAINER_NAME} - schedule: # same options as in TaskScheduleDefinition - # supports cron, ISO duration, "human duration" as used in code - frequency: { minutes: 30 } - # supports ISO duration, "human duration" as used in code - timeout: { minutes: 3 } -``` - -As this provider is not one of the default providers, you will first need to install -the Azure catalog plugin: - -```bash title="From your Backstage root directory" -yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-azure -``` - -Once you've done that, you'll also need to add the segment below to `packages/backend/src/plugins/catalog.ts`: - -```ts title="packages/backend/src/plugins/catalog.ts" -/* highlight-add-next-line */ -import { AzureDevOpsEntityProvider } from '@backstage/plugin-catalog-backend-module-azure'; - -const builder = await CatalogBuilder.create(env); -/** ... other processors and/or providers ... */ -/* highlight-add-start */ -builder.addEntityProvider( - AzureBlobStorageEntityProvider.fromConfig(env.config, { - logger: env.logger, - // optional: alternatively, use scheduler with schedule defined in app-config.yaml - schedule: env.scheduler.createScheduledTaskRunner({ - frequency: { minutes: 30 }, - timeout: { minutes: 3 }, - }), - // optional: alternatively, use schedule - scheduler: env.scheduler, - }), -); -/* highlight-add-end */ -``` diff --git a/docs/integrations/azure-blobStorage/discovery.md b/docs/integrations/azure-blobStorage/discovery.md index db6b7c49f1..32612daf75 100644 --- a/docs/integrations/azure-blobStorage/discovery.md +++ b/docs/integrations/azure-blobStorage/discovery.md @@ -7,7 +7,7 @@ description: Automatically discovering catalog entities from an Azure Blob Stora --- :::info -This documentation is written for [the new backend system](../../backend-system/index.md) which is the default since Backstage [version 1.24](../../releases/v1.24.0.md). If you are still on the old backend system, you may want to read [its own article](./discovery--old.md) instead, and [consider migrating](../../backend-system/building-backends/08-migrating.md)! +This documentation is written for [the new backend system](../../backend-system/index.md) which is the default since Backstage [version 1.24](../../releases/v1.24.0.md). ::: The Azure Blob Storage account integration has a special entity provider for discovering catalog diff --git a/microsite/sidebars.js b/microsite/sidebars.js index 871cec0a0f..ee429bc693 100644 --- a/microsite/sidebars.js +++ b/microsite/sidebars.js @@ -138,6 +138,14 @@ module.exports = { 'integrations/aws-s3/discovery', ], }, + { + type: 'category', + label: 'Azure Blob Storage', + items: [ + 'integrations/azure-blobStorage/locations', + 'integrations/azure-blobStorage/discovery', + ], + }, { type: 'category', label: 'Azure', diff --git a/packages/backend/package.json b/packages/backend/package.json index 6b0ca47f98..34de57574d 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -39,7 +39,6 @@ "@backstage/plugin-auth-backend-module-guest-provider": "workspace:^", "@backstage/plugin-auth-node": "workspace:^", "@backstage/plugin-catalog-backend": "workspace:^", - "@backstage/plugin-catalog-backend-module-azure": "workspace:^", "@backstage/plugin-catalog-backend-module-backstage-openapi": "workspace:^", "@backstage/plugin-catalog-backend-module-openapi": "workspace:^", "@backstage/plugin-catalog-backend-module-scaffolder-entity-model": "workspace:^", diff --git a/packages/backend/src/index.ts b/packages/backend/src/index.ts index 80537f1751..b9130d452f 100644 --- a/packages/backend/src/index.ts +++ b/packages/backend/src/index.ts @@ -56,6 +56,5 @@ backend.add(searchLoader); backend.add(import('@backstage/plugin-techdocs-backend')); backend.add(import('@backstage/plugin-signals-backend')); backend.add(import('@backstage/plugin-notifications-backend')); -backend.add(import('@backstage/plugin-catalog-backend-module-azure')); backend.start(); diff --git a/plugins/catalog-backend-module-azure/report.api.md b/plugins/catalog-backend-module-azure/report.api.md index 252513aa0a..60b7ee135b 100644 --- a/plugins/catalog-backend-module-azure/report.api.md +++ b/plugins/catalog-backend-module-azure/report.api.md @@ -15,25 +15,6 @@ import { SchedulerService } from '@backstage/backend-plugin-api'; import { SchedulerServiceTaskRunner } from '@backstage/backend-plugin-api'; import { ScmIntegrationRegistry } from '@backstage/integration'; -// @public -export class AzureBlobStorageEntityProvider implements EntityProvider { - // (undocumented) - connect(connection: EntityProviderConnection): Promise; - // (undocumented) - static fromConfig( - configRoot: Config, - options: { - logger: LoggerService; - schedule?: SchedulerServiceTaskRunner; - scheduler?: SchedulerService; - }, - ): AzureBlobStorageEntityProvider[]; - // (undocumented) - getProviderName(): string; - // (undocumented) - refresh(logger: LoggerService): Promise; -} - // @public export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor { constructor(options: { diff --git a/plugins/catalog-backend-module-azure/src/index.ts b/plugins/catalog-backend-module-azure/src/index.ts index da1aaef30f..dbbdff67ce 100644 --- a/plugins/catalog-backend-module-azure/src/index.ts +++ b/plugins/catalog-backend-module-azure/src/index.ts @@ -23,4 +23,3 @@ export { default } from './module'; export { AzureDevOpsDiscoveryProcessor } from './processors'; export { AzureDevOpsEntityProvider } from './providers'; -export { AzureBlobStorageEntityProvider } from './providers'; From 104271d49309a67b88b87b7a6acd3a140f57ec58 Mon Sep 17 00:00:00 2001 From: Nikunj Hudka Date: Fri, 22 Nov 2024 17:20:57 -0400 Subject: [PATCH 13/13] chore: yarn lock update Signed-off-by: Nikunj Hudka --- yarn.lock | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/yarn.lock b/yarn.lock index 7a68420f41..a76d05d9db 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5395,7 +5395,7 @@ __metadata: languageName: unknown linkType: soft -"@backstage/plugin-catalog-backend-module-azure@workspace:^, @backstage/plugin-catalog-backend-module-azure@workspace:plugins/catalog-backend-module-azure": +"@backstage/plugin-catalog-backend-module-azure@workspace:plugins/catalog-backend-module-azure": version: 0.0.0-use.local resolution: "@backstage/plugin-catalog-backend-module-azure@workspace:plugins/catalog-backend-module-azure" dependencies: @@ -27070,7 +27070,6 @@ __metadata: "@backstage/plugin-auth-backend-module-guest-provider": "workspace:^" "@backstage/plugin-auth-node": "workspace:^" "@backstage/plugin-catalog-backend": "workspace:^" - "@backstage/plugin-catalog-backend-module-azure": "workspace:^" "@backstage/plugin-catalog-backend-module-backstage-openapi": "workspace:^" "@backstage/plugin-catalog-backend-module-openapi": "workspace:^" "@backstage/plugin-catalog-backend-module-scaffolder-entity-model": "workspace:^"