added handling logic for multiple integrations and added docs on how to configure the plugin; added api report

Signed-off-by: Nikunj Hudka <nikunjhudka123@gmail.com>

app-config.yaml

@@ -257,6 +257,7 @@ catalog:
         - Location
   providers:
     azureBlob:
+      accountName: ${ACCOUNT_NAME}
       containerName: ${CONTAINER_NAME}
       schedule: # same options as in TaskScheduleDefinition
         # supports cron, ISO duration, "human duration" as used in code

docs/integrations/azure-blobStorage/discovery.md

@@ -0,0 +1,76 @@
+---
+id: discovery
+title: Azure Blob Storage Discovery
+sidebar_label: Discovery
+# prettier-ignore
+description: Automatically discovering catalog entities from an Azure Blob Storage account
+---
+
+:::info
+This documentation is written for [the new backend system](../../backend-system/index.md) which is the default since Backstage [version 1.24](../../releases/v1.24.0.md). If you are still on the old backend system, you may want to read [its own article](./discovery--old.md) instead, and [consider migrating](../../backend-system/building-backends/08-migrating.md)!
+:::
+
+The Azure Blob Storage account integration has a special entity provider for discovering catalog
+entities located in a stroage account container. If you have a conatiner that contains multiple
+catalog files, and you want to automatically discover them, you can use this
+provider. The provider will crawl your Blob Storage account container and register entities
+matching the configured path. This can be useful as an alternative to static
+locations or manually adding things to the catalog.
+
+To use the entity provider, you'll need an Azure Blob Storage account integration
+[set up](locations.md) with `accountName` and either `aadCredential`, `sasToken`, or `accountKey`
+
+At production deployments, you likely manage these with the permissions attached
+to your instance.
+
+In your configuration, you add a provider config per bucket:
+
+```yaml
+# app-config.yaml
+
+catalog:
+  providers:
+    azureBlob:
+      providerId:
+        accountName: ${ACCOUNT_NAME}
+        containerName: ${CONTAINER_NAME}
+        schedule: # same options as in TaskScheduleDefinition
+          # supports cron, ISO duration, "human duration" as used in code
+          frequency: { minutes: 30 }
+          # supports ISO duration, "human duration" as used in code
+          timeout: { minutes: 3 }
+```
+
+For simple setups, you can omit the provider ID at the config
+which has the same effect as using `default` for it.
+
+```yaml
+# app-config.yaml
+
+catalog:
+  providers:
+    azureBlob:
+      accountName: ${ACCOUNT_NAME}
+      containerName: ${CONTAINER_NAME}
+      schedule: # same options as in TaskScheduleDefinition
+        # supports cron, ISO duration, "human duration" as used in code
+        frequency: { minutes: 30 }
+        # supports ISO duration, "human duration" as used in code
+        timeout: { minutes: 3 }
+```
+
+As this provider is not one of the default providers, you will first need to install
+the Azure catalog plugin:
+
+```bash title="From your Backstage root directory"
+yarn --cwd packages/backend add @backstage/plugin-catalog-backend-module-azure
+```
+
+Then updated your backend by adding the following line:
+
+```ts title="packages/backend/src/index.ts"
+backend.add(import('@backstage/plugin-catalog-backend'));
+/* highlight-add-start */
+backend.add(import('@backstage/plugin-catalog-backend-module-azure'));
+/* highlight-add-end */
+```

docs/integrations/azure-blobStorage/locations.md

@@ -0,0 +1,51 @@
+---
+id: locations
+sidebar_label: Locations
+title: Azure Blob Storage account Locations
+# prettier-ignore
+description: Setting up an integration with Azure Blob Storage account
+---
+
+The Azure Blob Storage account integration supports loading catalog entities from an blob storage account container.
+Entities can be added to
+[static catalog configuration](../../features/software-catalog/configuration.md),
+or registered with the
+[catalog-import](https://github.com/backstage/backstage/tree/master/plugins/catalog-import)
+plugin.
+
+## Configuration
+
+To use this integration, add configuration to your `app-config.yaml`:
+
+Using Azure active directory credentials:
+
+```yaml
+integrations:
+  azureBlobStorage:
+    - accountName: ${ACCOUNT_NAME} # required
+      endpoint: ${CUSTOM_ENDPOINT} # custom endpoint will require either aadCredentials or sasToken
+      aadCredential:
+        clientId: ${CLIENT_ID}
+        tenantId: ${TENANT_ID}
+        clientSecret: ${CLIENT_SECRET}
+```
+
+Using Azure storage account SAS token:
+
+```yaml
+integrations:
+  azureBlobStorage:
+    - accountName: ${ACCOUNT_NAME} # required
+      endpoint: ${CUSTOM_ENDPOINT} # custom endpoint will require either aadCredentials or sasToken
+      sasToken: ${SAS_TOKEN}
+```
+
+Using Azure storage account access key:
+
+```yaml
+integrations:
+  azureBlobStorage:
+    - accountName: ${ACCOUNT_NAME} # required
+      endpoint: ${CUSTOM_ENDPOINT} # custom endpoint will require either aadCredentials or sasToken
+      accountKey: ${ACCOUNT_KEY}
+```

packages/backend-defaults/report-urlReader.api.md

@@ -7,6 +7,8 @@
 
 import { AwsCredentialsManager } from '@backstage/integration-aws-node';
 import { AwsS3Integration } from '@backstage/integration';
+import { AzureBlobStorageIntergation } from '@backstage/integration';
+import { AzureCredentialsManager } from '@backstage/integration';
 import { AzureDevOpsCredentialsProvider } from '@backstage/integration';
 import { AzureIntegration } from '@backstage/integration';
 import { BitbucketCloudIntegration } from '@backstage/integration';
@@ -60,6 +62,35 @@ export class AwsS3UrlReader implements UrlReaderService {
   toString(): string;
 }
 
+// @public
+export class AzureBlobStorageUrlReader implements UrlReaderService {
+  constructor(
+    credsManager: AzureCredentialsManager,
+    integration: AzureBlobStorageIntergation,
+    deps: {
+      treeResponseFactory: ReadTreeResponseFactory;
+    },
+  );
+  // (undocumented)
+  static factory: ReaderFactory;
+  // (undocumented)
+  read(url: string): Promise<Buffer>;
+  // (undocumented)
+  readTree(
+    url: string,
+    options?: UrlReaderServiceReadTreeOptions,
+  ): Promise<UrlReaderServiceReadTreeResponse>;
+  // (undocumented)
+  readUrl(
+    url: string,
+    options?: UrlReaderServiceReadUrlOptions,
+  ): Promise<UrlReaderServiceReadUrlResponse>;
+  // (undocumented)
+  search(): Promise<UrlReaderServiceSearchResponse>;
+  // (undocumented)
+  toString(): string;
+}
+
 // @public
 export class AzureUrlReader implements UrlReaderService {
   constructor(

packages/backend-defaults/src/entrypoints/urlReader/lib/AzureBlobStorageUrlReader.ts

@@ -54,6 +54,12 @@ export function parseUrl(url: string): { path: string; container: string } {
 
   return { path, container };
 }
+
+/**
+ * Implements a {@link @backstage/backend-plugin-api#UrlReaderService} for Azure storage accounts urls.
+ *
+ * @public
+ */
 export class AzureBlobStorageUrlReader implements UrlReaderService {
   static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
     const integrations = ScmIntegrations.fromConfig(config);
@@ -71,7 +77,9 @@ export class AzureBlobStorageUrlReader implements UrlReaderService {
       );
 
       const predicate = (url: URL) =>
-        url.host.endsWith(integrationConfig.config.host);
+        url.host.endsWith(
+          `${integrationConfig.config.accountName}.${integrationConfig.config.host}`,
+        );
       return { reader, predicate };
     });
   };
@@ -141,16 +149,13 @@ export class AzureBlobStorageUrlReader implements UrlReaderService {
       const containerClient = await this.createContainerClient(container);
       const blobClient = containerClient.getBlobClient(path);
 
-      const abortController = new AbortController();
-
       const getBlobOptions: BlobDownloadOptions = {
-        abortSignal: abortController.signal,
+        abortSignal: options?.signal,
         conditions: {
           ...(etag && { ifNoneMatch: etag }),
           ...(lastModifiedAfter && { ifModifiedSince: lastModifiedAfter }),
         },
       };
-      options?.signal?.addEventListener('abort', () => abortController.abort());
 
       const downloadBlockBlobResponse = await blobClient.download(
         0,
@@ -189,13 +194,14 @@ export class AzureBlobStorageUrlReader implements UrlReaderService {
       const blobs = containerClient.listBlobsFlat({ prefix: path });
 
       const responses = [];
-      const abortController = new AbortController();
+
       for await (const blob of blobs) {
         const blobClient = containerClient.getBlobClient(blob.name);
-        options?.signal?.addEventListener('abort', () =>
-          abortController.abort(),
+        const downloadBlockBlobResponse = await blobClient.download(
+          undefined,
+          undefined,
+          { abortSignal: options?.signal },
         );
-        const downloadBlockBlobResponse = await blobClient.download();
 
         responses.push({
           data: Readable.from(

packages/backend/package.json

@@ -39,6 +39,7 @@
     "@backstage/plugin-auth-backend-module-guest-provider": "workspace:^",
     "@backstage/plugin-auth-node": "workspace:^",
     "@backstage/plugin-catalog-backend": "workspace:^",
+    "@backstage/plugin-catalog-backend-module-azure": "workspace:^",
     "@backstage/plugin-catalog-backend-module-backstage-openapi": "workspace:^",
     "@backstage/plugin-catalog-backend-module-openapi": "workspace:^",
     "@backstage/plugin-catalog-backend-module-scaffolder-entity-model": "workspace:^",

packages/backend/src/index.ts

@@ -56,5 +56,6 @@ backend.add(searchLoader);
 backend.add(import('@backstage/plugin-techdocs-backend'));
 backend.add(import('@backstage/plugin-signals-backend'));
 backend.add(import('@backstage/plugin-notifications-backend'));
+backend.add(import('@backstage/plugin-catalog-backend-module-azure'));
 
 backend.start();

packages/integration/report.api.md

@@ -3,9 +3,12 @@
 > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
 
 ```ts
+import { AnonymousCredential } from '@azure/storage-blob';
 import { Config } from '@backstage/config';
 import { ConsumedResponse } from '@backstage/errors';
 import { RestEndpointMethodTypes } from '@octokit/rest';
+import { StorageSharedKeyCredential } from '@azure/storage-blob';
+import { TokenCredential } from '@azure/identity';
 
 // @public
 export class AwsCodeCommitIntegration implements ScmIntegration {
@@ -70,6 +73,43 @@ export type AwsS3IntegrationConfig = {
   externalId?: string;
 };
 
+// @public
+export type AzureBlobStorageIntegrationConfig = {
+  accountName?: string;
+  accountKey?: string;
+  sasToken?: string;
+  connectionString?: string;
+  endpointSuffix?: string;
+  host: string;
+  endpoint?: string;
+  aadCredential?: {
+    clientId: string;
+    tenantId: string;
+    clientSecret: string;
+  };
+};
+
+// @public
+export class AzureBlobStorageIntergation implements ScmIntegration {
+  constructor(integrationConfig: AzureBlobStorageIntegrationConfig);
+  // (undocumented)
+  get config(): AzureBlobStorageIntegrationConfig;
+  // (undocumented)
+  static factory: ScmIntegrationsFactory<AzureBlobStorageIntergation>;
+  // (undocumented)
+  resolveEditUrl(url: string): string;
+  // (undocumented)
+  resolveUrl(options: {
+    url: string;
+    base: string;
+    lineNumber?: number | undefined;
+  }): string;
+  // (undocumented)
+  get title(): string;
+  // (undocumented)
+  get type(): string;
+}
+
 // @public
 export type AzureClientSecretCredential = AzureCredentialBase & {
   kind: 'ClientSecret';
@@ -84,6 +124,16 @@ export type AzureCredentialBase = {
   organizations?: string[];
 };
 
+// @public
+export interface AzureCredentialsManager {
+  // (undocumented)
+  getCredentials(
+    accountName: string,
+  ): Promise<
+    TokenCredential | StorageSharedKeyCredential | AnonymousCredential
+  >;
+}
+
 // @public
 export type AzureDevOpsCredential =
   | AzureClientSecretCredential
@@ -257,6 +307,15 @@ export function buildGerritGitilesArchiveUrl(
   filePath: string,
 ): string;
 
+// @public
+export class DefaultAzureCredentialsManager implements AzureCredentialsManager {
+  static fromIntegrations(
+    integrations: ScmIntegrationRegistry,
+  ): DefaultAzureCredentialsManager;
+  // (undocumented)
+  getCredentials(accountName: string): Promise<TokenCredential>;
+}
+
 // @public
 export class DefaultAzureDevOpsCredentialsProvider
   implements AzureDevOpsCredentialsProvider
@@ -723,6 +782,8 @@ export interface IntegrationsByType {
   awsS3: ScmIntegrationsGroup<AwsS3Integration>;
   // (undocumented)
   azure: ScmIntegrationsGroup<AzureIntegration>;
+  // (undocumented)
+  azureBlobStorage: ScmIntegrationsGroup<AzureBlobStorageIntergation>;
   // @deprecated (undocumented)
   bitbucket: ScmIntegrationsGroup<BitbucketIntegration>;
   // (undocumented)
@@ -826,6 +887,16 @@ export function readAwsS3IntegrationConfigs(
   configs: Config[],
 ): AwsS3IntegrationConfig[];
 
+// @public
+export function readAzureBlobStorageIntegrationConfig(
+  config: Config,
+): AzureBlobStorageIntegrationConfig;
+
+// @public
+export function readAzureBlobStorageIntegrationConfigs(
+  configs: Config[],
+): AzureBlobStorageIntegrationConfig[];
+
 // @public
 export function readAzureIntegrationConfig(
   config: Config,
@@ -940,6 +1011,8 @@ export interface ScmIntegrationRegistry
   awsS3: ScmIntegrationsGroup<AwsS3Integration>;
   // (undocumented)
   azure: ScmIntegrationsGroup<AzureIntegration>;
+  // (undocumented)
+  azureBlobStorage: ScmIntegrationsGroup<AzureBlobStorageIntergation>;
   // @deprecated (undocumented)
   bitbucket: ScmIntegrationsGroup<BitbucketIntegration>;
   // (undocumented)
@@ -973,6 +1046,8 @@ export class ScmIntegrations implements ScmIntegrationRegistry {
   get awsS3(): ScmIntegrationsGroup<AwsS3Integration>;
   // (undocumented)
   get azure(): ScmIntegrationsGroup<AzureIntegration>;
+  // (undocumented)
+  get azureBlobStorage(): ScmIntegrationsGroup<AzureBlobStorageIntergation>;
   // @deprecated (undocumented)
   get bitbucket(): ScmIntegrationsGroup<BitbucketIntegration>;
   // (undocumented)

packages/integration/src/azureBlobStorage/AzureBlobStorageIntegration.ts

@@ -21,6 +21,11 @@ import {
   readAzureBlobStorageIntegrationConfigs,
 } from './config';
 
+/**
+ * Microsoft Azure Blob storage based integration.
+ *
+ * @public
+ */
 export class AzureBlobStorageIntergation implements ScmIntegration {
   static factory: ScmIntegrationsFactory<AzureBlobStorageIntergation> = ({
     config,

packages/integration/src/azureBlobStorage/DefaultAzureCredentialsProvider.ts

@@ -23,26 +23,58 @@ import { AzureBlobStorageIntegrationConfig } from './config';
 import { AzureCredentialsManager } from './types';
 import { ScmIntegrationRegistry } from '../registry';
 
+/**
+ * Default implementation of AzureCredentialsManager that supports multiple Azure Blob Storage integrations.
+ * @public
+ */
 export class DefaultAzureCredentialsManager implements AzureCredentialsManager {
-  private config: AzureBlobStorageIntegrationConfig;
   private cachedCredentials: Map<string, TokenCredential>;
 
-  constructor(config: AzureBlobStorageIntegrationConfig) {
-    this.config = config;
+  private constructor(
+    private readonly configProviders: Map<
+      string,
+      AzureBlobStorageIntegrationConfig
+    >,
+  ) {
     this.cachedCredentials = new Map<string, TokenCredential>();
   }
 
   /**
-   * Creates an instance of DefaultAzureCredentialsManager from a Backstage Config.
+   * Creates an instance of DefaultAzureCredentialsManager from a Backstage integration registry.
    */
   static fromIntegrations(
-    integration: ScmIntegrationRegistry,
+    integrations: ScmIntegrationRegistry,
   ): DefaultAzureCredentialsManager {
-    const azureConfig = integration.azureBlobStorage.list().length
-      ? integration.azureBlobStorage.list()[0].config
-      : { host: 'blob.core.windows.net' }; // Default to Azure Blob Storage host if no config found
+    const configProviders = integrations.azureBlobStorage
+      .list()
+      .reduce((acc, integration) => {
+        acc.set(
+          integration.config.accountName || 'default',
+          integration.config,
+        );
+        return acc;
+      }, new Map<string, AzureBlobStorageIntegrationConfig>());
 
-    return new DefaultAzureCredentialsManager(azureConfig);
+    return new DefaultAzureCredentialsManager(configProviders);
+  }
+
+  private createCredential(
+    config: AzureBlobStorageIntegrationConfig,
+  ): TokenCredential {
+    if (
+      config.aadCredential &&
+      config.aadCredential.clientId &&
+      config.aadCredential.clientSecret &&
+      config.aadCredential.tenantId
+    ) {
+      return new ClientSecretCredential(
+        config.aadCredential.tenantId,
+        config.aadCredential.clientId,
+        config.aadCredential.clientSecret,
+      );
+    }
+
+    return new DefaultAzureCredential();
   }
 
   async getCredentials(accountName: string): Promise<TokenCredential> {
@@ -50,23 +82,13 @@ export class DefaultAzureCredentialsManager implements AzureCredentialsManager {
       return this.cachedCredentials.get(accountName)!;
     }
 
-    let credential: TokenCredential;
-
-    if (
-      this.config.aadCredential &&
-      this.config.aadCredential.clientId &&
-      this.config.aadCredential.clientSecret &&
-      this.config.aadCredential.tenantId
-    ) {
-      credential = new ClientSecretCredential(
-        this.config.aadCredential.tenantId,
-        this.config.aadCredential.clientId,
-        this.config.aadCredential.clientSecret,
-      );
-    } else {
-      credential = new DefaultAzureCredential();
+    const config = this.configProviders.get(accountName);
+    if (!config) {
+      throw new Error(`No configuration found for account: ${accountName}`);
     }
 
+    const credential = this.createCredential(config);
+
     // Cache the credentials for future use
     this.cachedCredentials.set(accountName, credential);
 

packages/integration/src/azureBlobStorage/types.ts

@@ -20,6 +20,12 @@ import {
   AnonymousCredential,
 } from '@azure/storage-blob';
 
+/**
+ * This allows implementations to be provided to retrieve Azure Storage accounts credentials.
+ *
+ * @public
+ *
+ */
 export interface AzureCredentialsManager {
   getCredentials(
     accountName: string,

plugins/catalog-backend-module-azure/report.api.md

@@ -15,9 +15,7 @@ import { SchedulerService } from '@backstage/backend-plugin-api';
 import { SchedulerServiceTaskRunner } from '@backstage/backend-plugin-api';
 import { ScmIntegrationRegistry } from '@backstage/integration';
 
-// Warning: (ae-missing-release-tag) "AzureBlobStorageEntityProvider" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
-//
-// @public (undocumented)
+// @public
 export class AzureBlobStorageEntityProvider implements EntityProvider {
   // (undocumented)
   connect(connection: EntityProviderConnection): Promise<void>;

plugins/catalog-backend-module-azure/src/providers/AzureBlobStorageEntityProvider.ts

@@ -42,6 +42,13 @@ import {
 import { TokenCredential } from '@azure/identity';
 import { AzureBlobStorageConfig } from './types';
 
+/**
+ * Provider which discovers catalog files within an Azure Storage accounts.
+ *
+ * Use `AzureBlobStorageEntityProvider.fromConfig(...)` to create instances.
+ *
+ * @public
+ */
 export class AzureBlobStorageEntityProvider implements EntityProvider {
   private readonly logger: LoggerService;
   private connection?: EntityProviderConnection;
@@ -66,7 +73,12 @@ export class AzureBlobStorageEntityProvider implements EntityProvider {
     }
 
     return providerConfigs.map(providerConfig => {
-      const integration = scmIntegration.azureBlobStorage.list()[0];
+      const integration = scmIntegration.azureBlobStorage
+        .list()
+        .filter(
+          azureIntegration =>
+            azureIntegration.config.accountName === providerConfig.accountName,
+        )[0];
       if (!integration) {
         throw new Error(
           `There is no Azure blob storage integration for host. Please add a configuration entry for it under integrations.azure`,

plugins/catalog-backend-module-azure/src/providers/config.ts

@@ -98,6 +98,7 @@ function readAzureBlobStorageConfig(
   config: Config,
 ): AzureBlobStorageConfig {
   const containerName = config.getString('containerName');
+  const accountName = config.getString('accountName');
   const schedule = config.has('schedule')
     ? readSchedulerServiceTaskScheduleDefinitionFromConfig(
         config.getConfig('schedule'),
@@ -107,6 +108,7 @@ function readAzureBlobStorageConfig(
   return {
     id,
     containerName,
+    accountName,
     schedule,
   };
 }

plugins/catalog-backend-module-azure/src/providers/types.ts

@@ -30,5 +30,6 @@ export type AzureDevOpsConfig = {
 export type AzureBlobStorageConfig = {
   id: string;
   containerName: string;
+  accountName: string;
   schedule?: SchedulerServiceTaskScheduleDefinition;
 };

yarn.lock

@@ -5395,7 +5395,7 @@ __metadata:
   languageName: unknown
   linkType: soft
 
-"@backstage/plugin-catalog-backend-module-azure@workspace:plugins/catalog-backend-module-azure":
+"@backstage/plugin-catalog-backend-module-azure@workspace:^, @backstage/plugin-catalog-backend-module-azure@workspace:plugins/catalog-backend-module-azure":
   version: 0.0.0-use.local
   resolution: "@backstage/plugin-catalog-backend-module-azure@workspace:plugins/catalog-backend-module-azure"
   dependencies:
@@ -27070,6 +27070,7 @@ __metadata:
     "@backstage/plugin-auth-backend-module-guest-provider": "workspace:^"
     "@backstage/plugin-auth-node": "workspace:^"
     "@backstage/plugin-catalog-backend": "workspace:^"
+    "@backstage/plugin-catalog-backend-module-azure": "workspace:^"
     "@backstage/plugin-catalog-backend-module-backstage-openapi": "workspace:^"
     "@backstage/plugin-catalog-backend-module-openapi": "workspace:^"
     "@backstage/plugin-catalog-backend-module-scaffolder-entity-model": "workspace:^"