From 5985d458ee3032cb9253186edc018af13fd4ae37 Mon Sep 17 00:00:00 2001 From: titanventura Date: Thu, 27 Jul 2023 23:46:37 +0530 Subject: [PATCH 1/6] Added azurite connection string option to techdocs azureBlobStorage publisher Signed-off-by: titanventura --- .changeset/pretty-otters-whisper.md | 69 +++++++++++++++++++ docs/features/techdocs/configuration.md | 5 ++ .../src/stages/publish/azureBlobStorage.ts | 25 +++++-- 3 files changed, 94 insertions(+), 5 deletions(-) create mode 100644 .changeset/pretty-otters-whisper.md diff --git a/.changeset/pretty-otters-whisper.md b/.changeset/pretty-otters-whisper.md new file mode 100644 index 0000000000..91dd8344de --- /dev/null +++ b/.changeset/pretty-otters-whisper.md @@ -0,0 +1,69 @@ +--- +'@backstage/plugin-techdocs-node': minor +--- + +Add azurite support in techdocs through `techdocs.publisher.azureBlobStorage.azuriteConnectionString` + +These changes are **required** to `plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts` and `docs/features/techdocs/configuration.md` + +```diff +diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md +index 49ee9d7..a8d302c 100644 +--- a/docs/features/techdocs/configuration.md ++++ b/docs/features/techdocs/configuration.md +@@ -158,6 +158,11 @@ techdocs: + # (Required) Azure Blob Storage Container Name + containerName: 'techdocs-storage' + ++ # (Optional) Azurite connection string for local testing. ++ # Defaults to undefined ++ # if provided, takes higher priority, 'techdocs.publisher.azureBlobStorage.credentials' will become irrelevant ++ azuriteConnectionString: '' ++ + # (Required) An account name is required to write to a storage blob container. + # https://docs.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key + credentials: +diff --git a/plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts b/plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts +index bcbc10a..5f19b80 100644 +--- a/plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts ++++ b/plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts +@@ -78,6 +78,26 @@ export class AzureBlobStoragePublish implements PublisherBase { + ); + } + ++ const legacyPathCasing = ++ config.getOptionalBoolean( ++ 'techdocs.legacyUseCaseSensitiveTripletPaths', ++ ) || false; ++ ++ // Give more priority for azurite, if configured, return the AzureBlobStoragePublish object here itself ++ const azuriteConnString = config.getOptionalString( ++ 'techdocs.publisher.azureBlobStorage.azuriteConnectionString', ++ ); ++ if (azuriteConnString) { ++ const storageClient = ++ BlobServiceClient.fromConnectionString(azuriteConnString); ++ return new AzureBlobStoragePublish({ ++ storageClient: storageClient, ++ containerName: containerName, ++ legacyPathCasing: legacyPathCasing, ++ logger: logger, ++ }); ++ } ++ + let accountName = ''; + try { + accountName = config.getString( +@@ -108,11 +128,6 @@ export class AzureBlobStoragePublish implements PublisherBase { + credential, + ); + +- const legacyPathCasing = +- config.getOptionalBoolean( +- 'techdocs.legacyUseCaseSensitiveTripletPaths', +- ) || false; +- + return new AzureBlobStoragePublish({ + storageClient: storageClient, + containerName: containerName, +``` diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index 49ee9d72c2..a8d302c35b 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -158,6 +158,11 @@ techdocs: # (Required) Azure Blob Storage Container Name containerName: 'techdocs-storage' + # (Optional) Azurite connection string for local testing. + # Defaults to undefined + # if provided, takes higher priority, 'techdocs.publisher.azureBlobStorage.credentials' will become irrelevant + azuriteConnectionString: '' + # (Required) An account name is required to write to a storage blob container. # https://docs.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key credentials: diff --git a/plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts b/plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts index bcbc10a94d..5f19b8010f 100644 --- a/plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts +++ b/plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts @@ -78,6 +78,26 @@ export class AzureBlobStoragePublish implements PublisherBase { ); } + const legacyPathCasing = + config.getOptionalBoolean( + 'techdocs.legacyUseCaseSensitiveTripletPaths', + ) || false; + + // Give more priority for azurite, if configured, return the AzureBlobStoragePublish object here itself + const azuriteConnString = config.getOptionalString( + 'techdocs.publisher.azureBlobStorage.azuriteConnectionString', + ); + if (azuriteConnString) { + const storageClient = + BlobServiceClient.fromConnectionString(azuriteConnString); + return new AzureBlobStoragePublish({ + storageClient: storageClient, + containerName: containerName, + legacyPathCasing: legacyPathCasing, + logger: logger, + }); + } + let accountName = ''; try { accountName = config.getString( @@ -108,11 +128,6 @@ export class AzureBlobStoragePublish implements PublisherBase { credential, ); - const legacyPathCasing = - config.getOptionalBoolean( - 'techdocs.legacyUseCaseSensitiveTripletPaths', - ) || false; - return new AzureBlobStoragePublish({ storageClient: storageClient, containerName: containerName, From 7284a2e68b9dc965417245542904a4f54dcaaaf2 Mon Sep 17 00:00:00 2001 From: titanventura Date: Sat, 29 Jul 2023 15:46:31 +0530 Subject: [PATCH 2/6] Changed 'techdocs.publisher.azureBlobStorage.azuriteConnectionString' to 'techdocs.publisher.azureBlobStorage.connectionString' Signed-off-by: titanventura --- .changeset/soft-humans-smoke.md | 55 +++++++++++++++++++ docs/features/techdocs/configuration.md | 5 +- .../src/stages/publish/azureBlobStorage.ts | 15 +++-- 3 files changed, 67 insertions(+), 8 deletions(-) create mode 100644 .changeset/soft-humans-smoke.md diff --git a/.changeset/soft-humans-smoke.md b/.changeset/soft-humans-smoke.md new file mode 100644 index 0000000000..742fba73f9 --- /dev/null +++ b/.changeset/soft-humans-smoke.md @@ -0,0 +1,55 @@ +--- +'@backstage/plugin-techdocs-node': minor +--- + +Added connectionString option to 'techdocs.publisher.azureBlobStorage' + +These changes are **required** to `docs/features/techdocs/configuration.md` and `plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts` + +```diff +diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md +index a8d302c..687ad3c 100644 +--- a/docs/features/techdocs/configuration.md ++++ b/docs/features/techdocs/configuration.md +@@ -158,10 +158,11 @@ techdocs: + # (Required) Azure Blob Storage Container Name + containerName: 'techdocs-storage' + +- # (Optional) Azurite connection string for local testing. ++ # (Optional) Azure blob storage connection string. ++ # Can be useful for local testing through azurite + # Defaults to undefined + # if provided, takes higher priority, 'techdocs.publisher.azureBlobStorage.credentials' will become irrelevant +- azuriteConnectionString: '' ++ connectionString: '' + + # (Required) An account name is required to write to a storage blob container. + # https://docs.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key +diff --git a/plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts b/plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts +index 5f19b80..db2c380 100644 +--- a/plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts ++++ b/plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts +@@ -83,13 +83,16 @@ export class AzureBlobStoragePublish implements PublisherBase { + 'techdocs.legacyUseCaseSensitiveTripletPaths', + ) || false; + +- // Give more priority for azurite, if configured, return the AzureBlobStoragePublish object here itself +- const azuriteConnString = config.getOptionalString( +- 'techdocs.publisher.azureBlobStorage.azuriteConnectionString', +- ); +- if (azuriteConnString) { ++ // Give more priority for connectionString, if configured, return the AzureBlobStoragePublish object here itself ++ const connectionStringKey = ++ 'techdocs.publisher.azureBlobStorage.connectionString'; ++ const connectionString = config.getOptionalString(connectionStringKey); ++ if (connectionString) { ++ logger.info( ++ `using ${connectionStringKey} string to create BlobServiceClient`, ++ ); + const storageClient = +- BlobServiceClient.fromConnectionString(azuriteConnString); ++ BlobServiceClient.fromConnectionString(connectionString); + return new AzureBlobStoragePublish({ + storageClient: storageClient, + containerName: containerName, +``` diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index a8d302c35b..687ad3c61e 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -158,10 +158,11 @@ techdocs: # (Required) Azure Blob Storage Container Name containerName: 'techdocs-storage' - # (Optional) Azurite connection string for local testing. + # (Optional) Azure blob storage connection string. + # Can be useful for local testing through azurite # Defaults to undefined # if provided, takes higher priority, 'techdocs.publisher.azureBlobStorage.credentials' will become irrelevant - azuriteConnectionString: '' + connectionString: '' # (Required) An account name is required to write to a storage blob container. # https://docs.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key diff --git a/plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts b/plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts index 5f19b8010f..db2c3809fe 100644 --- a/plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts +++ b/plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts @@ -83,13 +83,16 @@ export class AzureBlobStoragePublish implements PublisherBase { 'techdocs.legacyUseCaseSensitiveTripletPaths', ) || false; - // Give more priority for azurite, if configured, return the AzureBlobStoragePublish object here itself - const azuriteConnString = config.getOptionalString( - 'techdocs.publisher.azureBlobStorage.azuriteConnectionString', - ); - if (azuriteConnString) { + // Give more priority for connectionString, if configured, return the AzureBlobStoragePublish object here itself + const connectionStringKey = + 'techdocs.publisher.azureBlobStorage.connectionString'; + const connectionString = config.getOptionalString(connectionStringKey); + if (connectionString) { + logger.info( + `using ${connectionStringKey} string to create BlobServiceClient`, + ); const storageClient = - BlobServiceClient.fromConnectionString(azuriteConnString); + BlobServiceClient.fromConnectionString(connectionString); return new AzureBlobStoragePublish({ storageClient: storageClient, containerName: containerName, From 59657978d6b386706d3d14039ca77c64ba8bf4f7 Mon Sep 17 00:00:00 2001 From: titanventura Date: Wed, 16 Aug 2023 10:52:34 +0530 Subject: [PATCH 3/6] Updated changelog to pass CI Signed-off-by: titanventura --- .changeset/soft-humans-smoke.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/soft-humans-smoke.md b/.changeset/soft-humans-smoke.md index 742fba73f9..2da0680bf0 100644 --- a/.changeset/soft-humans-smoke.md +++ b/.changeset/soft-humans-smoke.md @@ -2,7 +2,7 @@ '@backstage/plugin-techdocs-node': minor --- -Added connectionString option to 'techdocs.publisher.azureBlobStorage' +Added connection string option to `techdocs.publisher.azureBlobStorage` These changes are **required** to `docs/features/techdocs/configuration.md` and `plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts` From b8c7850c7bc7bc02e4b9ed1e66e847a2bbcebc49 Mon Sep 17 00:00:00 2001 From: titanventura Date: Tue, 22 Aug 2023 23:22:26 +0530 Subject: [PATCH 4/6] Refactored azureBlobStorage, changelog, techdocs-backend config Signed-off-by: titanventura --- .changeset/soft-humans-smoke.md | 55 --------------------------------- 1 file changed, 55 deletions(-) delete mode 100644 .changeset/soft-humans-smoke.md diff --git a/.changeset/soft-humans-smoke.md b/.changeset/soft-humans-smoke.md deleted file mode 100644 index 2da0680bf0..0000000000 --- a/.changeset/soft-humans-smoke.md +++ /dev/null @@ -1,55 +0,0 @@ ---- -'@backstage/plugin-techdocs-node': minor ---- - -Added connection string option to `techdocs.publisher.azureBlobStorage` - -These changes are **required** to `docs/features/techdocs/configuration.md` and `plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts` - -```diff -diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md -index a8d302c..687ad3c 100644 ---- a/docs/features/techdocs/configuration.md -+++ b/docs/features/techdocs/configuration.md -@@ -158,10 +158,11 @@ techdocs: - # (Required) Azure Blob Storage Container Name - containerName: 'techdocs-storage' - -- # (Optional) Azurite connection string for local testing. -+ # (Optional) Azure blob storage connection string. -+ # Can be useful for local testing through azurite - # Defaults to undefined - # if provided, takes higher priority, 'techdocs.publisher.azureBlobStorage.credentials' will become irrelevant -- azuriteConnectionString: '' -+ connectionString: '' - - # (Required) An account name is required to write to a storage blob container. - # https://docs.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key -diff --git a/plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts b/plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts -index 5f19b80..db2c380 100644 ---- a/plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts -+++ b/plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts -@@ -83,13 +83,16 @@ export class AzureBlobStoragePublish implements PublisherBase { - 'techdocs.legacyUseCaseSensitiveTripletPaths', - ) || false; - -- // Give more priority for azurite, if configured, return the AzureBlobStoragePublish object here itself -- const azuriteConnString = config.getOptionalString( -- 'techdocs.publisher.azureBlobStorage.azuriteConnectionString', -- ); -- if (azuriteConnString) { -+ // Give more priority for connectionString, if configured, return the AzureBlobStoragePublish object here itself -+ const connectionStringKey = -+ 'techdocs.publisher.azureBlobStorage.connectionString'; -+ const connectionString = config.getOptionalString(connectionStringKey); -+ if (connectionString) { -+ logger.info( -+ `using ${connectionStringKey} string to create BlobServiceClient`, -+ ); - const storageClient = -- BlobServiceClient.fromConnectionString(azuriteConnString); -+ BlobServiceClient.fromConnectionString(connectionString); - return new AzureBlobStoragePublish({ - storageClient: storageClient, - containerName: containerName, -``` From c258bd395ff67cb135dc2e8383dadc7d030f4cc6 Mon Sep 17 00:00:00 2001 From: titanventura Date: Tue, 22 Aug 2023 23:22:37 +0530 Subject: [PATCH 5/6] Refactored azureBlobStorage, changelog, techdocs-backend config Signed-off-by: titanventura --- .changeset/pretty-otters-whisper.md | 67 +---------------- plugins/techdocs-backend/config.d.ts | 7 +- .../src/stages/publish/azureBlobStorage.ts | 71 +++++++++---------- 3 files changed, 41 insertions(+), 104 deletions(-) diff --git a/.changeset/pretty-otters-whisper.md b/.changeset/pretty-otters-whisper.md index 91dd8344de..7019596b38 100644 --- a/.changeset/pretty-otters-whisper.md +++ b/.changeset/pretty-otters-whisper.md @@ -1,69 +1,6 @@ --- '@backstage/plugin-techdocs-node': minor +'@backstage/plugin-techdocs-backend': minor --- -Add azurite support in techdocs through `techdocs.publisher.azureBlobStorage.azuriteConnectionString` - -These changes are **required** to `plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts` and `docs/features/techdocs/configuration.md` - -```diff -diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md -index 49ee9d7..a8d302c 100644 ---- a/docs/features/techdocs/configuration.md -+++ b/docs/features/techdocs/configuration.md -@@ -158,6 +158,11 @@ techdocs: - # (Required) Azure Blob Storage Container Name - containerName: 'techdocs-storage' - -+ # (Optional) Azurite connection string for local testing. -+ # Defaults to undefined -+ # if provided, takes higher priority, 'techdocs.publisher.azureBlobStorage.credentials' will become irrelevant -+ azuriteConnectionString: '' -+ - # (Required) An account name is required to write to a storage blob container. - # https://docs.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key - credentials: -diff --git a/plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts b/plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts -index bcbc10a..5f19b80 100644 ---- a/plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts -+++ b/plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts -@@ -78,6 +78,26 @@ export class AzureBlobStoragePublish implements PublisherBase { - ); - } - -+ const legacyPathCasing = -+ config.getOptionalBoolean( -+ 'techdocs.legacyUseCaseSensitiveTripletPaths', -+ ) || false; -+ -+ // Give more priority for azurite, if configured, return the AzureBlobStoragePublish object here itself -+ const azuriteConnString = config.getOptionalString( -+ 'techdocs.publisher.azureBlobStorage.azuriteConnectionString', -+ ); -+ if (azuriteConnString) { -+ const storageClient = -+ BlobServiceClient.fromConnectionString(azuriteConnString); -+ return new AzureBlobStoragePublish({ -+ storageClient: storageClient, -+ containerName: containerName, -+ legacyPathCasing: legacyPathCasing, -+ logger: logger, -+ }); -+ } -+ - let accountName = ''; - try { - accountName = config.getString( -@@ -108,11 +128,6 @@ export class AzureBlobStoragePublish implements PublisherBase { - credential, - ); - -- const legacyPathCasing = -- config.getOptionalBoolean( -- 'techdocs.legacyUseCaseSensitiveTripletPaths', -- ) || false; -- - return new AzureBlobStoragePublish({ - storageClient: storageClient, - containerName: containerName, -``` +Add a `techdocs.publisher.azureBlobStorage.connectionString` app-config setting, which can be leveraged for improved Azurite support. diff --git a/plugins/techdocs-backend/config.d.ts b/plugins/techdocs-backend/config.d.ts index 3a05beabdc..14725bfa50 100644 --- a/plugins/techdocs-backend/config.d.ts +++ b/plugins/techdocs-backend/config.d.ts @@ -198,7 +198,12 @@ export interface Config { */ azureBlobStorage?: { /** - * (Required) Credentials used to access a storage container. + * (Optional) Connection string of the storage container. + * @visibility secret + */ + connectionString: string; + /** + * (Optional) Credentials used to access a storage container. * @visibility secret */ credentials: { diff --git a/plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts b/plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts index db2c3809fe..39a50e963a 100644 --- a/plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts +++ b/plugins/techdocs-node/src/stages/publish/azureBlobStorage.ts @@ -66,6 +66,7 @@ export class AzureBlobStoragePublish implements PublisherBase { } static fromConfig(config: Config, logger: Logger): PublisherBase { + let storageClient: BlobServiceClient; let containerName = ''; try { containerName = config.getString( @@ -87,49 +88,43 @@ export class AzureBlobStoragePublish implements PublisherBase { const connectionStringKey = 'techdocs.publisher.azureBlobStorage.connectionString'; const connectionString = config.getOptionalString(connectionStringKey); + if (connectionString) { logger.info( - `using ${connectionStringKey} string to create BlobServiceClient`, + `Using '${connectionStringKey}' configuration to create storage client`, ); - const storageClient = - BlobServiceClient.fromConnectionString(connectionString); - return new AzureBlobStoragePublish({ - storageClient: storageClient, - containerName: containerName, - legacyPathCasing: legacyPathCasing, - logger: logger, - }); - } - - let accountName = ''; - try { - accountName = config.getString( - 'techdocs.publisher.azureBlobStorage.credentials.accountName', - ); - } catch (error) { - throw new Error( - "Since techdocs.publisher.type is set to 'azureBlobStorage' in your app config, " + - 'techdocs.publisher.azureBlobStorage.credentials.accountName is required.', - ); - } - - // Credentials is an optional config. If missing, default Azure Blob Storage environment variables will be used. - // https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-app - const accountKey = config.getOptionalString( - 'techdocs.publisher.azureBlobStorage.credentials.accountKey', - ); - - let credential; - if (accountKey) { - credential = new StorageSharedKeyCredential(accountName, accountKey); + storageClient = BlobServiceClient.fromConnectionString(connectionString); } else { - credential = new DefaultAzureCredential(); - } + let accountName = ''; + try { + accountName = config.getString( + 'techdocs.publisher.azureBlobStorage.credentials.accountName', + ); + } catch (error) { + throw new Error( + "Since techdocs.publisher.type is set to 'azureBlobStorage' in your app config, " + + 'techdocs.publisher.azureBlobStorage.credentials.accountName is required.', + ); + } - const storageClient = new BlobServiceClient( - `https://${accountName}.blob.core.windows.net`, - credential, - ); + // Credentials is an optional config. If missing, default Azure Blob Storage environment variables will be used. + // https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-app + const accountKey = config.getOptionalString( + 'techdocs.publisher.azureBlobStorage.credentials.accountKey', + ); + + let credential; + if (accountKey) { + credential = new StorageSharedKeyCredential(accountName, accountKey); + } else { + credential = new DefaultAzureCredential(); + } + + storageClient = new BlobServiceClient( + `https://${accountName}.blob.core.windows.net`, + credential, + ); + } return new AzureBlobStoragePublish({ storageClient: storageClient, From 27d7c74922b25c7a8d6b4bbbe1fb5afea8ab9795 Mon Sep 17 00:00:00 2001 From: titanventura Date: Thu, 24 Aug 2023 20:51:40 +0530 Subject: [PATCH 6/6] Refactored config.d.ts to make connectionString and credentials optional Signed-off-by: titanventura --- plugins/techdocs-backend/config.d.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/techdocs-backend/config.d.ts b/plugins/techdocs-backend/config.d.ts index 14725bfa50..d00aabd025 100644 --- a/plugins/techdocs-backend/config.d.ts +++ b/plugins/techdocs-backend/config.d.ts @@ -201,12 +201,12 @@ export interface Config { * (Optional) Connection string of the storage container. * @visibility secret */ - connectionString: string; + connectionString?: string; /** * (Optional) Credentials used to access a storage container. * @visibility secret */ - credentials: { + credentials?: { /** * Account access name * @visibility secret