diff --git a/packages/techdocs-common/src/stages/publish/awsS3.test.ts b/packages/techdocs-common/src/stages/publish/awsS3.test.ts index 1f49668d35..abae5fda44 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.test.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.test.ts @@ -171,7 +171,13 @@ describe('AwsS3Publish', () => { bucketRootPath: 'backstage-data/techdocs', legacyUseCaseSensitiveTripletPaths: true, }); - expect(await publisher.publish({ entity, directory })).toBeUndefined(); + expect(await publisher.publish({ entity, directory })).toMatchObject({ + objects: expect.arrayContaining([ + 'test-namespace/TestKind/test-component-name/404.html', + `test-namespace/TestKind/test-component-name/index.html`, + `test-namespace/TestKind/test-component-name/assets/main.css`, + ]), + }); }); it('should publish a directory when sse is specified', async () => { diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts index abfa8ddd24..ed76edc0cb 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.ts @@ -39,6 +39,7 @@ import { import { PublisherBase, PublishRequest, + PublishResponse, ReadinessResponse, TechDocsMetadata, } from './types'; @@ -214,7 +215,11 @@ export class AwsS3Publish implements PublisherBase { * Upload all the files from the generated `directory` to the S3 bucket. * Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html */ - async publish({ entity, directory }: PublishRequest): Promise { + async publish({ + entity, + directory, + }: PublishRequest): Promise { + const objects: string[] = []; const useLegacyPathCasing = this.legacyPathCasing; const bucketRootPath = this.bucketRootPath; const sse = this.sse; @@ -263,6 +268,7 @@ export class AwsS3Publish implements PublisherBase { ...(sse && { ServerSideEncryption: sse }), } as aws.S3.PutObjectRequest; + objects.push(params.Key); return this.storageClient.upload(params).promise(); }, absoluteFilesToUpload, @@ -311,6 +317,7 @@ export class AwsS3Publish implements PublisherBase { const errorMessage = `Unable to delete file(s) from AWS S3. ${error}`; this.logger.error(errorMessage); } + return { objects }; } async fetchTechDocsMetadata( diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts index a2503bd7b2..0e261f7f86 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts @@ -161,7 +161,13 @@ describe('AzureBlobStoragePublish', () => { const publisher = createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); - expect(await publisher.publish({ entity, directory })).toBeUndefined(); + expect(await publisher.publish({ entity, directory })).toMatchObject({ + objects: expect.arrayContaining([ + 'test-namespace/TestKind/test-component-name/404.html', + `test-namespace/TestKind/test-component-name/index.html`, + `test-namespace/TestKind/test-component-name/assets/main.css`, + ]), + }); }); it('should fail to publish a directory', async () => { diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts index 7a58d92095..b082079be2 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts @@ -39,6 +39,7 @@ import { import { PublisherBase, PublishRequest, + PublishResponse, ReadinessResponse, TechDocsMetadata, } from './types'; @@ -156,7 +157,11 @@ export class AzureBlobStoragePublish implements PublisherBase { * Upload all the files from the generated `directory` to the Azure Blob Storage container. * Directory structure used in the container is - entityNamespace/entityKind/entityName/index.html */ - async publish({ entity, directory }: PublishRequest): Promise { + async publish({ + entity, + directory, + }: PublishRequest): Promise { + const objects: string[] = []; const useLegacyPathCasing = this.legacyPathCasing; // First, try to retrieve a list of all individual files currently existing @@ -194,14 +199,14 @@ export class AzureBlobStoragePublish implements PublisherBase { const relativeFilePath = path.normalize( path.relative(directory, absoluteFilePath), ); + const remotePath = getCloudPathForLocalPath( + entity, + relativeFilePath, + useLegacyPathCasing, + ); + objects.push(remotePath); const response = await container - .getBlockBlobClient( - getCloudPathForLocalPath( - entity, - relativeFilePath, - useLegacyPathCasing, - ), - ) + .getBlockBlobClient(remotePath) .uploadFile(absoluteFilePath); if (response._response.status >= 400) { @@ -264,6 +269,8 @@ export class AzureBlobStoragePublish implements PublisherBase { const errorMessage = `Unable to delete file(s) from Azure. ${error}`; this.logger.error(errorMessage); } + + return { objects }; } private download(containerName: string, blobPath: string): Promise { diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts index a52fd0f0bd..ea97d81269 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts @@ -164,7 +164,13 @@ describe('GoogleGCSPublish', () => { bucketRootPath: 'backstage-data/techdocs', legacyUseCaseSensitiveTripletPaths: true, }); - expect(await publisher.publish({ entity, directory })).toBeUndefined(); + expect(await publisher.publish({ entity, directory })).toMatchObject({ + objects: expect.arrayContaining([ + 'test-namespace/TestKind/test-component-name/404.html', + `test-namespace/TestKind/test-component-name/index.html`, + `test-namespace/TestKind/test-component-name/assets/main.css`, + ]), + }); }); it('should fail to publish a directory', async () => { diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index f072412c20..ba70c36e27 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -36,6 +36,7 @@ import { MigrateWriteStream } from './migrations'; import { PublisherBase, PublishRequest, + PublishResponse, ReadinessResponse, TechDocsMetadata, } from './types'; @@ -145,7 +146,11 @@ export class GoogleGCSPublish implements PublisherBase { * Upload all the files from the generated `directory` to the GCS bucket. * Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html */ - async publish({ entity, directory }: PublishRequest): Promise { + async publish({ + entity, + directory, + }: PublishRequest): Promise { + const objects: string[] = []; const useLegacyPathCasing = this.legacyPathCasing; const bucket = this.storageClient.bucket(this.bucketName); const bucketRootPath = this.bucketRootPath; @@ -178,14 +183,14 @@ export class GoogleGCSPublish implements PublisherBase { await bulkStorageOperation( async absoluteFilePath => { const relativeFilePath = path.relative(directory, absoluteFilePath); - return await bucket.upload(absoluteFilePath, { - destination: getCloudPathForLocalPath( - entity, - relativeFilePath, - useLegacyPathCasing, - bucketRootPath, - ), - }); + const destination = getCloudPathForLocalPath( + entity, + relativeFilePath, + useLegacyPathCasing, + bucketRootPath, + ); + objects.push(destination); + return await bucket.upload(absoluteFilePath, { destination }); }, absoluteFilesToUpload, { concurrencyLimit: 10 }, @@ -228,6 +233,8 @@ export class GoogleGCSPublish implements PublisherBase { const errorMessage = `Unable to delete file(s) from Google Cloud Storage. ${error}`; this.logger.error(errorMessage); } + + return { objects }; } fetchTechDocsMetadata(entityName: EntityName): Promise { diff --git a/packages/techdocs-common/src/stages/publish/local.ts b/packages/techdocs-common/src/stages/publish/local.ts index 70b4eb3ff2..67a4a22914 100644 --- a/packages/techdocs-common/src/stages/publish/local.ts +++ b/packages/techdocs-common/src/stages/publish/local.ts @@ -113,7 +113,7 @@ export class LocalPublish implements PublisherBase { } return new Promise((resolve, reject) => { - fs.copy(directory, publishDir, err => { + fs.copy(directory, publishDir, async err => { if (err) { this.logger.debug( `Failed to copy docs from ${directory} to ${publishDir}`, @@ -121,16 +121,21 @@ export class LocalPublish implements PublisherBase { reject(err); } this.logger.info(`Published site stored at ${publishDir}`); - this.discovery - .getBaseUrl('techdocs') - .then(techdocsApiUrl => { - resolve({ - remoteUrl: `${techdocsApiUrl}/static/docs/${entity.metadata.name}`, - }); - }) - .catch(reason => { - reject(reason); + + try { + const techdocsApiUrl = await this.discovery.getBaseUrl('techdocs'); + const objects = (await getFileTreeRecursively(publishDir)).map( + abs => { + return abs.split(`${staticDocsDir}/`)[1]; + }, + ); + resolve({ + remoteUrl: `${techdocsApiUrl}/static/docs/${entity.metadata.name}`, + objects, }); + } catch (reason) { + reject(reason); + } }); }); } diff --git a/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts b/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts index c2236fb880..b8ad43aac3 100644 --- a/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts +++ b/packages/techdocs-common/src/stages/publish/openStackSwift.test.ts @@ -170,7 +170,13 @@ describe('OpenStackSwiftPublish', () => { entity, directory: entityRootDir, }), - ).toBeUndefined(); + ).toMatchObject({ + objects: expect.arrayContaining([ + 'test-namespace/TestKind/test-component-name/404.html', + `test-namespace/TestKind/test-component-name/index.html`, + `test-namespace/TestKind/test-component-name/assets/main.css`, + ]), + }); }); it('should fail to publish a directory', async () => { diff --git a/packages/techdocs-common/src/stages/publish/openStackSwift.ts b/packages/techdocs-common/src/stages/publish/openStackSwift.ts index 7b623933e3..62b40f9a76 100644 --- a/packages/techdocs-common/src/stages/publish/openStackSwift.ts +++ b/packages/techdocs-common/src/stages/publish/openStackSwift.ts @@ -32,6 +32,7 @@ import { import { PublisherBase, PublishRequest, + PublishResponse, ReadinessResponse, TechDocsMetadata, } from './types'; @@ -139,8 +140,13 @@ export class OpenStackSwiftPublish implements PublisherBase { * Upload all the files from the generated `directory` to the OpenStack Swift container. * Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html */ - async publish({ entity, directory }: PublishRequest): Promise { + async publish({ + entity, + directory, + }: PublishRequest): Promise { try { + const objects: string[] = []; + // Note: OpenStack Swift manages creation of parent directories if they do not exist. // So collecting path of only the files is good enough. const allFilesToUpload = await getFileTreeRecursively(directory); @@ -161,6 +167,7 @@ export class OpenStackSwiftPublish implements PublisherBase { // The / delimiter is intentional since it represents the cloud storage and not the local file system. const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; const destination = `${entityRootDir}/${relativeFilePathPosix}`; // Swift container file relative path + objects.push(destination); // Rate limit the concurrent execution of file uploads to batches of 10 (per publish) const uploadFile = limiter(async () => { @@ -178,7 +185,7 @@ export class OpenStackSwiftPublish implements PublisherBase { this.logger.info( `Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`, ); - return; + return { objects }; } catch (e) { const errorMessage = `Unable to upload file(s) to OpenStack Swift. ${e}`; this.logger.error(errorMessage); diff --git a/packages/techdocs-common/src/stages/publish/types.ts b/packages/techdocs-common/src/stages/publish/types.ts index 229c853427..f68cb9cc8f 100644 --- a/packages/techdocs-common/src/stages/publish/types.ts +++ b/packages/techdocs-common/src/stages/publish/types.ts @@ -35,6 +35,12 @@ export type PublishRequest = { /* `remoteUrl` is the URL which serves files from the local publisher's static directory. */ export type PublishResponse = { remoteUrl?: string; + /** + * The list of objects (specifically their paths) that were published. + * Objects should not have a preceding slash, and should match how one would + * load the object over the `/static/docs/` TechDocs Backend Plugin endpoint. + */ + objects?: string[]; } | void; /**