paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 35 Packages Projects Releases Wiki Activity

Update all Publishers so that they resolve a list of objects on successful publish.

Browse Source 
Signed-off-by: Eric Peterson <ericpeterson@spotify.com>
This commit is contained in:
Eric Peterson
2021-11-12 10:43:05 +01:00
committed by Eric Peterson
parent 0a44eb7d52
commit 8b438c7717
10 changed files with 97 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/techdocs-common/src/stages/publish/awsS3.test.ts
+7 -1
View File
@@ -171,7 +171,13 @@ describe('AwsS3Publish', () => {
bucketRootPath: 'backstage-data/techdocs',
legacyUseCaseSensitiveTripletPaths: true,
});
expect(await publisher.publish({ entity, directory })).toBeUndefined();
expect(await publisher.publish({ entity, directory })).toMatchObject({
objects: expect.arrayContaining([
'test-namespace/TestKind/test-component-name/404.html',
`test-namespace/TestKind/test-component-name/index.html`,
`test-namespace/TestKind/test-component-name/assets/main.css`,
]),
});
});
it('should publish a directory when sse is specified', async () => {
packages/techdocs-common/src/stages/publish/awsS3.ts
+8 -1
View File
@@ -39,6 +39,7 @@ import {
import {
PublisherBase,
PublishRequest,
PublishResponse,
ReadinessResponse,
TechDocsMetadata,
} from './types';
@@ -214,7 +215,11 @@ export class AwsS3Publish implements PublisherBase {
* Upload all the files from the generated `directory` to the S3 bucket.
* Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html
*/
async publish({ entity, directory }: PublishRequest): Promise<void> {
async publish({
entity,
directory,
}: PublishRequest): Promise<PublishResponse> {
const objects: string[] = [];
const useLegacyPathCasing = this.legacyPathCasing;
const bucketRootPath = this.bucketRootPath;
const sse = this.sse;
@@ -263,6 +268,7 @@ export class AwsS3Publish implements PublisherBase {
...(sse && { ServerSideEncryption: sse }),
} as aws.S3.PutObjectRequest;
objects.push(params.Key);
return this.storageClient.upload(params).promise();
},
absoluteFilesToUpload,
@@ -311,6 +317,7 @@ export class AwsS3Publish implements PublisherBase {
const errorMessage = `Unable to delete file(s) from AWS S3. ${error}`;
this.logger.error(errorMessage);
}
return { objects };
}
async fetchTechDocsMetadata(
packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts
+7 -1
View File
@@ -161,7 +161,13 @@ describe('AzureBlobStoragePublish', () => {
const publisher = createPublisherFromConfig({
legacyUseCaseSensitiveTripletPaths: true,
});
expect(await publisher.publish({ entity, directory })).toBeUndefined();
expect(await publisher.publish({ entity, directory })).toMatchObject({
objects: expect.arrayContaining([
'test-namespace/TestKind/test-component-name/404.html',
`test-namespace/TestKind/test-component-name/index.html`,
`test-namespace/TestKind/test-component-name/assets/main.css`,
]),
});
});
it('should fail to publish a directory', async () => {
packages/techdocs-common/src/stages/publish/azureBlobStorage.ts
+15 -8
View File
@@ -39,6 +39,7 @@ import {
import {
PublisherBase,
PublishRequest,
PublishResponse,
ReadinessResponse,
TechDocsMetadata,
} from './types';
@@ -156,7 +157,11 @@ export class AzureBlobStoragePublish implements PublisherBase {
* Upload all the files from the generated `directory` to the Azure Blob Storage container.
* Directory structure used in the container is - entityNamespace/entityKind/entityName/index.html
*/
async publish({ entity, directory }: PublishRequest): Promise<void> {
async publish({
entity,
directory,
}: PublishRequest): Promise<PublishResponse> {
const objects: string[] = [];
const useLegacyPathCasing = this.legacyPathCasing;
// First, try to retrieve a list of all individual files currently existing
@@ -194,14 +199,14 @@ export class AzureBlobStoragePublish implements PublisherBase {
const relativeFilePath = path.normalize(
path.relative(directory, absoluteFilePath),
);
const remotePath = getCloudPathForLocalPath(
entity,
relativeFilePath,
useLegacyPathCasing,
);
objects.push(remotePath);
const response = await container
.getBlockBlobClient(
getCloudPathForLocalPath(
entity,
relativeFilePath,
useLegacyPathCasing,
),
)
.getBlockBlobClient(remotePath)
.uploadFile(absoluteFilePath);
if (response._response.status >= 400) {
@@ -264,6 +269,8 @@ export class AzureBlobStoragePublish implements PublisherBase {
const errorMessage = `Unable to delete file(s) from Azure. ${error}`;
this.logger.error(errorMessage);
}
return { objects };
}
private download(containerName: string, blobPath: string): Promise<Buffer> {
packages/techdocs-common/src/stages/publish/googleStorage.test.ts
+7 -1
View File
@@ -164,7 +164,13 @@ describe('GoogleGCSPublish', () => {
bucketRootPath: 'backstage-data/techdocs',
legacyUseCaseSensitiveTripletPaths: true,
});
expect(await publisher.publish({ entity, directory })).toBeUndefined();
expect(await publisher.publish({ entity, directory })).toMatchObject({
objects: expect.arrayContaining([
'test-namespace/TestKind/test-component-name/404.html',
`test-namespace/TestKind/test-component-name/index.html`,
`test-namespace/TestKind/test-component-name/assets/main.css`,
]),
});
});
it('should fail to publish a directory', async () => {
packages/techdocs-common/src/stages/publish/googleStorage.ts
+16 -9
View File
@@ -36,6 +36,7 @@ import { MigrateWriteStream } from './migrations';
import {
PublisherBase,
PublishRequest,
PublishResponse,
ReadinessResponse,
TechDocsMetadata,
} from './types';
@@ -145,7 +146,11 @@ export class GoogleGCSPublish implements PublisherBase {
* Upload all the files from the generated `directory` to the GCS bucket.
* Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html
*/
async publish({ entity, directory }: PublishRequest): Promise<void> {
async publish({
entity,
directory,
}: PublishRequest): Promise<PublishResponse> {
const objects: string[] = [];
const useLegacyPathCasing = this.legacyPathCasing;
const bucket = this.storageClient.bucket(this.bucketName);
const bucketRootPath = this.bucketRootPath;
@@ -178,14 +183,14 @@ export class GoogleGCSPublish implements PublisherBase {
await bulkStorageOperation(
async absoluteFilePath => {
const relativeFilePath = path.relative(directory, absoluteFilePath);
return await bucket.upload(absoluteFilePath, {
destination: getCloudPathForLocalPath(
entity,
relativeFilePath,
useLegacyPathCasing,
bucketRootPath,
),
});
const destination = getCloudPathForLocalPath(
entity,
relativeFilePath,
useLegacyPathCasing,
bucketRootPath,
);
objects.push(destination);
return await bucket.upload(absoluteFilePath, { destination });
},
absoluteFilesToUpload,
{ concurrencyLimit: 10 },
@@ -228,6 +233,8 @@ export class GoogleGCSPublish implements PublisherBase {
const errorMessage = `Unable to delete file(s) from Google Cloud Storage. ${error}`;
this.logger.error(errorMessage);
}
return { objects };
}
fetchTechDocsMetadata(entityName: EntityName): Promise<TechDocsMetadata> {
packages/techdocs-common/src/stages/publish/local.ts
+15 -10
View File
@@ -113,7 +113,7 @@ export class LocalPublish implements PublisherBase {
}
return new Promise((resolve, reject) => {
fs.copy(directory, publishDir, err => {
fs.copy(directory, publishDir, async err => {
if (err) {
this.logger.debug(
`Failed to copy docs from ${directory} to ${publishDir}`,
@@ -121,16 +121,21 @@ export class LocalPublish implements PublisherBase {
reject(err);
}
this.logger.info(`Published site stored at ${publishDir}`);
this.discovery
.getBaseUrl('techdocs')
.then(techdocsApiUrl => {
resolve({
remoteUrl: `${techdocsApiUrl}/static/docs/${entity.metadata.name}`,
});
})
.catch(reason => {
reject(reason);
try {
const techdocsApiUrl = await this.discovery.getBaseUrl('techdocs');
const objects = (await getFileTreeRecursively(publishDir)).map(
abs => {
return abs.split(`${staticDocsDir}/`)[1];
},
);
resolve({
remoteUrl: `${techdocsApiUrl}/static/docs/${entity.metadata.name}`,
objects,
});
} catch (reason) {
reject(reason);
}
});
});
}
packages/techdocs-common/src/stages/publish/openStackSwift.test.ts
+7 -1
View File
@@ -170,7 +170,13 @@ describe('OpenStackSwiftPublish', () => {
entity,
directory: entityRootDir,
}),
).toBeUndefined();
).toMatchObject({
objects: expect.arrayContaining([
'test-namespace/TestKind/test-component-name/404.html',
`test-namespace/TestKind/test-component-name/index.html`,
`test-namespace/TestKind/test-component-name/assets/main.css`,
]),
});
});
it('should fail to publish a directory', async () => {
packages/techdocs-common/src/stages/publish/openStackSwift.ts
+9 -2
View File
@@ -32,6 +32,7 @@ import {
import {
PublisherBase,
PublishRequest,
PublishResponse,
ReadinessResponse,
TechDocsMetadata,
} from './types';
@@ -139,8 +140,13 @@ export class OpenStackSwiftPublish implements PublisherBase {
* Upload all the files from the generated `directory` to the OpenStack Swift container.
* Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html
*/
async publish({ entity, directory }: PublishRequest): Promise<void> {
async publish({
entity,
directory,
}: PublishRequest): Promise<PublishResponse> {
try {
const objects: string[] = [];
// Note: OpenStack Swift manages creation of parent directories if they do not exist.
// So collecting path of only the files is good enough.
const allFilesToUpload = await getFileTreeRecursively(directory);
@@ -161,6 +167,7 @@ export class OpenStackSwiftPublish implements PublisherBase {
// The / delimiter is intentional since it represents the cloud storage and not the local file system.
const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
const destination = `${entityRootDir}/${relativeFilePathPosix}`; // Swift container file relative path
objects.push(destination);
// Rate limit the concurrent execution of file uploads to batches of 10 (per publish)
const uploadFile = limiter(async () => {
@@ -178,7 +185,7 @@ export class OpenStackSwiftPublish implements PublisherBase {
this.logger.info(
`Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`,
);
return;
return { objects };
} catch (e) {
const errorMessage = `Unable to upload file(s) to OpenStack Swift. ${e}`;
this.logger.error(errorMessage);
packages/techdocs-common/src/stages/publish/types.ts
+6
View File
@@ -35,6 +35,12 @@ export type PublishRequest = {
/* `remoteUrl` is the URL which serves files from the local publisher's static directory. */
export type PublishResponse = {
remoteUrl?: string;
/**
* The list of objects (specifically their paths) that were published.
* Objects should not have a preceding slash, and should match how one would
* load the object over the `/static/docs/` TechDocs Backend Plugin endpoint.
*/
objects?: string[];
} | void;
/**