paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 54 Packages Projects Releases Wiki Activity

Merge branch 'master' into techdocs-common/case-sensitive-migration

Browse Source 
Signed-off-by: Eric Peterson <ericpeterson@spotify.com>
This commit is contained in:
Eric Peterson
2021-08-12 11:27:43 +02:00
parent add5ee5d97 0c58dd73d0
commit 52547371fb
635 changed files with 9792 additions and 6037 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/techdocs-common/CHANGELOG.md
+24
View File
@@ -1,5 +1,29 @@
# @backstage/techdocs-common
## 0.8.0
### Minor Changes
- 48ea3d25b: TechDocs has dropped all support for the long-ago deprecated git-based common
prepares as well as all corresponding values in `backstage.io/techdocs-ref`
annotations.
Entities whose `backstage.io/techdocs-ref` annotation values still begin with
`github:`, `gitlab:`, `bitbucket:`, or `azure/api:` will no longer be generated
by TechDocs. Be sure to update these values so that they align with their
expected format and your usage of TechDocs.
For details, see [this explainer on TechDocs ref annotation values][how].
[how]: https://backstage.io/docs/features/techdocs/how-to-guides#how-to-understand-techdocs-ref-annotation-values
### Patch Changes
- Updated dependencies
- @backstage/backend-common@0.8.8
- @backstage/config@0.1.6
- @backstage/integration@0.5.9
## 0.7.1
### Patch Changes
packages/techdocs-common/__mocks__/@azure/storage-blob.ts
+44
View File
@@ -78,6 +78,36 @@ class BlockBlobClientFailUpload extends BlockBlobClient {
}
}
class ContainerClientIterator {
private containerName: string;
constructor(containerName) {
this.containerName = containerName;
}
async next() {
if (
this.containerName === 'delete_stale_files_success' ||
this.containerName === 'delete_stale_files_error'
) {
return {
value: {
segment: {
blobItems: [{ name: `stale_file.png` }],
},
},
};
}
return {
value: {
segment: {
blobItems: [],
},
},
};
}
}
export class ContainerClient {
getProperties(): Promise<ContainerGetPropertiesResponse> {
return Promise.resolve({
@@ -95,6 +125,20 @@ export class ContainerClient {
getBlockBlobClient(blobName: string) {
return new BlockBlobClient(blobName);
}
listBlobsFlat() {
return {
byPage: () => {
return new ContainerClientIterator(this.containerName);
},
};
}
deleteBlob() {
if (this.containerName === 'delete_stale_files_error') {
throw new Error('Message');
}
}
}
class ContainerClientFailGetProperties extends ContainerClient {
packages/techdocs-common/__mocks__/@google-cloud/storage.ts
+24
View File
@@ -55,6 +55,10 @@ class GCSFile {
return readable;
}
delete() {
return Promise.resolve();
}
}
class Bucket {
@@ -79,8 +83,28 @@ class Bucket {
}
file(destinationFilePath: string) {
if (this.bucketName === 'delete_stale_files_error') {
throw Error('Message');
}
return new GCSFile(destinationFilePath);
}
getFilesStream() {
const readable = new Readable();
readable._read = () => {};
process.nextTick(() => {
if (
this.bucketName === 'delete_stale_files_success' ||
this.bucketName === 'delete_stale_files_error'
) {
readable.emit('data', { name: 'stale-file.png' });
}
readable.emit('end');
});
return readable;
}
}
export class Storage {
packages/techdocs-common/__mocks__/aws-sdk.ts
+27
View File
@@ -79,6 +79,33 @@ export class S3 {
}),
};
}
listObjectsV2({ Bucket }) {
return {
promise: () => {
if (
Bucket === 'delete_stale_files_success' ||
Bucket === 'delete_stale_files_error'
) {
return Promise.resolve({
Contents: [{ Key: 'stale_file.png' }],
});
}
return Promise.resolve({});
},
};
}
deleteObject({ Bucket }) {
return {
promise: () => {
if (Bucket === 'delete_stale_files_error') {
throw new Error('Message');
}
return Promise.resolve();
},
};
}
}
export default {
packages/techdocs-common/package.json
+5 -5
View File
@@ -1,7 +1,7 @@
{
"name": "@backstage/techdocs-common",
"description": "Common functionalities for TechDocs, to be shared between techdocs-backend plugin and techdocs-cli",
"version": "0.7.1",
"version": "0.8.0",
"main": "src/index.ts",
"types": "src/index.ts",
"private": false,
@@ -38,11 +38,11 @@
"dependencies": {
"@azure/identity": "^1.5.0",
"@azure/storage-blob": "^12.5.0",
"@backstage/backend-common": "^0.8.7",
"@backstage/backend-common": "^0.8.8",
"@backstage/catalog-model": "^0.9.0",
"@backstage/config": "^0.1.5",
"@backstage/config": "^0.1.6",
"@backstage/errors": "^0.1.1",
"@backstage/integration": "^0.5.8",
"@backstage/integration": "^0.5.9",
"@google-cloud/storage": "^5.6.0",
"@types/express": "^4.17.6",
"aws-sdk": "^2.840.0",
@@ -58,7 +58,7 @@
"winston": "^3.2.1"
},
"devDependencies": {
"@backstage/cli": "^0.7.6",
"@backstage/cli": "^0.7.7",
"@types/fs-extra": "^9.0.5",
"@types/js-yaml": "^4.0.0",
"@types/mime-types": "^2.1.0",
packages/techdocs-common/src/stages/publish/awsS3.test.ts
+24 -4
View File
@@ -38,6 +38,8 @@ const getEntityRootDir = (entity: Entity) => {
};
const logger = getVoidLogger();
const loggerInfoSpy = jest.spyOn(logger, 'info');
const loggerErrorSpy = jest.spyOn(logger, 'error');
const createPublisherFromConfig = ({
bucketName = 'bucketName',
@@ -46,7 +48,7 @@ const createPublisherFromConfig = ({
bucketName?: string;
legacyUseCaseSensitiveTripletPaths?: boolean;
} = {}) => {
const config = new ConfigReader({
const mockConfig = new ConfigReader({
techdocs: {
requestUrl: 'http://localhost:7000',
publisher: {
@@ -63,7 +65,7 @@ const createPublisherFromConfig = ({
},
});
return AwsS3Publish.fromConfig(config, logger);
return AwsS3Publish.fromConfig(mockConfig, logger);
};
describe('AwsS3Publish', () => {
@@ -110,13 +112,13 @@ describe('AwsS3Publish', () => {
},
};
beforeAll(() => {
beforeEach(() => {
mockFs({
[directory]: files,
});
});
afterAll(() => {
afterEach(() => {
mockFs.restore();
});
@@ -177,6 +179,24 @@ describe('AwsS3Publish', () => {
message: expect.stringContaining(wrongPathToGeneratedDirectory),
});
});
it('should delete stale files after upload', async () => {
const bucketName = 'delete_stale_files_success';
const publisher = createPublisherFromConfig({ bucketName: bucketName });
await publisher.publish({ entity, directory });
expect(loggerInfoSpy).toHaveBeenLastCalledWith(
`Successfully deleted stale files for Entity ${entity.metadata.name}. Total number of files: 1`,
);
});
it('should log error when the stale files deletion fails', async () => {
const bucketName = 'delete_stale_files_error';
const publisher = createPublisherFromConfig({ bucketName: bucketName });
await publisher.publish({ entity, directory });
expect(loggerErrorSpy).toHaveBeenLastCalledWith(
'Unable to delete file(s) from AWS S3. Error: Message',
);
});
});
describe('hasDocsBeenGenerated', () => {
packages/techdocs-common/src/stages/publish/awsS3.ts
+81 -46
View File
@@ -13,14 +13,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
Entity,
EntityName,
ENTITY_DEFAULT_NAMESPACE,
} from '@backstage/catalog-model';
import { Entity, EntityName } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import aws, { Credentials } from 'aws-sdk';
import { ListObjectsV2Output, ManagedUpload } from 'aws-sdk/clients/s3';
import { ListObjectsV2Output } from 'aws-sdk/clients/s3';
import { CredentialsOptions } from 'aws-sdk/lib/credentials';
import express from 'express';
import fs from 'fs-extra';
@@ -30,8 +26,11 @@ import path from 'path';
import { Readable } from 'stream';
import { Logger } from 'winston';
import {
bulkStorageOperation,
getCloudPathForLocalPath,
getFileTreeRecursively,
getHeadersForFileExtension,
getStaleFiles,
lowerCaseEntityTriplet,
lowerCaseEntityTripletInStoragePath,
} from './helpers';
@@ -192,60 +191,94 @@ export class AwsS3Publish implements PublisherBase {
* Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html
*/
async publish({ entity, directory }: PublishRequest): Promise<void> {
const useLegacyPathCasing = this.legacyPathCasing;
// First, try to retrieve a list of all individual files currently existing
let existingFiles: string[] = [];
try {
// Note: S3 manages creation of parent directories if they do not exist.
// So collecting path of only the files is good enough.
const allFilesToUpload = await getFileTreeRecursively(directory);
const remoteFolder = getCloudPathForLocalPath(
entity,
undefined,
useLegacyPathCasing,
);
existingFiles = await this.getAllObjectsFromBucket({
prefix: remoteFolder,
});
} catch (e) {
this.logger.error(
`Unable to list files for Entity ${entity.metadata.name}: ${e.message}`,
);
}
const limiter = createLimiter(10);
const uploadPromises: Array<Promise<ManagedUpload.SendData>> = [];
for (const filePath of allFilesToUpload) {
// Remove the absolute path prefix of the source directory
// Path of all files to upload, relative to the root of the source directory
// e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png']
const relativeFilePath = path.relative(directory, filePath);
// Then, merge new files into the same folder
let absoluteFilesToUpload;
try {
// Remove the absolute path prefix of the source directory
// Path of all files to upload, relative to the root of the source directory
// e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png']
absoluteFilesToUpload = await getFileTreeRecursively(directory);
// Convert destination file path to a POSIX path for uploading.
// S3 expects / as path separator and relativeFilePath will contain \\ on Windows.
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html
const relativeFilePathPosix = relativeFilePath
.split(path.sep)
.join(path.posix.sep);
// The / delimiter is intentional since it represents the cloud storage and not the local file system.
const entityRootDir = `${
entity.metadata?.namespace ?? ENTITY_DEFAULT_NAMESPACE
}/${entity.kind}/${entity.metadata.name}`;
const relativeFilePathTriplet = `${entityRootDir}/${relativeFilePathPosix}`;
const destination = this.legacyPathCasing
? relativeFilePathTriplet
: lowerCaseEntityTripletInStoragePath(relativeFilePathTriplet); // S3 Bucket file relative path
// Rate limit the concurrent execution of file uploads to batches of 10 (per publish)
const uploadFile = limiter(() => {
const fileStream = fs.createReadStream(filePath);
await bulkStorageOperation(
async absoluteFilePath => {
const relativeFilePath = path.relative(directory, absoluteFilePath);
const fileStream = fs.createReadStream(absoluteFilePath);
const params = {
Bucket: this.bucketName,
Key: destination,
Key: getCloudPathForLocalPath(
entity,
relativeFilePath,
useLegacyPathCasing,
),
Body: fileStream,
};
return this.storageClient.upload(params).promise();
});
uploadPromises.push(uploadFile);
}
await Promise.all(uploadPromises);
this.logger.info(
`Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`,
},
absoluteFilesToUpload,
{ concurrencyLimit: 10 },
);
this.logger.info(
`Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${absoluteFilesToUpload.length}`,
);
return;
} catch (e) {
const errorMessage = `Unable to upload file(s) to AWS S3. ${e}`;
this.logger.error(errorMessage);
throw new Error(errorMessage);
}
// Last, try to remove the files that were *only* present previously
try {
const relativeFilesToUpload = absoluteFilesToUpload.map(
absoluteFilePath =>
getCloudPathForLocalPath(
entity,
path.relative(directory, absoluteFilePath),
useLegacyPathCasing,
),
);
const staleFiles = getStaleFiles(relativeFilesToUpload, existingFiles);
await bulkStorageOperation(
async relativeFilePath => {
return await this.storageClient
.deleteObject({
Bucket: this.bucketName,
Key: relativeFilePath,
})
.promise();
},
staleFiles,
{ concurrencyLimit: 10 },
);
this.logger.info(
`Successfully deleted stale files for Entity ${entity.metadata.name}. Total number of files: ${staleFiles.length}`,
);
} catch (error) {
const errorMessage = `Unable to delete file(s) from AWS S3. ${error}`;
this.logger.error(errorMessage);
}
}
async fetchTechDocsMetadata(
@@ -399,17 +432,19 @@ export class AwsS3Publish implements PublisherBase {
/**
* Returns a list of all object keys from the configured bucket.
*/
protected async getAllObjectsFromBucket(): Promise<string[]> {
protected async getAllObjectsFromBucket(
{ prefix } = { prefix: '' },
): Promise<string[]> {
const objects: string[] = [];
let nextContinuation: string | undefined;
let allObjects: ListObjectsV2Output;
// Iterate through every file in the root of the publisher.
do {
allObjects = await this.storageClient
.listObjectsV2({
Bucket: this.bucketName,
ContinuationToken: nextContinuation,
...(prefix ? { Prefix: prefix } : {}),
})
.promise();
objects.push(
packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts
+11 -10
View File
@@ -65,7 +65,6 @@ const createPublisherFromConfig = ({
legacyUseCaseSensitiveTripletPaths,
},
});
return AzureBlobStoragePublish.fromConfig(config, logger);
};
@@ -117,13 +116,13 @@ describe('AzureBlobStoragePublish', () => {
},
};
beforeAll(async () => {
beforeEach(async () => {
mockFs({
[directory]: files,
});
});
afterAll(() => {
afterEach(() => {
mockFs.restore();
});
@@ -136,11 +135,11 @@ describe('AzureBlobStoragePublish', () => {
});
it('should reject incorrect config', async () => {
const publisher = createPublisherFromConfig({
const errorPublisher = createPublisherFromConfig({
containerName: 'bad_container',
});
expect(await publisher.getReadiness()).toEqual({
expect(await errorPublisher.getReadiness()).toEqual({
isAvailable: false,
});
@@ -185,7 +184,7 @@ describe('AzureBlobStoragePublish', () => {
await expect(fails).rejects.toMatchObject({
message: expect.stringContaining(
`Unable to upload file(s) to Azure Blob Storage. Error: Failed to read template directory: ENOENT, no such file or directory`,
`Unable to upload file(s) to Azure. Error: Failed to read template directory: ENOENT, no such file or directory`,
),
});
@@ -206,13 +205,11 @@ describe('AzureBlobStoragePublish', () => {
error = e;
}
expect(error.message).toContain(
`Unable to upload file(s) to Azure Blob Storage.`,
);
expect(error.message).toContain(`Unable to upload file(s) to Azure`);
expect(logger.error).toHaveBeenCalledWith(
expect.stringContaining(
`Unable to upload file(s) to Azure Blob Storage. Error: Upload failed for ${path.join(
`Unable to upload file(s) to Azure. Error: Upload failed for ${path.join(
directory,
'404.html',
)} with status code 500`,
@@ -324,6 +321,10 @@ describe('AzureBlobStoragePublish', () => {
app = express().use(publisher.docsRouter());
});
afterEach(() => {
mockFs.restore();
});
it('should pass expected object path to bucket', async () => {
// Ensures leading slash is trimmed and encoded path is decoded.
const pngResponse = await request(app).get(
packages/techdocs-common/src/stages/publish/azureBlobStorage.ts
+119 -63
View File
@@ -16,13 +16,10 @@
import { DefaultAzureCredential } from '@azure/identity';
import {
BlobServiceClient,
ContainerClient,
StorageSharedKeyCredential,
} from '@azure/storage-blob';
import {
Entity,
EntityName,
ENTITY_DEFAULT_NAMESPACE,
} from '@backstage/catalog-model';
import { Entity, EntityName } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import express from 'express';
import JSON5 from 'json5';
@@ -30,9 +27,12 @@ import limiterFactory from 'p-limit';
import { default as path, default as platformPath } from 'path';
import { Logger } from 'winston';
import {
bulkStorageOperation,
getCloudPathForLocalPath,
getFileTreeRecursively,
getHeadersForFileExtension,
lowerCaseEntityTriplet,
getStaleFiles,
lowerCaseEntityTripletInStoragePath,
} from './helpers';
import {
@@ -150,82 +150,112 @@ export class AzureBlobStoragePublish implements PublisherBase {
* Directory structure used in the container is - entityNamespace/entityKind/entityName/index.html
*/
async publish({ entity, directory }: PublishRequest): Promise<void> {
const useLegacyPathCasing = this.legacyPathCasing;
// First, try to retrieve a list of all individual files currently existing
const remoteFolder = getCloudPathForLocalPath(
entity,
undefined,
useLegacyPathCasing,
);
let existingFiles: string[] = [];
try {
// Note: Azure Blob Storage manages creation of parent directories if they do not exist.
// So collecting path of only the files is good enough.
const allFilesToUpload = await getFileTreeRecursively(directory);
existingFiles = await this.getAllBlobsFromContainer({
prefix: remoteFolder,
maxPageSize: BATCH_CONCURRENCY,
});
} catch (e) {
this.logger.error(
`Unable to list files for Entity ${entity.metadata.name}: ${e.message}`,
);
}
// Bound the number of concurrent batches. We want a bit of concurrency for
// performance reasons, but not so much that we starve the connection pool
// or start thrashing.
const limiter = limiterFactory(BATCH_CONCURRENCY);
// Then, merge new files into the same folder
let absoluteFilesToUpload;
let container: ContainerClient;
try {
// Remove the absolute path prefix of the source directory
// Path of all files to upload, relative to the root of the source directory
// e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png']
absoluteFilesToUpload = await getFileTreeRecursively(directory);
const promises = allFilesToUpload.map(sourceFilePath => {
// Remove the absolute path prefix of the source directory
// Path of all files to upload, relative to the root of the source directory
// e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png']
const relativeFilePath = path.normalize(
path.relative(directory, sourceFilePath),
);
// Convert destination file path to a POSIX path for uploading.
// Azure Blob Storage expects / as path separator and relativeFilePath will contain \\ on Windows.
// https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#blob-names
const relativeFilePathPosix = relativeFilePath
.split(path.sep)
.join(path.posix.sep);
// The / delimiter is intentional since it represents the cloud storage and not the local file system.
const entityRootDir = `${
entity.metadata?.namespace ?? ENTITY_DEFAULT_NAMESPACE
}/${entity.kind}/${entity.metadata.name}`;
const relativeFilePathTriplet = `${entityRootDir}/${relativeFilePathPosix}`;
const destination = this.legacyPathCasing
? relativeFilePathTriplet
: lowerCaseEntityTripletInStoragePath(relativeFilePathTriplet); // Azure Blob Storage Container file relative path
return limiter(async () => {
const response = await this.storageClient
.getContainerClient(this.containerName)
.getBlockBlobClient(destination)
.uploadFile(sourceFilePath);
container = this.storageClient.getContainerClient(this.containerName);
const failedOperations: Error[] = [];
await bulkStorageOperation(
async absoluteFilePath => {
const relativeFilePath = path.normalize(
path.relative(directory, absoluteFilePath),
);
const response = await container
.getBlockBlobClient(
getCloudPathForLocalPath(
entity,
relativeFilePath,
useLegacyPathCasing,
),
)
.uploadFile(absoluteFilePath);
if (response._response.status >= 400) {
return {
...response,
error: new Error(
`Upload failed for ${sourceFilePath} with status code ${response._response.status}`,
failedOperations.push(
new Error(
`Upload failed for ${absoluteFilePath} with status code ${response._response.status}`,
),
};
);
}
return {
...response,
error: undefined,
};
});
});
const responses = await Promise.all(promises);
return response;
},
absoluteFilesToUpload,
{ concurrencyLimit: BATCH_CONCURRENCY },
);
const failed = responses.filter(r => r.error);
if (failed.length === 0) {
this.logger.info(
`Successfully uploaded the ${responses.length} generated file(s) for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`,
);
} else {
if (failedOperations.length > 0) {
throw new Error(
failed
.map(r => r.error?.message)
failedOperations
.map(r => r.message)
.filter(Boolean)
.join(' '),
);
}
this.logger.info(
`Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${absoluteFilesToUpload.length}`,
);
} catch (e) {
const errorMessage = `Unable to upload file(s) to Azure Blob Storage. ${e}`;
const errorMessage = `Unable to upload file(s) to Azure. ${e}`;
this.logger.error(errorMessage);
throw new Error(errorMessage);
}
// Last, try to remove the files that were *only* present previously
try {
const relativeFilesToUpload = absoluteFilesToUpload.map(
absoluteFilePath =>
getCloudPathForLocalPath(
entity,
path.relative(directory, absoluteFilePath),
useLegacyPathCasing,
),
);
const staleFiles = getStaleFiles(relativeFilesToUpload, existingFiles);
await bulkStorageOperation(
async relativeFilePath => {
return await container.deleteBlob(relativeFilePath);
},
staleFiles,
{ concurrencyLimit: BATCH_CONCURRENCY },
);
this.logger.info(
`Successfully deleted stale files for Entity ${entity.metadata.name}. Total number of files: ${staleFiles.length}`,
);
} catch (error) {
const errorMessage = `Unable to delete file(s) from Azure. ${error}`;
this.logger.error(errorMessage);
}
}
private download(containerName: string, blobPath: string): Promise<Buffer> {
@@ -387,4 +417,30 @@ export class AzureBlobStoragePublish implements PublisherBase {
await Promise.all(promises);
}
protected async getAllBlobsFromContainer({
prefix,
maxPageSize,
}: {
prefix: string;
maxPageSize: number;
}): Promise<string[]> {
const blobs: string[] = [];
const container = this.storageClient.getContainerClient(this.containerName);
let iterator = container.listBlobsFlat({ prefix }).byPage({ maxPageSize });
let response = (await iterator.next()).value;
do {
for (const blob of response?.segment?.blobItems ?? []) {
blobs.push(blob.name);
}
iterator = container
.listBlobsFlat({ prefix })
.byPage({ continuationToken: response.continuationToken, maxPageSize });
response = (await iterator.next()).value;
} while (response && response.continuationToken);
return blobs;
}
}
packages/techdocs-common/src/stages/publish/googleStorage.test.ts
+20 -1
View File
@@ -38,6 +38,8 @@ const getEntityRootDir = (entity: Entity) => {
};
const logger = getVoidLogger();
jest.spyOn(logger, 'info').mockReturnValue(logger);
jest.spyOn(logger, 'error').mockReturnValue(logger);
const createPublisherFromConfig = ({
bucketName = 'bucketName',
@@ -59,7 +61,6 @@ const createPublisherFromConfig = ({
legacyUseCaseSensitiveTripletPaths,
},
});
return GoogleGCSPublish.fromConfig(config, logger);
};
@@ -176,6 +177,24 @@ describe('GoogleGCSPublish', () => {
message: expect.stringContaining(wrongPathToGeneratedDirectory),
});
});
it('should delete stale files after upload', async () => {
const bucketName = 'delete_stale_files_success';
const publisher = createPublisherFromConfig({ bucketName });
await publisher.publish({ entity, directory });
expect(logger.info).toHaveBeenLastCalledWith(
`Successfully deleted stale files for Entity ${entity.metadata.name}. Total number of files: 1`,
);
});
it('should log error when the stale files deletion fails', async () => {
const bucketName = 'delete_stale_files_error';
const publisher = createPublisherFromConfig({ bucketName });
await publisher.publish({ entity, directory });
expect(logger.error).toHaveBeenLastCalledWith(
'Unable to delete file(s) from Google Cloud Storage. Error: Message',
);
});
});
describe('hasDocsBeenGenerated', () => {
packages/techdocs-common/src/stages/publish/googleStorage.ts
+98 -52
View File
@@ -13,20 +13,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
Entity,
EntityName,
ENTITY_DEFAULT_NAMESPACE,
} from '@backstage/catalog-model';
import { Entity, EntityName } from '@backstage/catalog-model';
import { Config } from '@backstage/config';
import {
FileExistsResponse,
Storage,
UploadResponse,
} from '@google-cloud/storage';
import { File, FileExistsResponse, Storage } from '@google-cloud/storage';
import express from 'express';
import JSON5 from 'json5';
import createLimiter from 'p-limit';
import path from 'path';
import { Readable } from 'stream';
import { Logger } from 'winston';
@@ -35,6 +26,9 @@ import {
getHeadersForFileExtension,
lowerCaseEntityTriplet,
lowerCaseEntityTripletInStoragePath,
bulkStorageOperation,
getCloudPathForLocalPath,
getStaleFiles,
} from './helpers';
import { MigrateWriteStream } from './migrations';
import {
@@ -135,55 +129,83 @@ export class GoogleGCSPublish implements PublisherBase {
* Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html
*/
async publish({ entity, directory }: PublishRequest): Promise<void> {
const useLegacyPathCasing = this.legacyPathCasing;
const bucket = this.storageClient.bucket(this.bucketName);
// First, try to retrieve a list of all individual files currently existing
let existingFiles: string[] = [];
try {
// Note: GCS manages creation of parent directories if they do not exist.
// So collecting path of only the files is good enough.
const allFilesToUpload = await getFileTreeRecursively(directory);
const remoteFolder = getCloudPathForLocalPath(
entity,
undefined,
useLegacyPathCasing,
);
existingFiles = await this.getFilesForFolder(remoteFolder);
} catch (e) {
this.logger.error(
`Unable to list files for Entity ${entity.metadata.name}: ${e.message}`,
);
}
const limiter = createLimiter(10);
const uploadPromises: Array<Promise<UploadResponse>> = [];
allFilesToUpload.forEach(sourceFilePath => {
// Remove the absolute path prefix of the source directory
// Path of all files to upload, relative to the root of the source directory
// e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png']
const relativeFilePath = path.relative(directory, sourceFilePath);
// Then, merge new files into the same folder
let absoluteFilesToUpload;
try {
// Remove the absolute path prefix of the source directory
// Path of all files to upload, relative to the root of the source directory
// e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png']
absoluteFilesToUpload = await getFileTreeRecursively(directory);
// Convert destination file path to a POSIX path for uploading.
// GCS expects / as path separator and relativeFilePath will contain \\ on Windows.
// https://cloud.google.com/storage/docs/gsutil/addlhelp/HowSubdirectoriesWork
const relativeFilePathPosix = relativeFilePath
.split(path.sep)
.join(path.posix.sep);
// The / delimiter is intentional since it represents the cloud storage and not the local file system.
const entityRootDir = `${
entity.metadata?.namespace ?? ENTITY_DEFAULT_NAMESPACE
}/${entity.kind}/${entity.metadata.name}`;
const relativeFilePathTriplet = `${entityRootDir}/${relativeFilePathPosix}`;
const destination = this.legacyPathCasing
? relativeFilePathTriplet
: lowerCaseEntityTripletInStoragePath(relativeFilePathTriplet); // GCS Bucket file relative path
// Rate limit the concurrent execution of file uploads to batches of 10 (per publish)
const uploadFile = limiter(() =>
this.storageClient.bucket(this.bucketName).upload(sourceFilePath, {
destination,
}),
);
uploadPromises.push(uploadFile);
});
await Promise.all(uploadPromises);
await bulkStorageOperation(
async absoluteFilePath => {
const relativeFilePath = path.relative(directory, absoluteFilePath);
return await bucket.upload(absoluteFilePath, {
destination: getCloudPathForLocalPath(
entity,
relativeFilePath,
useLegacyPathCasing,
),
});
},
absoluteFilesToUpload,
{ concurrencyLimit: 10 },
);
this.logger.info(
`Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`,
`Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${absoluteFilesToUpload.length}`,
);
} catch (e) {
const errorMessage = `Unable to upload file(s) to Google Cloud Storage. ${e}`;
this.logger.error(errorMessage);
throw new Error(errorMessage);
}
// Last, try to remove the files that were *only* present previously
try {
const relativeFilesToUpload = absoluteFilesToUpload.map(
absoluteFilePath =>
getCloudPathForLocalPath(
entity,
path.relative(directory, absoluteFilePath),
useLegacyPathCasing,
),
);
const staleFiles = getStaleFiles(relativeFilesToUpload, existingFiles);
await bulkStorageOperation(
async relativeFilePath => {
return await bucket.file(relativeFilePath).delete();
},
staleFiles,
{ concurrencyLimit: 10 },
);
this.logger.info(
`Successfully deleted stale files for Entity ${entity.metadata.name}. Total number of files: ${staleFiles.length}`,
);
} catch (error) {
const errorMessage = `Unable to delete file(s) from Google Cloud Storage. ${error}`;
this.logger.error(errorMessage);
}
}
fetchTechDocsMetadata(entityName: EntityName): Promise<TechDocsMetadata> {
@@ -206,9 +228,8 @@ export class GoogleGCSPublish implements PublisherBase {
fileStreamChunks.push(chunk);
})
.on('end', () => {
const techdocsMetadataJson = Buffer.concat(fileStreamChunks).toString(
'utf-8',
);
const techdocsMetadataJson =
Buffer.concat(fileStreamChunks).toString('utf-8');
resolve(JSON5.parse(techdocsMetadataJson));
});
});
@@ -294,4 +315,29 @@ export class GoogleGCSPublish implements PublisherBase {
});
});
}
private getFilesForFolder(folder: string): Promise<string[]> {
const fileMetadataStream: Readable = this.storageClient
.bucket(this.bucketName)
.getFilesStream({ prefix: folder });
return new Promise((resolve, reject) => {
const files: string[] = [];
fileMetadataStream.on('error', error => {
// push file to file array
reject(error);
});
fileMetadataStream.on('data', (file: File) => {
// push file to file array
files.push(file.name);
});
fileMetadataStream.on('end', () => {
// resolve promise
resolve(files);
});
});
}
}
packages/techdocs-common/src/stages/publish/helpers.test.ts
+120
View File
@@ -16,9 +16,13 @@
import mockFs from 'mock-fs';
import * as os from 'os';
import * as path from 'path';
import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model';
import {
getStaleFiles,
getFileTreeRecursively,
getCloudPathForLocalPath,
getHeadersForFileExtension,
bulkStorageOperation,
lowerCaseEntityTripletInStoragePath,
} from './helpers';
@@ -99,3 +103,119 @@ describe('lowerCaseEntityTripletInStoragePath', () => {
).toThrowError(error);
});
});
describe('getStaleFiles', () => {
const defaultFiles = [
'default/Component/backstage/index.html',
'default/Component/backstage/techdocs_metadata.json',
'default/Component/backstage/assests/javascripts/bundle.7f4f3c92.min.js',
'default/Component/backstage/assets/stylesheets/main.fe0cca5b.min.css',
];
it('should return empty array if there is no stale file', () => {
const oldFiles = [...defaultFiles];
const newFiles = [...defaultFiles];
const staleFiles = getStaleFiles(newFiles, oldFiles);
expect(staleFiles).toHaveLength(0);
});
it('should return all stale files when they exists', () => {
const oldFiles = [...defaultFiles, 'stale_file.png'];
const newFiles = [...defaultFiles];
const staleFiles = getStaleFiles(newFiles, oldFiles);
expect(staleFiles).toHaveLength(1);
expect(staleFiles).toEqual(expect.arrayContaining(['stale_file.png']));
});
});
describe('getCloudPathForLocalPath', () => {
const entity: Entity = {
apiVersion: 'version',
metadata: { namespace: 'custom', name: 'backstage' },
kind: 'Component',
};
it('should compose a remote bucket path including entity information', () => {
const remoteBucket = getCloudPathForLocalPath(entity);
expect(remoteBucket).toBe('custom/component/backstage/');
});
it('should compose a remote filename including entity information', () => {
const localPath = 'index.html';
const remoteBucket = getCloudPathForLocalPath(entity, localPath);
expect(remoteBucket).toBe(`custom/component/backstage/${localPath}`);
});
it('should use the default namespace when it is undefined', () => {
const localPath = 'index.html';
const {
kind,
metadata: { name },
} = entity;
const remoteBucket = getCloudPathForLocalPath(
{ kind, metadata: { name } } as Entity,
localPath,
);
expect(remoteBucket).toBe(
`${ENTITY_DEFAULT_NAMESPACE}/component/backstage/${localPath}`,
);
});
it('should preserve case when legacy flag is passed', () => {
const remoteBucket = getCloudPathForLocalPath(entity, undefined, true);
expect(remoteBucket).toBe('custom/Component/backstage/');
});
it('should throw error when entity is invalid', () => {
expect(() => getCloudPathForLocalPath({} as Entity)).toThrow();
});
});
describe('bulkStorageOperation', () => {
const length = 26;
const args = Array.from({ length });
const createConcurrentRequestCounter = (
callback: (count: number) => void,
) => {
let count = 0;
return () =>
new Promise(resolve => {
callback(++count);
setTimeout(() => {
count--;
resolve(null);
}, 100);
});
};
it('should take care of rate limit by default', async () => {
const operation = createConcurrentRequestCounter((count: number) => {
expect(count <= 25).toBeTruthy();
});
await bulkStorageOperation(operation, args);
});
it('should accept the number of concurrency limit', async () => {
const concurrencyLimit = 10;
const operation = createConcurrentRequestCounter((count: number) => {
expect(count <= concurrencyLimit).toBeTruthy();
});
await bulkStorageOperation(operation, args, { concurrencyLimit });
});
it('should wait for all promises be resolved', async () => {
const callback = jest.fn();
const operation = createConcurrentRequestCounter(callback);
await bulkStorageOperation(operation, args);
expect(callback).toHaveBeenCalledTimes(length);
});
it('should call operation with the correct argument', async () => {
const files = ['file1.txt', 'file2.txt'];
const fn = jest.fn();
await bulkStorageOperation(fn, files);
expect(fn).toHaveBeenCalledTimes(2);
expect(fn).toHaveBeenNthCalledWith(1, files[0]);
expect(fn).toHaveBeenNthCalledWith(2, files[1]);
});
});
packages/techdocs-common/src/stages/publish/helpers.ts
+49
View File
@@ -13,7 +13,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model';
import mime from 'mime-types';
import path from 'path';
import createLimiter from 'p-limit';
import recursiveReadDir from 'recursive-readdir';
/**
@@ -126,3 +129,49 @@ export const lowerCaseEntityTripletInStoragePath = (
}
return lowerCaseEntityTriplet(originalPath);
};
// Only returns the files that existed previously and are not present anymore.
export const getStaleFiles = (
newFiles: string[],
oldFiles: string[],
): string[] => {
const staleFiles = new Set(oldFiles);
newFiles.forEach(newFile => {
staleFiles.delete(newFile);
});
return Array.from(staleFiles);
};
// Compose actual filename on remote bucket including entity information
export const getCloudPathForLocalPath = (
entity: Entity,
localPath = '',
useLegacyPathCasing = false,
): string => {
// Convert destination file path to a POSIX path for uploading.
// GCS expects / as path separator and relativeFilePath will contain \\ on Windows.
// https://cloud.google.com/storage/docs/gsutil/addlhelp/HowSubdirectoriesWork
const relativeFilePathPosix = localPath.split(path.sep).join(path.posix.sep);
// The / delimiter is intentional since it represents the cloud storage and not the local file system.
const entityRootDir = `${
entity.metadata?.namespace ?? ENTITY_DEFAULT_NAMESPACE
}/${entity.kind}/${entity.metadata.name}`;
const relativeFilePathTriplet = `${entityRootDir}/${relativeFilePathPosix}`;
const destination = useLegacyPathCasing
? relativeFilePathTriplet
: lowerCaseEntityTripletInStoragePath(relativeFilePathTriplet);
return destination; // Remote storage file relative path
};
// Perform rate limited generic operations by passing a function and a list of arguments
export const bulkStorageOperation = async <T>(
operation: (arg: T) => Promise<unknown>,
args: T[],
{ concurrencyLimit } = { concurrencyLimit: 25 },
) => {
const limiter = createLimiter(concurrencyLimit);
await Promise.all(args.map(arg => limiter(operation, arg)));
};