diff --git a/packages/techdocs-common/__mocks__/@google-cloud/storage.ts b/packages/techdocs-common/__mocks__/@google-cloud/storage.ts index 18fe62af14..ae2bce7a8f 100644 --- a/packages/techdocs-common/__mocks__/@google-cloud/storage.ts +++ b/packages/techdocs-common/__mocks__/@google-cloud/storage.ts @@ -107,6 +107,17 @@ class Bucket { file(destinationFilePath: string) { return new GCSFile(destinationFilePath); } + + getFilesStream() { + const readable = new Readable(); + readable._read = () => {}; + + process.nextTick(() => { + readable.emit('end'); + }); + + return readable; + } } export class Storage { diff --git a/packages/techdocs-common/__mocks__/aws-sdk.ts b/packages/techdocs-common/__mocks__/aws-sdk.ts index 2826a8d8bb..ec0273e277 100644 --- a/packages/techdocs-common/__mocks__/aws-sdk.ts +++ b/packages/techdocs-common/__mocks__/aws-sdk.ts @@ -104,6 +104,14 @@ export class S3 { }), }; } + + listObjectsV2() { + return { + promise: () => { + return Promise.resolve([]); + }, + }; + } } export default { diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index 349d6ead97..24f84f0136 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -15,7 +15,7 @@ */ import { Entity, EntityName } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; -import { FileExistsResponse, Storage } from '@google-cloud/storage'; +import { File, FileExistsResponse, Storage } from '@google-cloud/storage'; import express from 'express'; import JSON5 from 'json5'; import path from 'path'; @@ -119,9 +119,7 @@ export class GoogleGCSPublish implements PublisherBase { // First, retrieve a list of all individual files in currently existing const remoteFolder = getCloudPathForLocalPath(entity); - const existingFiles = ( - await bucket.getFiles({ prefix: remoteFolder }) - )[0].map(file => file.name); + const existingFiles = await this.getFilesForFolder(remoteFolder); // Then, merge new files into the same folder let absoluteFilesToUpload; @@ -276,4 +274,29 @@ export class GoogleGCSPublish implements PublisherBase { }); }); } + + private getFilesForFolder(folder: string): Promise { + const fileMetadataStream: Readable = this.storageClient + .bucket(this.bucketName) + .getFilesStream({ prefix: folder }); + + return new Promise((resolve, reject) => { + const files: string[] = []; + + fileMetadataStream.on('error', error => { + // push file to file array + reject(error); + }); + + fileMetadataStream.on('data', (file: File) => { + // push file to file array + files.push(file.name); + }); + + fileMetadataStream.on('end', () => { + // resolve promise + resolve(files); + }); + }); + } }