diff --git a/packages/techdocs-common/__mocks__/@google-cloud/storage.ts b/packages/techdocs-common/__mocks__/@google-cloud/storage.ts index b84018c089..503ee397b4 100644 --- a/packages/techdocs-common/__mocks__/@google-cloud/storage.ts +++ b/packages/techdocs-common/__mocks__/@google-cloud/storage.ts @@ -13,10 +13,31 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import fs from 'fs-extra'; +import path from 'path'; + type storageOptions = { keyFilename?: string; }; +/** + * @param sourceFile contains either / or \ as file separator depending upon OS. + */ +const checkFileExists = async (sourceFile: string): Promise => { + // sourceFile will always have / as file separator irrespective of OS since S3 expects /. + // Normalize sourceFile to OS specific path before checking if file exists. + const filePath = sourceFile.split(path.posix.sep).join(path.sep); + + try { + await fs.access(filePath, fs.constants.F_OK); + return true; + } catch (err) { + console.log("File doesn't exist"); + console.log(filePath); + return false; + } +}; + class Bucket { private readonly bucketName; @@ -31,8 +52,12 @@ class Bucket { } upload(source: string, { destination }) { - return new Promise(resolve => { - resolve({ source, destination }); + return new Promise(async (resolve, reject) => { + if (await checkFileExists(source)) { + resolve({ source, destination }); + } else { + reject(`Source file ${source} does not exist.`); + } }); } } diff --git a/packages/techdocs-common/__mocks__/aws-sdk.ts b/packages/techdocs-common/__mocks__/aws-sdk.ts index b546494118..6957ea7430 100644 --- a/packages/techdocs-common/__mocks__/aws-sdk.ts +++ b/packages/techdocs-common/__mocks__/aws-sdk.ts @@ -22,7 +22,7 @@ import path from 'path'; const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; /** - * @param Key Contains either / or \ as file separator depending upon OS. + * @param Key contains either / or \ as file separator depending upon OS. */ const checkFileExists = async (Key: string): Promise => { // Key will always have / as file separator irrespective of OS since S3 expects /. diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts index f8fb00647c..bf6822b82c 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts @@ -13,18 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import mockFs from 'mock-fs'; -import * as winston from 'winston'; +import { getVoidLogger } from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; +import mockFs from 'mock-fs'; +import os from 'os'; +import path from 'path'; import { GoogleGCSPublish } from './googleStorage'; import { PublisherBase } from './types'; -const createMockEntity = (annotations = {}) => { +const createMockEntity = (annotations = {}): Entity => { return { apiVersion: 'version', kind: 'TestKind', metadata: { name: 'test-component-name', + namespace: 'test-namespace', annotations: { ...annotations, }, @@ -32,12 +36,24 @@ const createMockEntity = (annotations = {}) => { }; }; -const logger = winston.createLogger(); +const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; + +const getEntityRootDir = (entity: Entity) => { + const { + kind, + metadata: { namespace, name }, + } = entity; + + return path.join(rootDir, namespace as string, kind, name); +}; + +const logger = getVoidLogger(); jest.spyOn(logger, 'info').mockReturnValue(logger); let publisher: PublisherBase; beforeEach(async () => { + mockFs.restore(); const mockConfig = new ConfigReader({ techdocs: { requestUrl: 'http://localhost:7000', @@ -55,9 +71,16 @@ beforeEach(async () => { }); describe('GoogleGCSPublish', () => { + afterEach(() => { + mockFs.restore(); + }); + it('should publish a directory', async () => { + const entity = createMockEntity(); + const entityRootDir = getEntityRootDir(entity); + mockFs({ - '/path/to/generatedDirectory': { + [entityRootDir]: { 'index.html': '', '404.html': '', assets: { @@ -66,11 +89,10 @@ describe('GoogleGCSPublish', () => { }, }); - const entity = createMockEntity(); expect( await publisher.publish({ entity, - directory: '/path/to/generatedDirectory', + directory: entityRootDir, }), ).toBeUndefined(); mockFs.restore(); diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index 8876007504..1041794161 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -13,20 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import path from 'path'; -import express from 'express'; -import { - Storage, - UploadResponse, - FileExistsResponse, -} from '@google-cloud/storage'; -import { Logger } from 'winston'; import { Entity, EntityName } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; -import { getHeadersForFileExtension, getFileTreeRecursively } from './helpers'; -import { PublisherBase, PublishRequest, TechDocsMetadata } from './types'; +import { + FileExistsResponse, + Storage, + UploadResponse, +} from '@google-cloud/storage'; +import express from 'express'; import JSON5 from 'json5'; import createLimiter from 'p-limit'; +import path from 'path'; +import { Logger } from 'winston'; +import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers'; +import { PublisherBase, PublishRequest, TechDocsMetadata } from './types'; export class GoogleGCSPublish implements PublisherBase { static async fromConfig( @@ -105,19 +105,27 @@ export class GoogleGCSPublish implements PublisherBase { const limiter = createLimiter(10); const uploadPromises: Array> = []; - allFilesToUpload.forEach(filePath => { + allFilesToUpload.forEach(sourceFilePath => { // Remove the absolute path prefix of the source directory // Path of all files to upload, relative to the root of the source directory // e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png'] - const relativeFilePath = filePath.replace(`${directory}/`, ''); + const relativeFilePath = path.relative(directory, sourceFilePath); + + // Convert destination file path to a POSIX path for uploading. + // GCS expects / as path separator and relativeFilePath will contain \\ on Windows. + // https://cloud.google.com/storage/docs/gsutil/addlhelp/HowSubdirectoriesWork + const relativeFilePathPosix = relativeFilePath + .split(path.sep) + .join(path.posix.sep); + const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; - const destination = `${entityRootDir}/${relativeFilePath}`; // GCS Bucket file relative path + const destination = `${entityRootDir}/${relativeFilePathPosix}`; // GCS Bucket file relative path // Rate limit the concurrent execution of file uploads to batches of 10 (per publish) const uploadFile = limiter(() => - this.storageClient - .bucket(this.bucketName) - .upload(filePath, { destination }), + this.storageClient.bucket(this.bucketName).upload(sourceFilePath, { + destination, + }), ); uploadPromises.push(uploadFile); }); @@ -130,7 +138,7 @@ export class GoogleGCSPublish implements PublisherBase { resolve(undefined); }) .catch((err: Error) => { - const errorMessage = `Unable to upload file(s) to Google Cloud Storage. Error ${err.message}`; + const errorMessage = `Unable to upload file(s) to Google Cloud Storage. Error ${err}`; this.logger.error(errorMessage); reject(errorMessage); });