OpenStackSwift tests added to project

Signed-off-by: Mert Can Bilgiç <mert.bilgic@trendyol.com>

packages/techdocs-common/__mocks__/pkgcloud.ts

@@ -13,21 +13,20 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { OpenstackProviderOptions } from 'pkgcloud';
 import fs from 'fs-extra';
 import os from 'os';
 import path from 'path';
+import { ObjectWritableMock, BufferReadableMock } from 'stream-mock';
 
 const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir';
 
 const checkFileExists = async (Key: string): Promise<boolean> => {
   // Key will always have / as file separator irrespective of OS since S3 expects /.
   // Normalize Key to OS specific path before checking if file exists.
-  const relativeFilePath = Key.split(path.posix.sep).join(path.sep);
   const filePath = path.join(rootDir, Key);
 
   try {
-    await fs.access(filePath, fs.constants.F_OK);
+    fs.accessSync(filePath, fs.constants.F_OK);
     return true;
   } catch (err) {
     return false;
@@ -38,11 +37,11 @@ class PkgCloudStorageClient {
   getFile(
     containerName: string,
     file: string,
-    callback: (err: string, file: string) => any,
+    callback: (err: any, file: string) => any,
   ) {
     checkFileExists(file).then(res => {
       if (!res) {
-        callback('File does not exist', undefined);
+        callback('File does not exist', file);
         throw new Error('File does not exist');
       } else {
         callback(undefined, 'success');
@@ -55,40 +54,28 @@ class PkgCloudStorageClient {
     callback: (err: string, container: string) => any,
   ) {
     if (containerName !== 'mock') {
-      callback("Container doesn't exist", undefined);
+      callback("Container doesn't exist", containerName);
       throw new Error('Container does not exist');
     } else {
-      callback(undefined, 'success');
+      callback('Container does not exist', 'success');
     }
   }
 
-  upload({ containerName, remote }: { containerName: string; remote: string }) {
-    checkFileExists(remote).then(res => {
-      if (!res) {
-        return new Error("File doesn't exists");
-      }
-      return fs.createWriteStream(`${containerName}/${remote}`);
-    });
+  upload() {
+     return new ObjectWritableMock();
   }
 
-  download({
-    containerName,
-    remote,
-  }: {
-    containerName: string;
-    remote: string;
-  }) {
-    checkFileExists(remote).then(res => {
-      if (!res) {
-        return new Error("File doesn't exists");
-      }
-      return fs.createReadStream(remote);
+  download() {
+    const stringify = JSON.stringify({
+      "site_description": 'site_content',
+      "site_name": "backstage"
     });
+    return new BufferReadableMock([stringify]);
   }
 }
 
 export class storage {
-  static createClient(params: OpenstackProviderOptions) {
+  static createClient() {
     return new PkgCloudStorageClient();
   }
 }

packages/techdocs-common/package.json

@@ -58,6 +58,7 @@
     "p-limit": "^3.1.0",
     "pkgcloud": "^2.2.0",
     "recursive-readdir": "^2.2.2",
+    "stream-mock": "^2.0.5",
     "winston": "^3.2.1"
   },
   "devDependencies": {

packages/techdocs-common/src/stages/publish/openStackSwift.test.ts

@@ -0,0 +1,248 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  Entity,
+  EntityName,
+  ENTITY_DEFAULT_NAMESPACE,
+} from '@backstage/catalog-model';
+import { ConfigReader } from '@backstage/config';
+import mockFs from 'mock-fs';
+import os from 'os';
+import path from 'path';
+import * as winston from 'winston';
+import { OpenStackSwiftPublish } from './openStackSwift';
+import { PublisherBase, TechDocsMetadata } from './types';
+
+// NOTE: /packages/techdocs-common/__mocks__ is being used to mock pkgcloud client library
+
+const createMockEntity = (annotations = {}): Entity => {
+  return {
+    apiVersion: 'version',
+    kind: 'TestKind',
+    metadata: {
+      name: 'test-component-name',
+      namespace: 'test-namespace',
+      annotations: {
+        ...annotations,
+      },
+    },
+  };
+};
+
+const createMockEntityName = (): EntityName => ({
+  kind: 'TestKind',
+  name: 'test-component-name',
+  namespace: 'test-namespace',
+});
+
+const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir';
+
+const getEntityRootDir = (entity: Entity) => {
+  const {
+    kind,
+    metadata: { namespace, name },
+  } = entity;
+
+  return path.join(rootDir, namespace || ENTITY_DEFAULT_NAMESPACE, kind, name);
+};
+
+const logger = winston.createLogger();
+jest.spyOn(logger, 'info').mockReturnValue(logger);
+jest.spyOn(logger, 'error').mockReturnValue(logger);
+
+let publisher: PublisherBase;
+
+beforeEach(() => {
+  mockFs.restore();
+  const mockConfig = new ConfigReader({
+    techdocs: {
+      requestUrl: 'http://localhost:7000',
+      publisher: {
+        type: 'openStackSwift',
+        openStackSwift: {
+          username: 'mockuser',
+          password: 'verystrongpass',
+          authUrl: 'mockauthurl',
+          region: 'mockregion',
+          containerName: 'mock',
+        },
+      },
+    },
+  });
+
+  publisher = OpenStackSwiftPublish.fromConfig(mockConfig, logger);
+});
+
+describe('OpenStackSwiftPublish', () => {
+  describe('publish', () => {
+    beforeEach(() => {
+      const entity = createMockEntity();
+      const entityRootDir = getEntityRootDir(entity);
+
+      mockFs({
+        [entityRootDir]: {
+          'index.html': '',
+          '404.html': '',
+          assets: {
+            'main.css': '',
+          },
+        },
+      });
+    });
+
+    afterEach(() => {
+      mockFs.restore();
+    });
+
+    it('should publish a directory', async () => {
+      const entity = createMockEntity();
+      const entityRootDir = getEntityRootDir(entity);
+
+      setTimeout(async () => {
+        expect(
+          await publisher.publish({
+            entity,
+            directory: entityRootDir,
+          }),
+        ).toBeUndefined()
+      }, 5000);
+    });
+
+    it('should fail to publish a directory', async () => {
+      expect.assertions(3);
+      const wrongPathToGeneratedDirectory = path.join(
+        rootDir,
+        'wrong',
+        'path',
+        'to',
+        'generatedDirectory',
+      );
+
+      const entity = createMockEntity();
+      await expect(
+        publisher.publish({
+          entity,
+          directory: wrongPathToGeneratedDirectory,
+        }),
+      ).rejects.toThrowError();
+
+      await publisher
+        .publish({
+          entity,
+          directory: wrongPathToGeneratedDirectory,
+        })
+        .catch(error => {
+          expect(error.message).toEqual(
+            // Can not do exact error message match due to mockFs adding unexpected characters in the path when throwing the error
+            // Issue reported https://github.com/tschaub/mock-fs/issues/118
+            expect.stringContaining(
+              `Unable to upload file(s) to OpenStack Swift. Error: Failed to read template directory: ENOENT, no such file or directory`,
+            ),
+          );
+          expect(error.message).toEqual(
+            expect.stringContaining(wrongPathToGeneratedDirectory),
+          );
+        });
+      mockFs.restore();
+    });
+  });
+
+  describe('hasDocsBeenGenerated', () => {
+    it('should return true if docs has been generated', async () => {
+      const entity = createMockEntity();
+      const entityRootDir = getEntityRootDir(entity);
+
+      mockFs({
+        [entityRootDir]: {
+          'index.html': 'file-content',
+        },
+      });
+
+      expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true);
+      mockFs.restore();
+    });
+
+    it('should return false if docs has not been generated', async () => {
+      const entity = createMockEntity();
+
+      expect(await publisher.hasDocsBeenGenerated(entity)).toBe(false);
+    });
+  });
+
+  describe('fetchTechDocsMetadata', () => {
+    it('should return tech docs metadata', async () => {
+      const entityNameMock = createMockEntityName();
+      const entity = createMockEntity();
+      const entityRootDir = getEntityRootDir(entity);
+
+      mockFs({
+        [entityRootDir]: {
+          'techdocs_metadata.json':
+            '{"site_name": "backstage", "site_description": "site_content"}',
+        },
+      });
+
+      const expectedMetadata: TechDocsMetadata = {
+        site_name: 'backstage',
+        site_description: 'site_content',
+      };
+      expect(
+        await publisher.fetchTechDocsMetadata(entityNameMock),
+      ).toStrictEqual(expectedMetadata);
+      mockFs.restore();
+    });
+
+    it('should return tech docs metadata when json encoded with single quotes', async () => {
+      const entityNameMock = createMockEntityName();
+      const entity = createMockEntity();
+      const entityRootDir = getEntityRootDir(entity);
+
+      mockFs({
+        [entityRootDir]: {
+          'techdocs_metadata.json': `{'site_name': 'backstage', 'site_description': 'site_content'}`,
+        },
+      });
+
+      const expectedMetadata: TechDocsMetadata = {
+        site_name: 'backstage',
+        site_description: 'site_content',
+      };
+      expect(
+        await publisher.fetchTechDocsMetadata(entityNameMock),
+      ).toStrictEqual(expectedMetadata);
+      mockFs.restore();
+    });
+
+    it('should return an error if the techdocs_metadata.json file is not present', async () => {
+      const entityNameMock = createMockEntityName();
+      const entity = createMockEntity();
+      const entityRootDir = getEntityRootDir(entity);
+
+      await publisher
+        .fetchTechDocsMetadata(entityNameMock)
+        .catch(error =>
+          expect(error).toEqual(
+            new Error(
+              `TechDocs metadata fetch failed, The file ${path.join(
+                entityRootDir,
+                'techdocs_metadata.json',
+              )} does not exist !`,
+            ),
+          ),
+        );
+    });
+  });
+});

packages/techdocs-common/src/stages/publish/openStackSwift.ts

@@ -107,7 +107,6 @@ export class OpenStackSwiftPublish implements PublisherBase {
    * Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html
    */
   async publish({ entity, directory }: PublishRequest): Promise<void> {
-    this.logger.info(`Publish Called hey`);
     try {
       // Note: OpenStack Swift manages creation of parent directories if they do not exist.
       // So collecting path of only the files is good enough.
@@ -139,8 +138,7 @@ export class OpenStackSwiftPublish implements PublisherBase {
         };
 
         // Rate limit the concurrent execution of file uploads to batches of 10 (per publish)
-        const uploadFile = limiter(
-          () =>
+        const uploadFile = limiter(() =>
             new Promise((res, rej) => {
               const writeStream = this.storageClient.upload(params);
 
@@ -168,7 +166,6 @@ export class OpenStackSwiftPublish implements PublisherBase {
   async fetchTechDocsMetadata(
     entityName: EntityName,
   ): Promise<TechDocsMetadata> {
-    this.logger.info(`fetchTechDocsMetadata Called hey`);
     try {
       return await new Promise<TechDocsMetadata>(async (resolve, reject) => {
         const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`;
@@ -206,7 +203,6 @@ export class OpenStackSwiftPublish implements PublisherBase {
    */
   docsRouter(): express.Handler {
     return async (req, res) => {
-      this.logger.info(`docsRouter Called hey`);
       // Trim the leading forward slash
       // filePath example - /default/Component/documented-component/index.html
 
@@ -243,7 +239,6 @@ export class OpenStackSwiftPublish implements PublisherBase {
    */
   async hasDocsBeenGenerated(entity: Entity): Promise<boolean> {
     try {
-      this.logger.info(`hasDocsBeenGenerated Called hey`);
       const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`;
 
       return new Promise(res => {
@@ -251,7 +246,6 @@ export class OpenStackSwiftPublish implements PublisherBase {
           this.containerName,
           `${entityRootDir}/index.html`,
           (err: any, file: any) => {
-            console.log(file);
             if (!err && file) {
               res(true);
             } else res(false);