feat(scaffolder): Bring up to date with the latest changes in the inventory service

packages/backend/src/index.ts

@@ -33,7 +33,11 @@ import {
   createRouter as inventoryRouter,
   StaticInventory,
 } from '@backstage/plugin-inventory-backend';
-import { createScaffolder } from '@backstage/plugin-scaffolder-backend';
+import {
+  createRouter as scaffolderRouter,
+  DiskStorage,
+  CookieCutter,
+} from '@backstage/plugin-scaffolder-backend';
 import compression from 'compression';
 import cors from 'cors';
 import express from 'express';
@@ -54,6 +58,9 @@ async function main() {
     ]),
   );
 
+  const storage = new DiskStorage({ logger });
+  const templater = new CookieCutter();
+
   const app = express();
 
   app.use(helmet());
@@ -62,7 +69,7 @@ async function main() {
   app.use(express.json());
   app.use(requestLoggingHandler());
   app.use('/inventory', await inventoryRouter({ inventory, logger }));
-  app.use('/scaffolder', createScaffolder());
+  app.use('/scaffolder', await scaffolderRouter({ storage, templater, logger }));
   app.use(notFoundHandler());
   app.use(errorHandler());
 

plugins/scaffolder-backend/package.json

@@ -17,10 +17,16 @@
     "supertest": "^4.0.2"
   },
   "dependencies": {
+    "@backstage/backend-common": "0.1.1-alpha.4",
     "dockerode": "^3.2.0",
     "express": "^4.17.1",
     "fs-extra": "^9.0.0",
     "globby": "^11.0.0",
+    "compression": "^1.7.4",
+    "cors": "^2.8.5",
+    "express-promise-router": "^3.0.3",
+    "helmet": "^3.22.0",
+    "morgan": "^1.10.0",
     "winston": "^3.2.1"
   }
 }

plugins/scaffolder-backend/src/index.ts

@@ -14,31 +14,5 @@
  * limitations under the License.
  */
 
-import { TemplaterConfig, createTemplater } from './lib/templater';
-import { createStorage, StorageConfig } from './lib/storage';
-import express from 'express';
-
-export const createScaffolder = (config?: TemplaterConfig & StorageConfig) => {
-  const store = createStorage(config);
-
-  const templater = createTemplater(config);
-
-  const router = express.Router();
-
-  router.get('/v1/templates', async (_, res) => {
-    const templates = await store.list();
-    res.status(200).json(templates);
-  });
-
-  router.post('/v1/job/create', async (_, res) => {
-    // TODO(blam): Actually make this function work'
-
-    const mock = 'templateid';
-    res.status(201).json({ accepted: true });
-
-    const path = await store.prepare(mock);
-    await templater.run({ directory: path, values: { componentId: 'test' } });
-  });
-
-  return router;
-};
+export * from './scaffolder';
+export * from './service/router';

plugins/scaffolder-backend/src/lib/logger.ts

@@ -1,41 +0,0 @@
-/*
- * Copyright 2020 Spotify AB
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-import * as winston from 'winston';
-
-export const logger = winston.createLogger({
-  level: 'info',
-  format: winston.format.json(),
-  transports: [
-    //
-    // - Write all logs with level `error` and below to `error.log`
-    // - Write all logs with level `info` and below to `combined.log`
-    //
-    new winston.transports.File({ filename: 'error.log', level: 'error' }),
-    new winston.transports.File({ filename: 'combined.log' }),
-  ],
-});
-
-//
-// If we're not in production then log to the `console` with the format:
-// `${info.level}: ${info.message} JSON.stringify({ ...rest }) `
-//
-if (process.env.NODE_ENV !== 'production') {
-  logger.add(
-    new winston.transports.Console({
-      format: winston.format.simple(),
-    }),
-  );
-}

plugins/scaffolder-backend/src/run.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { getRootLogger } from '@backstage/backend-common';
+import { startStandaloneServer } from './service/standaloneServer';
+
+const port = process.env.PLUGIN_PORT ? Number(process.env.PLUGIN_PORT) : 3004;
+const enableCors = process.env.PLUGIN_CORS
+  ? Boolean(process.env.PLUGIN_CORS)
+  : false;
+const logger = getRootLogger();
+
+startStandaloneServer({ port, enableCors, logger }).catch((err) => {
+  logger.error(err);
+  process.exit(1);
+});
+
+process.on('SIGINT', () => {
+  logger.info('CTRL+C pressed; exiting.');
+  process.exit(0);
+});

plugins/scaffolder-backend/src/scaffolder/index.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export * from './storage';
+export * from './templater';
+
+export * from './storage/disk';
+export * from './templater/cookiecutter';

plugins/scaffolder-backend/src/scaffolder/storage/disk.test.ts

@@ -15,6 +15,7 @@
  */
 import { DiskStorage } from './disk';
 import * as path from 'path';
+import { Logger } from 'winston';
 describe('Disk Storage', () => {
   it('should load a simple template from a simple directory', async () => {
     const testTemplateDir = path.resolve(
@@ -23,7 +24,7 @@ describe('Disk Storage', () => {
     );
     const templateInfo = require(`${testTemplateDir}/mock-template/template-info.json`);
 
-    const repository = new DiskStorage(testTemplateDir);
+    const repository = new DiskStorage({ directory: testTemplateDir});
 
     await repository.reindex();
 
@@ -42,7 +43,8 @@ describe('Disk Storage', () => {
       '../../../test/mock-multiple-templates-dir',
     );
 
-    const repository = new DiskStorage(testTemplateDir);
+    const repository = new DiskStorage({ directory: testTemplateDir});
+
 
     await repository.reindex();
 
@@ -57,7 +59,7 @@ describe('Disk Storage', () => {
       '/some-folder-that-deffo-does-not-exist',
     );
 
-    const repository = new DiskStorage(testTemplateDir);
+    const repository = new DiskStorage({ directory: testTemplateDir});
 
     await repository.reindex();
 
@@ -72,7 +74,7 @@ describe('Disk Storage', () => {
       '../../../test/mock-failing-template-dir',
     );
 
-    const repository = new DiskStorage(testTemplateDir);
+    const repository = new DiskStorage({ directory: testTemplateDir});
 
     await repository.reindex();
 

plugins/scaffolder-backend/src/scaffolder/storage/disk.ts

@@ -16,8 +16,8 @@
 
 import globby from 'globby';
 import fs from 'fs-extra';
-import { logger } from '../logger';
 import { Template, StorageBase as Base } from '.';
+import { Logger } from 'winston';
 
 interface DiskIndexEntry {
   contents: Template;
@@ -28,8 +28,11 @@ export class DiskStorage implements Base {
   private repository: Template[] = [];
   private localIndex: DiskIndexEntry[] = [];
 
-  constructor(private repoDir = `${__dirname}/../../../sample-templates`) {
-
+  private repoDir: string;
+  private logger?: Logger;
+  constructor({ directory = `${__dirname}/../../../sample-templates`, logger }: { directory?: string, logger?: Logger }) {
+    this.repoDir = directory;
+    this.logger = logger;
   }
 
   public async list(): Promise<Template[]> {
@@ -81,7 +84,7 @@ export class DiskStorage implements Base {
             { location: currentFile.location, contents: parsed },
           ];
         } catch (ex) {
-          logger.error('Failure parsing JSON for template', {
+          this.logger?.error('Failure parsing JSON for template', {
             path: currentFile.location,
           });
         }

plugins/scaffolder-backend/src/scaffolder/storage/index.test.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 import { StorageBase, createStorage } from '.';
+import winston from 'winston';
 
 describe('Storage Interface Test', () => {
   const mockStore = new (class MockStorage implements StorageBase {
@@ -28,17 +29,19 @@ describe('Storage Interface Test', () => {
     };
   })();
 
+  const logger = winston.createLogger();
+
   afterEach(() => mockStore.reset());
 
   it('should call list of the set repo when calling list', async () => {
-    const store = createStorage({ store: mockStore });
+    const store = createStorage({ store: mockStore, logger });
     await store.list();
 
     expect(mockStore.list).toHaveBeenCalled();
   });
 
   it('should reindex on the repo when calling reindex', async () => {
-    const store = createStorage({ store: mockStore });
+    const store = createStorage({ store: mockStore, logger });
 
     await store.reindex();
 
@@ -46,7 +49,7 @@ describe('Storage Interface Test', () => {
   });
 
   it('should call prepare with the correct id when calling prepare', async () => {
-    const store = createStorage({ store: mockStore });
+    const store = createStorage({ store: mockStore, logger });
 
     await store.prepare('testid');
 

plugins/scaffolder-backend/src/scaffolder/storage/index.ts

@@ -1,4 +1,4 @@
-import { DiskStorage } from './disk';
+import { Logger } from "winston";
 
 /*
  * Copyright 2020 Spotify AB
@@ -33,6 +33,7 @@ export abstract class StorageBase {
 
 export interface StorageConfig {
   store?: StorageBase;
+  logger?: Logger;
 }
 
 class Storage implements StorageBase {
@@ -48,7 +49,7 @@ class Storage implements StorageBase {
 }
 
 export const createStorage = (
-  config: StorageConfig = { store: new DiskStorage() },
+  storageConfig: StorageConfig,
 ): StorageBase => {
-  return new Storage(config);
+  return new Storage(storageConfig);
 };

plugins/scaffolder-backend/src/scaffolder/templater/index.ts

@@ -14,11 +14,10 @@
  * limitations under the License.
  */
 
-import { CookieCutter } from './cookiecutter';
-
 export interface RequiredTemplateValues {
   componentId: string;
 }
+
 export interface TemplaterRunOptions {
   directory: string;
   values: RequiredTemplateValues & object;
@@ -45,8 +44,6 @@ class Templater implements TemplaterBase {
   }
 }
 
-export const createTemplater = (
-  config: TemplaterConfig = { templater: new CookieCutter() },
-): TemplaterBase => {
-  return new Templater(config);
+export const createTemplater = (templaterConfig: TemplaterConfig): TemplaterBase => {
+  return new Templater(templaterConfig);
 };

plugins/scaffolder-backend/src/service/router.ts

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import { Logger } from 'winston';
+import Router from 'express-promise-router';
+import express from 'express';
+import { StorageBase, TemplaterBase } from '../scaffolder';
+
+export interface RouterOptions {
+  storage: StorageBase;
+  templater: TemplaterBase,
+  logger: Logger;
+}
+
+export async function createRouter(
+  options: RouterOptions,
+): Promise<express.Router> {
+  const router = Router();
+  const {storage, templater, logger: parentLogger} = options;
+  const logger = parentLogger.child({ plugin: 'scaffolder' });
+
+  router
+    .get('/v1/templates', async (_, res) => {
+      const templates = await storage.list();
+      res.status(200).json(templates);
+    }).post('/v1/job/create', async (_, res) => {
+      // TODO(blam): Actually make this function work
+      const mock = 'templateid';
+      res.status(201).json({ accepted: true });
+
+      const path = await storage.prepare(mock);
+      await templater.run({ directory: path, values: { componentId: 'test' } });
+  });
+
+  const app = express();
+  app.set('logger', logger);
+  app.use('/', router);
+
+  return app;
+}

plugins/scaffolder-backend/src/service/standaloneApplication.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  errorHandler,
+  notFoundHandler,
+  requestLoggingHandler,
+} from '@backstage/backend-common';
+import compression from 'compression';
+import cors from 'cors';
+import express from 'express';
+import helmet from 'helmet';
+import { Logger } from 'winston';
+import { StorageBase, TemplaterBase } from '../scaffolder';
+import { createRouter } from './router';
+
+export interface ApplicationOptions {
+  enableCors: boolean;
+  storage: StorageBase;
+  templater: TemplaterBase; 
+  logger: Logger;
+}
+
+export async function createStandaloneApplication(
+  options: ApplicationOptions,
+): Promise<express.Application> {
+  const { enableCors, storage, templater, logger } = options;
+  const app = express();
+
+  app.use(helmet());
+  if (enableCors) {
+    app.use(cors());
+  }
+  app.use(compression());
+  app.use(express.json());
+  app.use(requestLoggingHandler());
+  app.use('/', await createRouter({ templater, storage, logger }));
+  app.use(notFoundHandler());
+  app.use(errorHandler());
+
+  return app;
+}

plugins/scaffolder-backend/src/service/standaloneServer.ts

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Server } from 'http';
+import { Logger } from 'winston';
+import {
+  createStorage,
+  createTemplater,
+  DiskStorage,
+  CookieCutter,
+} from '../scaffolder';
+import { createStandaloneApplication } from './standaloneApplication';
+
+export interface ServerOptions {
+  port: number;
+  enableCors: boolean;
+  logger: Logger;
+}
+
+export async function startStandaloneServer(
+  options: ServerOptions,
+): Promise<Server> {
+  const logger = options.logger.child({ service: 'scaffolder-backend' });
+  const store = new DiskStorage({ logger });
+  const templater = new CookieCutter();
+  logger.debug('Creating application...');
+  
+  const app = await createStandaloneApplication({
+    enableCors: options.enableCors,
+    storage: createStorage({ store, logger }),
+    templater: createTemplater({ templater }),
+    logger,
+  });
+
+  logger.debug('Starting application server...');
+  return await new Promise((resolve, reject) => {
+    const server = app.listen(options.port, (err?: Error) => {
+      if (err) {
+        reject(err);
+        return;
+      }
+
+      logger.info(`Listening on port ${options.port}`);
+      resolve(server);
+    });
+  });
+}