Merge branch 'mob/create-vcs-step' of github.com:spotify/backstage into shmidt-i/scaffolder-flow-frontend

plugins/scaffolder-backend/package.json

@@ -43,6 +43,7 @@
   },
   "devDependencies": {
     "@backstage/cli": "^0.1.1-alpha.12",
+    "@octokit/types": "^5.0.1",
     "@types/fs-extra": "^9.0.1",
     "@types/git-url-parse": "^9.0.0",
     "@types/nodegit": "0.26.5",

plugins/scaffolder-backend/src/scaffolder/index.ts

@@ -13,6 +13,5 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-export * from './stages/templater';
-export * from './stages/prepare';
+export * from './stages';
 export * from './jobs';

plugins/scaffolder-backend/src/scaffolder/jobs/logger.test.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { makeLogStream } from './logger';
+describe('Logger', () => {
+  const mockMeta = { test: 'blob' };
+
+  it('should return empty log lines by default', async () => {
+    const { log } = makeLogStream(mockMeta);
+
+    expect(log).toEqual([]);
+  });
+  it('should add lines to the log when using the logger that is returned', async () => {
+    const { logger, log } = makeLogStream(mockMeta);
+
+    logger.info('TEST LINE');
+    logger.warn('WARN LINE');
+
+    const [first, second] = log;
+    expect(log.length).toBe(2);
+
+    expect(first).toContain('info');
+    expect(first).toContain('TEST LINE');
+
+    expect(second).toContain('warn');
+    expect(second).toContain('WARN LINE');
+  });
+
+  it('should add lines from writing to the stream that is returned', async () => {
+    const { stream, log } = makeLogStream(mockMeta);
+    const textLine = 'SOMETHING';
+    stream.write(textLine);
+
+    expect(log).toContain(textLine);
+  });
+});

plugins/scaffolder-backend/src/scaffolder/stages/index.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export * from './prepare';
+export * from './publish';
+export * from './templater';

plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts

@@ -19,7 +19,6 @@ const mocks = {
   CheckoutOptions: jest.fn(() => {}),
 };
 jest.doMock('nodegit', () => mocks);
-// require('nodegit');
 
 import { GithubPreparer } from './github';
 import {

plugins/scaffolder-backend/src/scaffolder/stages/publish/__mocks__/@octokit/rest/index.ts

@@ -13,11 +13,16 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { TemplateEntityV1alpha1 } from "@backstage/catalog-model";
-import { RequiredTemplateValues } from "../templater";
-import { JsonValue } from "@backstage/config";
 
-export type Storer = {
-  createRemote(opts: { entity: TemplateEntityV1alpha1, values: RequiredTemplateValues & Record<string, JsonValue>}): Promise<string>;
-  pushToRemote(directory: string, remote: string): Promise<void>;
+export const mockGithubClient = {
+  repos: {
+    createInOrg: jest.fn(),
+    createForAuthenticatedUser: jest.fn(),
+  },
+};
+
+export class Octokit {
+  constructor() {
+    return mockGithubClient;
+  }
 }

plugins/scaffolder-backend/src/scaffolder/stages/publish/__mocks__/nodegit/index.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export const mockIndex = {
+  addAll: jest.fn(),
+  write: jest.fn(),
+  writeTree: jest.fn().mockResolvedValue('mockoid'),
+};
+
+export const mockRepo = {
+  refreshIndex: jest.fn().mockResolvedValue(mockIndex),
+  createCommit: jest.fn(),
+};
+
+export const mockRemote = {
+  push: jest.fn(),
+};
+
+const Repository = { init: jest.fn().mockResolvedValue(mockRepo) };
+const Remote = { create: jest.fn().mockResolvedValue(mockRemote) };
+const Signature = { now: jest.fn() };
+const Cred = {
+  userpassPlaintextNew: jest.fn(),
+};
+
+export { Repository, Remote, Signature, Cred };

plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts

@@ -0,0 +1,203 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+jest.mock('@octokit/rest');
+jest.mock('nodegit');
+
+import { Octokit } from '@octokit/rest';
+import * as NodeGit from 'nodegit';
+import { OctokitResponse, ReposCreateInOrgResponseData } from '@octokit/types';
+import { GithubPublisher } from './github';
+
+const { mockGithubClient } = require('@octokit/rest') as {
+  mockGithubClient: { repos: jest.Mocked<Octokit['repos']> };
+};
+
+const {
+  Repository,
+  mockRepo,
+  mockIndex,
+  Signature,
+  Remote,
+  mockRemote,
+  Cred,
+} = require('nodegit') as {
+  Repository: jest.Mocked<{ init: any }>;
+  Signature: jest.Mocked<{ now: any }>;
+  Cred: jest.Mocked<{ userpassPlaintextNew: any }>;
+  Remote: jest.Mocked<{ create: any }>;
+
+  mockIndex: jest.Mocked<NodeGit.Index>;
+  mockRepo: jest.Mocked<NodeGit.Repository>;
+  mockRemote: jest.Mocked<NodeGit.Remote>;
+};
+
+describe('Github Publisher', () => {
+  const publisher = new GithubPublisher({ client: new Octokit() });
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('publish: createRemoteInGithub', () => {
+    it('should use octokit to create a repo in an organisation if the organisation property is set', async () => {
+      mockGithubClient.repos.createInOrg.mockResolvedValue({
+        data: {
+          clone_url: 'mockclone',
+        },
+      } as OctokitResponse<ReposCreateInOrgResponseData>);
+
+      await publisher.publish({
+        values: {
+          isOrg: true,
+          storePath: 'blam/test',
+          owner: 'bob',
+        },
+        directory: '/tmp/test',
+      });
+
+      expect(mockGithubClient.repos.createInOrg).toHaveBeenCalledWith({
+        org: 'blam',
+        name: 'test',
+      });
+    });
+
+    it('should use octokit to create a repo in the authed user if the organisation property is not set', async () => {
+      mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({
+        data: {
+          clone_url: 'mockclone',
+        },
+      } as OctokitResponse<ReposCreateInOrgResponseData>);
+
+      await publisher.publish({
+        values: {
+          storePath: 'blam/test',
+          owner: 'bob',
+        },
+        directory: '/tmp/test',
+      });
+
+      expect(
+        mockGithubClient.repos.createForAuthenticatedUser,
+      ).toHaveBeenCalledWith({
+        name: 'test',
+      });
+    });
+  });
+
+  describe('publish: createGitDirectory', () => {
+    const values = {
+      isOrg: true,
+      storePath: 'blam/test',
+      owner: 'lols',
+    };
+
+    const mockDir = '/tmp/test/dir';
+
+    mockGithubClient.repos.createInOrg.mockResolvedValue({
+      data: {
+        clone_url: 'mockclone',
+      },
+    } as OctokitResponse<ReposCreateInOrgResponseData>);
+    it('should call init on the repo with the directory', async () => {
+      await publisher.publish({
+        values,
+        directory: mockDir,
+      });
+
+      expect(Repository.init).toHaveBeenCalledWith(mockDir, 0);
+    });
+
+    it('should call refresh index on the index and write the new files', async () => {
+      await publisher.publish({
+        values,
+        directory: mockDir,
+      });
+
+      expect(mockRepo.refreshIndex).toHaveBeenCalled();
+    });
+
+    it('should call add all files and write', async () => {
+      await publisher.publish({
+        values,
+        directory: mockDir,
+      });
+
+      expect(mockIndex.addAll).toHaveBeenCalled();
+      expect(mockIndex.write).toHaveBeenCalled();
+      expect(mockIndex.writeTree).toHaveBeenCalled();
+    });
+
+    it('should create a commit with on head with the right name and commiter', async () => {
+      const mockSignature = { mockSignature: 'bloblly' };
+      Signature.now.mockReturnValue(mockSignature);
+
+      await publisher.publish({
+        values,
+        directory: mockDir,
+      });
+
+      expect(Signature.now).toHaveBeenCalledTimes(2);
+      expect(Signature.now).toHaveBeenCalledWith(
+        'Scaffolder',
+        'scaffolder@backstage.io',
+      );
+
+      expect(mockRepo.createCommit).toHaveBeenCalledWith(
+        'HEAD',
+        mockSignature,
+        mockSignature,
+        'initial commit',
+        'mockoid',
+        [],
+      );
+    });
+
+    it('creates a remote with the repo and remote', async () => {
+      await publisher.publish({
+        values,
+        directory: mockDir,
+      });
+
+      expect(Remote.create).toHaveBeenCalledWith(
+        mockRepo,
+        'origin',
+        'mockclone',
+      );
+    });
+
+    it('shoud push to the remote repo', async () => {
+      await publisher.publish({
+        values,
+        directory: mockDir,
+      });
+
+      const [remotes, { callbacks }] = mockRemote.push.mock
+        .calls[0] as NodeGit.PushOptions[];
+
+      expect(remotes).toEqual(['refs/heads/master:refs/heads/master']);
+
+      process.env.GITHUb_ACCESS_TOKEN = 'blob';
+
+      callbacks?.credentials?.();
+
+      expect(Cred.userpassPlaintextNew).toHaveBeenCalledWith(
+        process.env.GITHUB_ACCESS_TOKEN,
+        'x-oauth-basic',
+      );
+    });
+  });
+});

plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts

@@ -14,38 +14,47 @@
  * limitations under the License.
  */
 
-import { Storer } from './types';
+import { Publisher } from './types';
 import { Octokit } from '@octokit/rest';
-import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
+
 import { JsonValue } from '@backstage/config';
 import { RequiredTemplateValues } from '../templater';
 import { Repository, Remote, Signature, Cred } from 'nodegit';
 
-export class GithubStorer implements Storer {
+export class GithubPublisher implements Publisher {
   private client: Octokit;
   constructor({ client }: { client: Octokit }) {
     this.client = client;
   }
 
-  async createRemote({
+  async publish({
     values,
+    directory,
   }: {
-    entity: TemplateEntityV1alpha1;
     values: RequiredTemplateValues & Record<string, JsonValue>;
-  }) {
-    const [owner, name] = values.storePath.split('/');
+    directory: string;
+  }): Promise<{ remoteUrl: string }> {
+    const remoteUrl = await this.createRemote(values);
+    await this.pushToRemote(directory, remoteUrl);
 
-    const {
-      data: { clone_url: cloneUrl },
-    } = await this.client.repos.createInOrg({
-      name,
-      org: owner,
-    });
-
-    return cloneUrl;
+    return { remoteUrl };
   }
 
-  async pushToRemote(directory: string, remote: string): Promise<void> {
+  private async createRemote(
+    values: RequiredTemplateValues & Record<string, JsonValue>,
+  ) {
+    const [owner, name] = values.storePath.split('/');
+
+    const repoCreationPromise = values.isOrg
+      ? this.client.repos.createInOrg({ name, org: owner })
+      : this.client.repos.createForAuthenticatedUser({ name });
+
+    const { data } = await repoCreationPromise;
+
+    return data?.clone_url;
+  }
+
+  private async pushToRemote(directory: string, remote: string): Promise<void> {
     const repo = await Repository.init(directory, 0);
     const index = await repo.refreshIndex();
     await index.addAll();

plugins/scaffolder-backend/src/scaffolder/stages/publish/index.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export * from './github';

plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
+import { RequiredTemplateValues } from '../templater';
+import { JsonValue } from '@backstage/config';
+
+export type Publisher = {
+  publish(opts: {
+    entity: TemplateEntityV1alpha1;
+    values: RequiredTemplateValues & Record<string, JsonValue>;
+    directory: string;
+  }): Promise<{ remoteUrl: string }>;
+};

plugins/scaffolder-backend/src/service/router.ts

@@ -22,13 +22,13 @@ import express from 'express';
 import Router from 'express-promise-router';
 import { Logger } from 'winston';
 import {
+  GithubPublisher,
   JobProcessor,
   PreparerBuilder,
   RequiredTemplateValues,
+  StageContext,
   TemplaterBase,
 } from '../scaffolder';
-import { StageContext } from '../scaffolder/jobs/types';
-import { GithubStorer } from '../scaffolder/stages/store/github';
 
 export interface RouterOptions {
   preparers: PreparerBuilder;
@@ -44,7 +44,7 @@ export async function createRouter(
 
   const githubClient = new Octokit({ auth: process.env.GITHUB_ACCESS_TOKEN });
   const { preparers, templater, logger: parentLogger, dockerClient } = options;
-  const githubStorer = new GithubStorer({ client: githubClient });
+  const githubPulisher = new GithubPublisher({ client: githubClient });
   const logger = parentLogger.child({ plugin: 'scaffolder' });
   const jobProcessor = new JobProcessor();
 
@@ -117,26 +117,16 @@ export async function createRouter(
             },
           },
           {
-            name: 'Create VCS Repo',
+            name: 'Publish template',
             handler: async (ctx: StageContext<{ resultDir: string }>) => {
-              ctx.logger.info('Should now create the VCS repo');
-              const remoteUrl = await githubStorer.createRemote({
+              ctx.logger.info('Should not store the template');
+              const { remoteUrl } = await githubPulisher.publish({
                 values: ctx.values,
-                entity: ctx.entity,
+                directory: ctx.resultDir,
               });
-
               return { remoteUrl };
             },
           },
-          {
-            name: 'Push to remote',
-            handler: async (
-              ctx: StageContext<{ resultDir: string; remoteUrl: string }>,
-            ) => {
-              ctx.logger.info('Should now push to the remote');
-              await githubStorer.pushToRemote(ctx.resultDir, ctx.remoteUrl);
-            },
-          },
         ],
       });