diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index 47f5a31158..30de3a8b6f 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -24,8 +24,9 @@ "@backstage/backend-common": "^0.1.1-alpha.12", "@backstage/catalog-model": "^0.1.1-alpha.12", "@backstage/config": "^0.1.1-alpha.12", - "@types/express": "^4.17.6", + "@octokit/rest": "^18.0.0", "@types/dockerode": "^2.5.32", + "@types/express": "^4.17.6", "compression": "^1.7.4", "cors": "^2.8.5", "dockerode": "^3.2.0", @@ -42,6 +43,7 @@ }, "devDependencies": { "@backstage/cli": "^0.1.1-alpha.12", + "@octokit/types": "^5.0.1", "@types/fs-extra": "^9.0.1", "@types/git-url-parse": "^9.0.0", "@types/nodegit": "0.26.5", diff --git a/plugins/scaffolder-backend/sample-templates/react-ssr-template/hooks/post_gen_project.sh b/plugins/scaffolder-backend/sample-templates/react-ssr-template/hooks/post_gen_project.sh index 75c35f2034..1e302afa5e 100644 --- a/plugins/scaffolder-backend/sample-templates/react-ssr-template/hooks/post_gen_project.sh +++ b/plugins/scaffolder-backend/sample-templates/react-ssr-template/hooks/post_gen_project.sh @@ -1,7 +1,3 @@ #!/bin/sh -# Move all template files to the root folder -mv ./* ../ -cd .. -rm -rf {{cookiecutter.component_id}} -# # # # # # # # +echo "Could do something here?" diff --git a/plugins/scaffolder-backend/sample-templates/react-ssr-template/{{cookiecutter.component_id}}/.github/workflows/build.yml b/plugins/scaffolder-backend/sample-templates/react-ssr-template/{{cookiecutter.component_id}}/.github/workflows/build.yml index d1563ba3e6..e3ca9f44e2 100644 --- a/plugins/scaffolder-backend/sample-templates/react-ssr-template/{{cookiecutter.component_id}}/.github/workflows/build.yml +++ b/plugins/scaffolder-backend/sample-templates/react-ssr-template/{{cookiecutter.component_id}}/.github/workflows/build.yml @@ -1,9 +1,6 @@ name: Frontend CI -on: - push: - paths: - - '.' +on: [push, pull_request] jobs: build: diff --git a/plugins/scaffolder-backend/src/scaffolder/__mocks__/@octokit/rest/index.ts b/plugins/scaffolder-backend/src/scaffolder/__mocks__/@octokit/rest/index.ts new file mode 100644 index 0000000000..85bed3a223 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/__mocks__/@octokit/rest/index.ts @@ -0,0 +1,28 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export const mockGithubClient = { + repos: { + createInOrg: jest.fn(), + createForAuthenticatedUser: jest.fn(), + }, +}; + +export class Octokit { + constructor() { + return mockGithubClient; + } +} diff --git a/plugins/scaffolder-backend/src/scaffolder/__mocks__/nodegit/index.ts b/plugins/scaffolder-backend/src/scaffolder/__mocks__/nodegit/index.ts new file mode 100644 index 0000000000..e3fb5000d0 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/__mocks__/nodegit/index.ts @@ -0,0 +1,39 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export const mockIndex = { + addAll: jest.fn(), + write: jest.fn(), + writeTree: jest.fn().mockResolvedValue('mockoid'), +}; + +export const mockRepo = { + refreshIndex: jest.fn().mockResolvedValue(mockIndex), + createCommit: jest.fn(), +}; + +export const mockRemote = { + push: jest.fn(), +}; + +const Repository = { init: jest.fn().mockResolvedValue(mockRepo) }; +const Remote = { create: jest.fn().mockResolvedValue(mockRemote) }; +const Signature = { now: jest.fn() }; +const Cred = { + userpassPlaintextNew: jest.fn(), +}; + +export { Repository, Remote, Signature, Cred }; diff --git a/plugins/scaffolder-backend/src/scaffolder/index.ts b/plugins/scaffolder-backend/src/scaffolder/index.ts index a1ee172184..470bc230d6 100644 --- a/plugins/scaffolder-backend/src/scaffolder/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/index.ts @@ -13,7 +13,5 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export * from './stages/templater'; -export * from './stages/templater/cookiecutter'; -export * from './stages/prepare'; +export * from './stages'; export * from './jobs'; diff --git a/plugins/scaffolder-backend/src/scaffolder/jobs/logger.test.ts b/plugins/scaffolder-backend/src/scaffolder/jobs/logger.test.ts new file mode 100644 index 0000000000..12f893c146 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/jobs/logger.test.ts @@ -0,0 +1,49 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { makeLogStream } from './logger'; + +describe('Logger', () => { + const mockMeta = { test: 'blob' }; + + it('should return empty log lines by default', async () => { + const { log } = makeLogStream(mockMeta); + + expect(log).toEqual([]); + }); + it('should add lines to the log when using the logger that is returned', async () => { + const { logger, log } = makeLogStream(mockMeta); + + logger.info('TEST LINE'); + logger.warn('WARN LINE'); + + const [first, second] = log; + expect(log.length).toBe(2); + + expect(first).toContain('info'); + expect(first).toContain('TEST LINE'); + + expect(second).toContain('warn'); + expect(second).toContain('WARN LINE'); + }); + + it('should add lines from writing to the stream that is returned', async () => { + const { stream, log } = makeLogStream(mockMeta); + const textLine = 'SOMETHING'; + stream.write(textLine); + + expect(log).toContain(textLine); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/jobs/logger.ts b/plugins/scaffolder-backend/src/scaffolder/jobs/logger.ts index d2eaa13c3b..2e88dd34fe 100644 --- a/plugins/scaffolder-backend/src/scaffolder/jobs/logger.ts +++ b/plugins/scaffolder-backend/src/scaffolder/jobs/logger.ts @@ -17,7 +17,7 @@ import { PassThrough } from 'stream'; import winston from 'winston'; import { JsonValue } from '@backstage/config'; -export const useLogStream = (meta: Record) => { +export const makeLogStream = (meta: Record) => { const log: string[] = []; // Create an empty stream to collect all the log lines into diff --git a/plugins/scaffolder-backend/src/scaffolder/jobs/processor.test.ts b/plugins/scaffolder-backend/src/scaffolder/jobs/processor.test.ts index 501e72c078..a8cfe9f3ea 100644 --- a/plugins/scaffolder-backend/src/scaffolder/jobs/processor.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/jobs/processor.test.ts @@ -16,6 +16,7 @@ import { JobProcessor } from './processor'; import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import { StageInput } from './types'; +import { RequiredTemplateValues } from '../stages/templater'; describe('JobProcessor', () => { const mockEntity: TemplateEntityV1alpha1 = { @@ -41,7 +42,10 @@ describe('JobProcessor', () => { }, }; - const mockValues = { component_id: 'bob' }; + const mockValues: RequiredTemplateValues = { + owner: 'blobby', + storePath: 'spotify/mock-repo', + }; describe('create', () => { it('creates should create a new job with a unique id', async () => { diff --git a/plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts b/plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts index 1c76ae252d..0a3b556941 100644 --- a/plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts +++ b/plugins/scaffolder-backend/src/scaffolder/jobs/processor.ts @@ -20,7 +20,7 @@ import * as uuid from 'uuid'; import Docker from 'dockerode'; import { RequiredTemplateValues, TemplaterBase } from '../stages/templater'; import { PreparerBuilder } from '../stages/prepare'; -import { useLogStream } from './logger'; +import { makeLogStream } from './logger'; export type JobProcessorArguments = { preparers: PreparerBuilder; @@ -46,7 +46,7 @@ export class JobProcessor implements Processor { stages: StageInput[]; }): Job { const id = uuid.v4(); - const { logger, stream } = useLogStream({ id }); + const { logger, stream } = makeLogStream({ id }); const context: StageContext = { entity, @@ -87,7 +87,7 @@ export class JobProcessor implements Processor { for (const stage of job.stages) { // Create a logger for each stage so we can create seperate // Streams for each step. - const { logger, log, stream } = useLogStream({ + const { logger, log, stream } = makeLogStream({ id: job.id, stage: stage.name, }); diff --git a/plugins/scaffolder-backend/src/scaffolder/jobs/types.ts b/plugins/scaffolder-backend/src/scaffolder/jobs/types.ts index fd554e2033..73da47aed9 100644 --- a/plugins/scaffolder-backend/src/scaffolder/jobs/types.ts +++ b/plugins/scaffolder-backend/src/scaffolder/jobs/types.ts @@ -31,7 +31,7 @@ export type StageContext = { export type ProcessorStatus = 'PENDING' | 'STARTED' | 'COMPLETED' | 'FAILED'; -export interface Stage extends StageInput { +export interface StageResult extends StageInput { log: string[]; status: ProcessorStatus; startedAt?: number; @@ -47,7 +47,7 @@ export type Job = { id: string; context: StageContext; status: ProcessorStatus; - stages: Stage[]; + stages: StageResult[]; error?: Error; }; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/index.ts b/plugins/scaffolder-backend/src/scaffolder/stages/index.ts new file mode 100644 index 0000000000..78ea30db79 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/stages/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export * from './prepare'; +export * from './publish'; +export * from './templater'; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.ts index 7a924d9a99..7a8361d6ea 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/file.ts @@ -43,6 +43,7 @@ export class FilePreparer implements PreparerBase { await fs.copy(parentDirectory, tempDir, { filter: src => src !== location, + recursive: true, }); return tempDir; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts index 86377be6d0..f8552e063f 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/prepare/github.test.ts @@ -19,7 +19,6 @@ const mocks = { CheckoutOptions: jest.fn(() => {}), }; jest.doMock('nodegit', () => mocks); -// require('nodegit'); import { GithubPreparer } from './github'; import { diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts new file mode 100644 index 0000000000..164ec316df --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.test.ts @@ -0,0 +1,203 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +jest.mock('@octokit/rest'); +jest.mock('nodegit'); + +import { Octokit } from '@octokit/rest'; +import * as NodeGit from 'nodegit'; +import { OctokitResponse, ReposCreateInOrgResponseData } from '@octokit/types'; +import { GithubPublisher } from './github'; + +const { mockGithubClient } = require('@octokit/rest') as { + mockGithubClient: { repos: jest.Mocked }; +}; + +const { + Repository, + mockRepo, + mockIndex, + Signature, + Remote, + mockRemote, + Cred, +} = require('nodegit') as { + Repository: jest.Mocked<{ init: any }>; + Signature: jest.Mocked<{ now: any }>; + Cred: jest.Mocked<{ userpassPlaintextNew: any }>; + Remote: jest.Mocked<{ create: any }>; + + mockIndex: jest.Mocked; + mockRepo: jest.Mocked; + mockRemote: jest.Mocked; +}; + +describe('Github Publisher', () => { + const publisher = new GithubPublisher({ client: new Octokit() }); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('publish: createRemoteInGithub', () => { + it('should use octokit to create a repo in an organisation if the organisation property is set', async () => { + mockGithubClient.repos.createInOrg.mockResolvedValue({ + data: { + clone_url: 'mockclone', + }, + } as OctokitResponse); + + await publisher.publish({ + values: { + isOrg: true, + storePath: 'blam/test', + owner: 'bob', + }, + directory: '/tmp/test', + }); + + expect(mockGithubClient.repos.createInOrg).toHaveBeenCalledWith({ + org: 'blam', + name: 'test', + }); + }); + + it('should use octokit to create a repo in the authed user if the organisation property is not set', async () => { + mockGithubClient.repos.createForAuthenticatedUser.mockResolvedValue({ + data: { + clone_url: 'mockclone', + }, + } as OctokitResponse); + + await publisher.publish({ + values: { + storePath: 'blam/test', + owner: 'bob', + }, + directory: '/tmp/test', + }); + + expect( + mockGithubClient.repos.createForAuthenticatedUser, + ).toHaveBeenCalledWith({ + name: 'test', + }); + }); + }); + + describe('publish: createGitDirectory', () => { + const values = { + isOrg: true, + storePath: 'blam/test', + owner: 'lols', + }; + + const mockDir = '/tmp/test/dir'; + + mockGithubClient.repos.createInOrg.mockResolvedValue({ + data: { + clone_url: 'mockclone', + }, + } as OctokitResponse); + it('should call init on the repo with the directory', async () => { + await publisher.publish({ + values, + directory: mockDir, + }); + + expect(Repository.init).toHaveBeenCalledWith(mockDir, 0); + }); + + it('should call refresh index on the index and write the new files', async () => { + await publisher.publish({ + values, + directory: mockDir, + }); + + expect(mockRepo.refreshIndex).toHaveBeenCalled(); + }); + + it('should call add all files and write', async () => { + await publisher.publish({ + values, + directory: mockDir, + }); + + expect(mockIndex.addAll).toHaveBeenCalled(); + expect(mockIndex.write).toHaveBeenCalled(); + expect(mockIndex.writeTree).toHaveBeenCalled(); + }); + + it('should create a commit with on head with the right name and commiter', async () => { + const mockSignature = { mockSignature: 'bloblly' }; + Signature.now.mockReturnValue(mockSignature); + + await publisher.publish({ + values, + directory: mockDir, + }); + + expect(Signature.now).toHaveBeenCalledTimes(2); + expect(Signature.now).toHaveBeenCalledWith( + 'Scaffolder', + 'scaffolder@backstage.io', + ); + + expect(mockRepo.createCommit).toHaveBeenCalledWith( + 'HEAD', + mockSignature, + mockSignature, + 'initial commit', + 'mockoid', + [], + ); + }); + + it('creates a remote with the repo and remote', async () => { + await publisher.publish({ + values, + directory: mockDir, + }); + + expect(Remote.create).toHaveBeenCalledWith( + mockRepo, + 'origin', + 'mockclone', + ); + }); + + it('shoud push to the remote repo', async () => { + await publisher.publish({ + values, + directory: mockDir, + }); + + const [remotes, { callbacks }] = mockRemote.push.mock + .calls[0] as NodeGit.PushOptions[]; + + expect(remotes).toEqual(['refs/heads/master:refs/heads/master']); + + process.env.GITHUb_ACCESS_TOKEN = 'blob'; + + callbacks?.credentials?.(); + + expect(Cred.userpassPlaintextNew).toHaveBeenCalledWith( + process.env.GITHUB_ACCESS_TOKEN, + 'x-oauth-basic', + ); + }); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts new file mode 100644 index 0000000000..eba7def9af --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/github.ts @@ -0,0 +1,84 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Publisher } from './types'; +import { Octokit } from '@octokit/rest'; + +import { JsonValue } from '@backstage/config'; +import { RequiredTemplateValues } from '../templater'; +import { Repository, Remote, Signature, Cred } from 'nodegit'; + +export class GithubPublisher implements Publisher { + private client: Octokit; + constructor({ client }: { client: Octokit }) { + this.client = client; + } + + async publish({ + values, + directory, + }: { + values: RequiredTemplateValues & Record; + directory: string; + }): Promise<{ remoteUrl: string }> { + const remoteUrl = await this.createRemote(values); + await this.pushToRemote(directory, remoteUrl); + + return { remoteUrl }; + } + + private async createRemote( + values: RequiredTemplateValues & Record, + ) { + const [owner, name] = values.storePath.split('/'); + + const repoCreationPromise = values.isOrg + ? this.client.repos.createInOrg({ name, org: owner }) + : this.client.repos.createForAuthenticatedUser({ name }); + + const { data } = await repoCreationPromise; + + return data?.clone_url; + } + + private async pushToRemote(directory: string, remote: string): Promise { + const repo = await Repository.init(directory, 0); + const index = await repo.refreshIndex(); + await index.addAll(); + await index.write(); + const oid = await index.writeTree(); + await repo.createCommit( + 'HEAD', + Signature.now('Scaffolder', 'scaffolder@backstage.io'), + Signature.now('Scaffolder', 'scaffolder@backstage.io'), + 'initial commit', + oid, + [], + ); + + const remoteRepo = await Remote.create(repo, 'origin', remote); + await remoteRepo.push(['refs/heads/master:refs/heads/master'], { + callbacks: { + credentials: () => { + return Cred.userpassPlaintextNew( + process.env.GITHUB_ACCESS_TOKEN as string, + 'x-oauth-basic', + ); + }, + }, + }); + } +} diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/index.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/index.ts new file mode 100644 index 0000000000..dcfd2c9c34 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export * from './github'; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts b/plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts new file mode 100644 index 0000000000..17b357a4ef --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/stages/publish/types.ts @@ -0,0 +1,36 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; +import { RequiredTemplateValues } from '../templater'; +import { JsonValue } from '@backstage/config'; + +/** + * Publisher is in charge of taking a folder created by + * the templater, and pushing it to a remote storage + */ +export type Publisher = { + /** + * + * @param opts object containing the template entity from the service + * catalog, plus the values from the form and the directory that has + * been templated + */ + publish(opts: { + entity: TemplateEntityV1alpha1; + values: RequiredTemplateValues & Record; + directory: string; + }): Promise<{ remoteUrl: string }>; +}; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.test.ts index 994307e8b7..31bcdf189b 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.test.ts @@ -45,8 +45,10 @@ describe('CookieCutter Templater', () => { const tempdir = await mkTemp(); const values = { - component_id: 'test', + owner: 'blobby', + storePath: 'spotify/end-repo', description: 'description', + component_id: 'newthing', }; await cookie.run({ directory: tempdir, values, dockerClient: mockDocker }); @@ -65,8 +67,9 @@ describe('CookieCutter Templater', () => { await fs.writeJSON(`${tempdir}/cookiecutter.json`, existingJson); const values = { - component_id: 'hello', - description: 'im something cool', + owner: 'blobby', + storePath: 'spotify/end-repo', + component_id: 'something', }; await cookie.run({ directory: tempdir, values, dockerClient: mockDocker }); @@ -82,8 +85,8 @@ describe('CookieCutter Templater', () => { await fs.writeFile(`${tempdir}/cookiecutter.json`, "{'"); const values = { - component_id: 'hello', - description: 'im something cool', + owner: 'blobby', + storePath: 'spotify/end-repo', }; await expect( @@ -95,8 +98,9 @@ describe('CookieCutter Templater', () => { const tempdir = await mkTemp(); const values = { - component_id: 'test', - description: 'description', + owner: 'blobby', + storePath: 'spotify/end-repo', + component_id: 'newthing', }; await cookie.run({ directory: tempdir, values, dockerClient: mockDocker }); @@ -122,17 +126,18 @@ describe('CookieCutter Templater', () => { const tempdir = await mkTemp(); const values = { - component_id: 'test', - description: 'description', + owner: 'blobby', + storePath: 'spotify/end-repo', + component_id: 'newthing', }; - const returnPath = await cookie.run({ + const { resultDir } = await cookie.run({ directory: tempdir, values, dockerClient: mockDocker, }); - expect(returnPath.startsWith(`${tempdir}-result`)).toBeTruthy(); + expect(resultDir.startsWith(`${tempdir}-result`)).toBeTruthy(); }); it('should pass through the streamer to the run docker helper', async () => { @@ -141,8 +146,9 @@ describe('CookieCutter Templater', () => { const tempdir = await mkTemp(); const values = { - component_id: 'test', - description: 'description', + owner: 'blobby', + storePath: 'spotify/end-repo', + component_id: 'newthing', }; await cookie.run({ diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts index cef8a64b60..fff349cf82 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/cookiecutter.ts @@ -1,5 +1,3 @@ -import { TemplaterBase, TemplaterRunOptions } from '.'; - /* * Copyright 2020 Spotify AB * @@ -18,6 +16,9 @@ import { TemplaterBase, TemplaterRunOptions } from '.'; import fs from 'fs-extra'; import { JsonValue } from '@backstage/config'; import { runDockerContainer } from './helpers'; +import { TemplaterBase, TemplaterRunOptions } from '.'; +import path from 'path'; +import { TemplaterRunResult } from './types'; export class CookieCutter implements TemplaterBase { private async fetchTemplateCookieCutter( @@ -34,7 +35,7 @@ export class CookieCutter implements TemplaterBase { } } - public async run(options: TemplaterRunOptions): Promise { + public async run(options: TemplaterRunOptions): Promise { // First lets grab the default cookiecutter.json file const cookieCutterJson = await this.fetchTemplateCookieCutter( options.directory, @@ -66,6 +67,8 @@ export class CookieCutter implements TemplaterBase { dockerClient: options.dockerClient, }); - return resultDir; + return { + resultDir: path.resolve(resultDir, options.values.component_id as string), + }; } } diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.test.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.test.ts index d27eb4f93f..02bc801385 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.test.ts @@ -26,6 +26,9 @@ describe('helpers', () => { jest .spyOn(mockDocker, 'run') .mockResolvedValue([{ Error: null, StatusCode: 0 }]); + jest + .spyOn(mockDocker, 'pull') + .mockResolvedValue([{ Error: null, StatusCode: 0 }]); }); describe('runDockerContainer', () => { @@ -34,6 +37,17 @@ describe('helpers', () => { const templateDir = os.tmpdir(); const resultDir = os.tmpdir(); + it('will pull the docker container before running', async () => { + await runDockerContainer({ + imageName, + args, + templateDir, + resultDir, + dockerClient: mockDocker, + }); + + expect(mockDocker.pull).toHaveBeenCalledWith(imageName, {}); + }); it('should call the dockerClient run command with the correct arguments passed through', async () => { await runDockerContainer({ imageName, diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts index 2856a016e4..9ef85224d4 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/helpers.ts @@ -44,6 +44,7 @@ export const runDockerContainer = async ({ templateDir, dockerClient, }: RunDockerContainerOptions) => { + await dockerClient.pull(imageName, {}); const [{ Error: error, StatusCode: statusCode }] = await dockerClient.run( imageName, args, diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/index.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/index.ts index d0bbb0aa6c..0813d0ab86 100644 --- a/plugins/scaffolder-backend/src/scaffolder/stages/templater/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/index.ts @@ -13,45 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -import type { Writable } from 'stream'; -import Docker from 'dockerode'; -import { JsonValue } from '@backstage/config'; - -export interface RequiredTemplateValues { - component_id: string; -} - -export interface TemplaterRunOptions { - directory: string; - values: RequiredTemplateValues & Record; - logStream?: Writable; - dockerClient: Docker; -} - -export type TemplaterBase = { - // runs the templating with the values and returns the directory to push the VCS - run(opts: TemplaterRunOptions): Promise; -}; - -export interface TemplaterConfig { - templater?: TemplaterBase; -} - -class Templater implements TemplaterBase { - templater?: TemplaterBase; - - constructor({ templater }: TemplaterConfig) { - this.templater = templater; - } - - public async run(opts: TemplaterRunOptions) { - return this.templater!.run(opts); - } -} - -export const createTemplater = ( - templaterConfig: TemplaterConfig, -): TemplaterBase => { - return new Templater(templaterConfig); -}; +export * from './cookiecutter'; +export * from './types'; +export * from './helpers'; diff --git a/plugins/scaffolder-backend/src/scaffolder/stages/templater/types.ts b/plugins/scaffolder-backend/src/scaffolder/stages/templater/types.ts new file mode 100644 index 0000000000..519c24e381 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/stages/templater/types.ts @@ -0,0 +1,57 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { Writable } from 'stream'; +import Docker from 'dockerode'; +import { JsonValue } from '@backstage/config'; + +/** + * Currently the required template values. The owner + * and where to store the result from templating + */ +export type RequiredTemplateValues = { + owner: string; + storePath: string; +}; + +/** + * The returned directory from the templater which is ready + * to pass to the next stage of the scaffolder which is publishing + */ +export type TemplaterRunResult = { + resultDir: string; +}; + +/** + * The values that the templater will recieve. The directory of the + * skeleton, with the values from the frontend. A dedicated log stream and a docker + * client to run any templater on top of your directory. + */ +export type TemplaterRunOptions = { + directory: string; + values: RequiredTemplateValues & Record; + logStream?: Writable; + dockerClient: Docker; +}; + +export type TemplaterBase = { + // runs the templating with the values and returns the directory to push the VCS + run(opts: TemplaterRunOptions): Promise; +}; + +export type TemplaterConfig = { + templater?: TemplaterBase; +}; diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 1fb47ff56f..47c91eaed2 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -17,11 +17,19 @@ import { Logger } from 'winston'; import Router from 'express-promise-router'; import express from 'express'; -import { PreparerBuilder, TemplaterBase, JobProcessor } from '../scaffolder'; +import { + PreparerBuilder, + TemplaterBase, + JobProcessor, + RequiredTemplateValues, + StageContext, + GithubPublisher, +} from '../scaffolder'; import { TemplateEntityV1alpha1 } from '@backstage/catalog-model'; import Docker from 'dockerode'; import {} from '@backstage/backend-common'; -import { StageContext } from '../scaffolder/jobs/types'; +import { Octokit } from '@octokit/rest'; +import { JsonValue } from '@backstage/config'; export interface RouterOptions { preparers: PreparerBuilder; @@ -34,9 +42,11 @@ export async function createRouter( options: RouterOptions, ): Promise { const router = Router(); - const { preparers, templater, logger: parentLogger, dockerClient } = options; - const logger = parentLogger.child({ plugin: 'scaffolder' }); + const githubClient = new Octokit({ auth: process.env.GITHUB_ACCESS_TOKEN }); + const { preparers, templater, logger: parentLogger, dockerClient } = options; + const githubPulisher = new GithubPublisher({ client: githubClient }); + const logger = parentLogger.child({ plugin: 'scaffolder' }); const jobProcessor = new JobProcessor(); router @@ -48,7 +58,9 @@ export async function createRouter( return; } - res.send(job.stages[Number(params.index)].log.join('')); + const { log } = job.stages[Number(params.index)] ?? { log: [] }; + + res.send(log.join('')); }) .get('/v1/job/:jobId', ({ params }, res) => { const job = jobProcessor.get(params.jobId); @@ -73,37 +85,14 @@ export async function createRouter( error: job.error, }); }) - .post('/v1/jobs', async (_, res) => { - // TODO(blam): Create a unique job here and return the ID so that - // The end user can poll for updates on the current job - - // TODO(blam): Take this entity from the post body sent from the frontend - const mockEntity: TemplateEntityV1alpha1 = { - apiVersion: 'backstage.io/v1alpha1', - kind: 'Template', - metadata: { - annotations: { - 'backstage.io/managed-by-location': - 'github:https://github.com/benjdlambert/backstage-graphql-template/blob/master/template.yaml', - }, - name: 'graphql-starter', - title: 'GraphQL Service', - description: - 'A GraphQL starter template for backstage to get you up and running\nthe best pracices with GraphQL\n', - uid: '9cf16bad-16e0-4213-b314-c4eec773c50b', - etag: 'ZTkxMjUxMjUtYWY3Yi00MjU2LWFkYWMtZTZjNjU5ZjJhOWM2', - - generation: 1, - }, - spec: { - type: 'cookiecutter', - path: './template', - }, - }; + .post('/v1/jobs', async (req, res) => { + const template: TemplateEntityV1alpha1 = req.body.template; + const values: RequiredTemplateValues & Record = + req.body.values; const job = jobProcessor.create({ - entity: mockEntity, - values: { component_id: 'blob' }, + entity: template, + values, stages: [ { name: 'Prepare the skeleton', @@ -129,15 +118,14 @@ export async function createRouter( }, }, { - name: 'Create VCS Repo', + name: 'Publish template', handler: async (ctx: StageContext<{ resultDir: string }>) => { - ctx.logger.info('Should now create the VCS repo'); - }, - }, - { - name: 'Push to remote', - handler: async ctx => { - ctx.logger.info('Should now push to the remote'); + ctx.logger.info('Should not store the template'); + const { remoteUrl } = await githubPulisher.publish({ + values: ctx.values, + directory: ctx.resultDir, + }); + return { remoteUrl }; }, }, ], diff --git a/yarn.lock b/yarn.lock index 143c2569f2..581662e5eb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2248,6 +2248,18 @@ dependencies: "@octokit/types" "^2.0.0" +"@octokit/core@^3.0.0": + version "3.1.0" + resolved "https://registry.npmjs.org/@octokit/core/-/core-3.1.0.tgz#9c3c9b23f7504668cfa057f143ccbf0c645a0ac9" + integrity sha512-yPyQSmxIXLieEIRikk2w8AEtWkFdfG/LXcw1KvEtK3iP0ENZLW/WYQmdzOKqfSaLhooz4CJ9D+WY79C8ZliACw== + dependencies: + "@octokit/auth-token" "^2.4.0" + "@octokit/graphql" "^4.3.1" + "@octokit/request" "^5.4.0" + "@octokit/types" "^5.0.0" + before-after-hook "^2.1.0" + universal-user-agent "^5.0.0" + "@octokit/endpoint@^5.5.0": version "5.5.3" resolved "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-5.5.3.tgz#0397d1baaca687a4c8454ba424a627699d97c978" @@ -2257,6 +2269,24 @@ is-plain-object "^3.0.0" universal-user-agent "^5.0.0" +"@octokit/endpoint@^6.0.1": + version "6.0.3" + resolved "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-6.0.3.tgz#dd09b599662d7e1b66374a177ab620d8cdf73487" + integrity sha512-Y900+r0gIz+cWp6ytnkibbD95ucEzDSKzlEnaWS52hbCDNcCJYO5mRmWW7HRAnDc7am+N/5Lnd8MppSaTYx1Yg== + dependencies: + "@octokit/types" "^5.0.0" + is-plain-object "^3.0.0" + universal-user-agent "^5.0.0" + +"@octokit/graphql@^4.3.1": + version "4.5.1" + resolved "https://registry.npmjs.org/@octokit/graphql/-/graphql-4.5.1.tgz#162aed1490320b88ce34775b3f6b8de945529fa9" + integrity sha512-qgMsROG9K2KxDs12CO3bySJaYoUu2aic90qpFrv7A8sEBzZ7UFGvdgPKiLw5gOPYEYbS0Xf8Tvf84tJutHPulQ== + dependencies: + "@octokit/request" "^5.3.0" + "@octokit/types" "^5.0.0" + universal-user-agent "^5.0.0" + "@octokit/plugin-enterprise-rest@^6.0.1": version "6.0.1" resolved "https://registry.npmjs.org/@octokit/plugin-enterprise-rest/-/plugin-enterprise-rest-6.0.1.tgz#e07896739618dab8da7d4077c658003775f95437" @@ -2269,6 +2299,13 @@ dependencies: "@octokit/types" "^2.0.1" +"@octokit/plugin-paginate-rest@^2.2.0": + version "2.2.3" + resolved "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-2.2.3.tgz#a6ad4377e7e7832fb4bdd9d421e600cb7640ac27" + integrity sha512-eKTs91wXnJH8Yicwa30jz6DF50kAh7vkcqCQ9D7/tvBAP5KKkg6I2nNof8Mp/65G0Arjsb4QcOJcIEQY+rK1Rg== + dependencies: + "@octokit/types" "^5.0.0" + "@octokit/plugin-request-log@^1.0.0": version "1.0.0" resolved "https://registry.npmjs.org/@octokit/plugin-request-log/-/plugin-request-log-1.0.0.tgz#eef87a431300f6148c39a7f75f8cfeb218b2547e" @@ -2282,6 +2319,14 @@ "@octokit/types" "^2.0.1" deprecation "^2.3.1" +"@octokit/plugin-rest-endpoint-methods@4.0.0": + version "4.0.0" + resolved "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-4.0.0.tgz#b02a2006dda8e908c3f8ab381dd5475ef5a810a8" + integrity sha512-emS6gysz4E9BNi9IrCl7Pm4kR+Az3MmVB0/DoDCmF4U48NbYG3weKyDlgkrz6Jbl4Mu4nDx8YWZwC4HjoTdcCA== + dependencies: + "@octokit/types" "^5.0.0" + deprecation "^2.3.1" + "@octokit/request-error@^1.0.1", "@octokit/request-error@^1.0.2": version "1.2.1" resolved "https://registry.npmjs.org/@octokit/request-error/-/request-error-1.2.1.tgz#ede0714c773f32347576c25649dc013ae6b31801" @@ -2291,6 +2336,15 @@ deprecation "^2.0.0" once "^1.4.0" +"@octokit/request-error@^2.0.0": + version "2.0.2" + resolved "https://registry.npmjs.org/@octokit/request-error/-/request-error-2.0.2.tgz#0e76b83f5d8fdda1db99027ea5f617c2e6ba9ed0" + integrity sha512-2BrmnvVSV1MXQvEkrb9zwzP0wXFNbPJij922kYBTLIlIafukrGOb+ABBT2+c6wZiuyWDH1K1zmjGQ0toN/wMWw== + dependencies: + "@octokit/types" "^5.0.1" + deprecation "^2.0.0" + once "^1.4.0" + "@octokit/request@^5.2.0": version "5.3.2" resolved "https://registry.npmjs.org/@octokit/request/-/request-5.3.2.tgz#1ca8b90a407772a1ee1ab758e7e0aced213b9883" @@ -2305,6 +2359,20 @@ once "^1.4.0" universal-user-agent "^5.0.0" +"@octokit/request@^5.3.0", "@octokit/request@^5.4.0": + version "5.4.5" + resolved "https://registry.npmjs.org/@octokit/request/-/request-5.4.5.tgz#8df65bd812047521f7e9db6ff118c06ba84ac10b" + integrity sha512-atAs5GAGbZedvJXXdjtKljin+e2SltEs48B3naJjqWupYl2IUBbB/CJisyjbNHcKpHzb3E+OYEZ46G8eakXgQg== + dependencies: + "@octokit/endpoint" "^6.0.1" + "@octokit/request-error" "^2.0.0" + "@octokit/types" "^5.0.0" + deprecation "^2.0.0" + is-plain-object "^3.0.0" + node-fetch "^2.3.0" + once "^1.4.0" + universal-user-agent "^5.0.0" + "@octokit/rest@^16.28.4": version "16.43.1" resolved "https://registry.npmjs.org/@octokit/rest/-/rest-16.43.1.tgz#3b11e7d1b1ac2bbeeb23b08a17df0b20947eda6b" @@ -2327,6 +2395,16 @@ once "^1.4.0" universal-user-agent "^4.0.0" +"@octokit/rest@^18.0.0": + version "18.0.0" + resolved "https://registry.npmjs.org/@octokit/rest/-/rest-18.0.0.tgz#7f401d9ce13530ad743dfd519ae62ce49bcc0358" + integrity sha512-4G/a42lry9NFGuuECnua1R1eoKkdBYJap97jYbWDNYBOUboWcM75GJ1VIcfvwDV/pW0lMPs7CEmhHoVrSV5shg== + dependencies: + "@octokit/core" "^3.0.0" + "@octokit/plugin-paginate-rest" "^2.2.0" + "@octokit/plugin-request-log" "^1.0.0" + "@octokit/plugin-rest-endpoint-methods" "4.0.0" + "@octokit/types@^2.0.0", "@octokit/types@^2.0.1": version "2.5.0" resolved "https://registry.npmjs.org/@octokit/types/-/types-2.5.0.tgz#f1bbd147e662ae2c79717d518aac686e58257773" @@ -2334,6 +2412,13 @@ dependencies: "@types/node" ">= 8" +"@octokit/types@^5.0.0", "@octokit/types@^5.0.1": + version "5.0.1" + resolved "https://registry.npmjs.org/@octokit/types/-/types-5.0.1.tgz#5459e9a5e9df8565dcc62c17a34491904d71971e" + integrity sha512-GorvORVwp244fGKEt3cgt/P+M0MGy4xEDbckw+K5ojEezxyMDgCaYPKVct+/eWQfZXOT7uq0xRpmrl/+hliabA== + dependencies: + "@types/node" ">= 8" + "@open-draft/until@^1.0.0": version "1.0.3" resolved "https://registry.npmjs.org/@open-draft/until/-/until-1.0.3.tgz#db9cc719191a62e7d9200f6e7bab21c5b848adca" @@ -5243,7 +5328,7 @@ bcrypt-pbkdf@^1.0.0, bcrypt-pbkdf@^1.0.2: dependencies: tweetnacl "^0.14.3" -before-after-hook@^2.0.0: +before-after-hook@^2.0.0, before-after-hook@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.1.0.tgz#b6c03487f44e24200dd30ca5e6a1979c5d2fb635" integrity sha512-IWIbu7pMqyw3EAJHzzHbWa85b6oud/yfKYg5rqB5hNE8CeMi3nX+2C2sj0HswfblST86hpVEOAb9x34NZd6P7A==