Merge pull request #5415 from SDA-SE/feat/rundocker-api

Refactor the runDockerContainer function to a ContainerRunnerApi interface
This commit is contained in:
Dominik Henneke
2021-05-05 12:00:40 +02:00
committed by GitHub
41 changed files with 689 additions and 371 deletions
-2
View File
@@ -38,7 +38,6 @@
"@gitbeaker/core": "^28.0.2",
"@gitbeaker/node": "^28.0.2",
"@octokit/rest": "^18.5.3",
"@types/dockerode": "^3.2.1",
"@types/express": "^4.17.6",
"@types/git-url-parse": "^9.0.0",
"azure-devops-node-api": "^10.1.1",
@@ -46,7 +45,6 @@
"compression": "^1.7.4",
"cors": "^2.8.5",
"cross-fetch": "^3.0.6",
"dockerode": "^3.2.1",
"express": "^4.17.1",
"express-promise-router": "^4.1.0",
"fs-extra": "^9.0.0",
@@ -17,6 +17,7 @@
import { UrlReader } from '@backstage/backend-common';
import { CatalogApi } from '@backstage/catalog-client';
import { ScmIntegrations } from '@backstage/integration';
import { TemplaterBuilder } from '../../stages';
import { createCatalogRegisterAction } from './catalog';
import { createFetchCookiecutterAction, createFetchPlainAction } from './fetch';
import {
@@ -26,23 +27,14 @@ import {
createPublishGithubPullRequestAction,
createPublishGitlabAction,
} from './publish';
import Docker from 'dockerode';
import { TemplaterBuilder } from '../../stages';
export const createBuiltinActions = (options: {
reader: UrlReader;
integrations: ScmIntegrations;
dockerClient: Docker;
catalogClient: CatalogApi;
templaters: TemplaterBuilder;
}) => {
const {
reader,
integrations,
dockerClient,
templaters,
catalogClient,
} = options;
const { reader, integrations, templaters, catalogClient } = options;
return [
createFetchPlainAction({
@@ -52,7 +44,6 @@ export const createBuiltinActions = (options: {
createFetchCookiecutterAction({
reader,
integrations,
dockerClient,
templaters,
}),
createPublishGithubAction({
@@ -15,16 +15,16 @@
*/
jest.mock('./helpers');
import { getVoidLogger, UrlReader } from '@backstage/backend-common';
import { ConfigReader } from '@backstage/config';
import { ScmIntegrations } from '@backstage/integration';
import mock from 'mock-fs';
import os from 'os';
import { resolve as resolvePath } from 'path';
import { createFetchCookiecutterAction } from './cookiecutter';
import { ScmIntegrations } from '@backstage/integration';
import { ConfigReader } from '@backstage/config';
import { Templaters } from '../../../stages/templater';
import { PassThrough } from 'stream';
import { getVoidLogger, UrlReader } from '@backstage/backend-common';
import { Templaters } from '../../../stages/templater';
import { createFetchCookiecutterAction } from './cookiecutter';
import { fetchContents } from './helpers';
import mock from 'mock-fs';
describe('fetch:cookiecutter', () => {
const integrations = ScmIntegrations.fromConfig(
@@ -40,7 +40,6 @@ describe('fetch:cookiecutter', () => {
const templaters = new Templaters();
const cookiecutterTemplater = { run: jest.fn() };
const mockDockerClient = {};
const mockTmpDir = os.tmpdir();
const mockContext = {
input: {
@@ -67,7 +66,6 @@ describe('fetch:cookiecutter', () => {
const action = createFetchCookiecutterAction({
integrations,
templaters,
dockerClient: mockDockerClient as any,
reader: mockReader,
});
@@ -102,7 +100,6 @@ describe('fetch:cookiecutter', () => {
expect(cookiecutterTemplater.run).toHaveBeenCalledWith({
workspacePath: mockTmpDir,
dockerClient: mockDockerClient,
logStream: mockContext.logStream,
values: mockContext.input.values,
});
@@ -123,7 +120,6 @@ describe('fetch:cookiecutter', () => {
expect(cookiecutterTemplater.run).toHaveBeenCalledWith({
workspacePath: mockTmpDir,
dockerClient: mockDockerClient,
logStream: mockContext.logStream,
values: {
...mockContext.input.values,
@@ -166,7 +162,6 @@ describe('fetch:cookiecutter', () => {
const newAction = createFetchCookiecutterAction({
integrations,
templaters: templatersWithoutCookiecutter,
dockerClient: mockDockerClient as any,
reader: mockReader,
});
@@ -14,24 +14,22 @@
* limitations under the License.
*/
import fs from 'fs-extra';
import { resolve as resolvePath } from 'path';
import Docker from 'dockerode';
import { UrlReader } from '@backstage/backend-common';
import { JsonObject } from '@backstage/config';
import { InputError } from '@backstage/errors';
import { ScmIntegrations } from '@backstage/integration';
import { JsonObject } from '@backstage/config';
import fs from 'fs-extra';
import { resolve as resolvePath } from 'path';
import { TemplaterBuilder, TemplaterValues } from '../../../stages/templater';
import { fetchContents } from './helpers';
import { createTemplateAction } from '../../createTemplateAction';
import { fetchContents } from './helpers';
export function createFetchCookiecutterAction(options: {
dockerClient: Docker;
reader: UrlReader;
integrations: ScmIntegrations;
templaters: TemplaterBuilder;
}) {
const { dockerClient, reader, templaters, integrations } = options;
const { reader, templaters, integrations } = options;
return createTemplateAction<{
url: string;
@@ -134,7 +132,6 @@ export function createFetchCookiecutterAction(options: {
// Will execute the template in ./template and put the result in ./result
await cookiecutter.run({
workspacePath: workDir,
dockerClient,
logStream: ctx.logStream,
values,
});
@@ -19,20 +19,12 @@ import fs from 'fs-extra';
import { Processor, Job, StageContext, StageInput } from './types';
import { TemplateEntityV1alpha1 } from '@backstage/catalog-model';
import * as uuid from 'uuid';
import Docker from 'dockerode';
import path from 'path';
import { TemplaterValues, TemplaterBase } from '../stages/templater';
import { PreparerBuilder } from '../stages/prepare';
import { TemplaterValues } from '../stages/templater';
import { makeLogStream } from './logger';
import { Logger } from 'winston';
import { Config } from '@backstage/config';
export type JobProcessorArguments = {
preparers: PreparerBuilder;
templater: TemplaterBase;
dockerClient: Docker;
};
export type JobAndDirectoryTuple = {
job: Job;
directory: string;
@@ -14,21 +14,19 @@
* limitations under the License.
*/
import { FilePreparer, PreparerBuilder } from './prepare';
import Docker from 'dockerode';
import { TemplaterBuilder, TemplaterValues } from './templater';
import { PublisherBuilder } from './publish';
import { createTemplateAction } from '../actions';
import { FilePreparer, PreparerBuilder } from './prepare';
import { PublisherBuilder } from './publish';
import { TemplaterBuilder, TemplaterValues } from './templater';
type Options = {
dockerClient: Docker;
preparers: PreparerBuilder;
templaters: TemplaterBuilder;
publishers: PublisherBuilder;
};
export function createLegacyActions(options: Options) {
const { dockerClient, preparers, templaters, publishers } = options;
const { preparers, templaters, publishers } = options;
return [
createTemplateAction({
@@ -55,7 +53,6 @@ export function createLegacyActions(options: Options) {
const templater = templaters.get(ctx.input.templater as string);
await templater.run({
workspacePath: ctx.workspacePath,
dockerClient,
logStream: ctx.logStream,
values: ctx.input.values as TemplaterValues,
});
@@ -14,16 +14,14 @@
* limitations under the License.
*/
const runDockerContainer = jest.fn();
const runCommand = jest.fn();
const commandExists = jest.fn();
jest.mock('./helpers', () => ({ runCommand }));
jest.mock('@backstage/backend-common', () => ({ runDockerContainer }));
jest.mock('command-exists-promise', () => commandExists);
jest.mock('fs-extra');
import Docker from 'dockerode';
import { ContainerRunner } from '@backstage/backend-common';
import fs from 'fs-extra';
import parseGitUrl from 'git-url-parse';
import path from 'path';
@@ -31,7 +29,9 @@ import { PassThrough } from 'stream';
import { CookieCutter } from './cookiecutter';
describe('CookieCutter Templater', () => {
const mockDocker = {} as Docker;
const containerRunner: jest.Mocked<ContainerRunner> = {
runContainer: jest.fn(),
};
beforeEach(() => {
jest.clearAllMocks();
@@ -50,11 +50,10 @@ describe('CookieCutter Templater', () => {
jest.spyOn(fs, 'readdir').mockResolvedValueOnce(['newthing'] as any);
const templater = new CookieCutter();
const templater = new CookieCutter({ containerRunner });
await templater.run({
workspacePath: 'tempdir',
values,
dockerClient: mockDocker,
});
expect(fs.ensureDir).toBeCalledWith(path.join('tempdir', 'intermediate'));
@@ -83,11 +82,10 @@ describe('CookieCutter Templater', () => {
},
};
const templater = new CookieCutter();
const templater = new CookieCutter({ containerRunner });
await templater.run({
workspacePath: 'tempdir',
values,
dockerClient: mockDocker,
});
expect(fs.writeJSON).toBeCalledWith(
@@ -115,12 +113,11 @@ describe('CookieCutter Templater', () => {
},
};
const templater = new CookieCutter();
const templater = new CookieCutter({ containerRunner });
await expect(
templater.run({
workspacePath: 'tempdir',
values,
dockerClient: mockDocker,
}),
).rejects.toThrow('BAM');
});
@@ -140,23 +137,16 @@ describe('CookieCutter Templater', () => {
.spyOn(fs, 'realpath')
.mockImplementation(x => Promise.resolve(x.toString()));
const templater = new CookieCutter();
const templater = new CookieCutter({ containerRunner });
await templater.run({
workspacePath: 'tempdir',
values,
dockerClient: mockDocker,
});
expect(runDockerContainer).toHaveBeenCalledWith({
expect(containerRunner.runContainer).toHaveBeenCalledWith({
imageName: 'spotify/backstage-cookiecutter',
args: [
'cookiecutter',
'--no-input',
'-o',
'/output',
'/input',
'--verbose',
],
command: 'cookiecutter',
args: ['--no-input', '-o', '/output', '/input', '--verbose'],
envVars: { HOME: '/tmp' },
mountDirs: {
[path.join('tempdir', 'template')]: '/input',
@@ -164,7 +154,6 @@ describe('CookieCutter Templater', () => {
},
workingDir: '/input',
logStream: undefined,
dockerClient: mockDocker,
});
});
@@ -177,14 +166,13 @@ describe('CookieCutter Templater', () => {
jest.spyOn(fs, 'readdir').mockResolvedValueOnce(['newthing'] as any);
const templater = new CookieCutter();
const templater = new CookieCutter({ containerRunner });
await templater.run({
workspacePath: 'tempdir',
values,
dockerClient: mockDocker,
});
expect(runDockerContainer).toHaveBeenCalledWith(
expect(containerRunner.runContainer).toHaveBeenCalledWith(
expect.objectContaining({
imageName: 'foo/cookiecutter-image-with-extensions',
}),
@@ -205,24 +193,17 @@ describe('CookieCutter Templater', () => {
jest.spyOn(fs, 'readdir').mockResolvedValueOnce(['newthing'] as any);
const templater = new CookieCutter();
const templater = new CookieCutter({ containerRunner });
await templater.run({
workspacePath: 'tempdir',
values,
logStream: stream,
dockerClient: mockDocker,
});
expect(runDockerContainer).toHaveBeenCalledWith({
expect(containerRunner.runContainer).toHaveBeenCalledWith({
imageName: 'spotify/backstage-cookiecutter',
args: [
'cookiecutter',
'--no-input',
'-o',
'/output',
'/input',
'--verbose',
],
command: 'cookiecutter',
args: ['--no-input', '-o', '/output', '/input', '--verbose'],
envVars: { HOME: '/tmp' },
mountDirs: {
[path.join('tempdir', 'template')]: '/input',
@@ -230,7 +211,6 @@ describe('CookieCutter Templater', () => {
},
workingDir: '/input',
logStream: stream,
dockerClient: mockDocker,
});
});
@@ -250,12 +230,11 @@ describe('CookieCutter Templater', () => {
jest.spyOn(fs, 'readdir').mockResolvedValueOnce(['newthing'] as any);
commandExists.mockImplementationOnce(() => () => true);
const templater = new CookieCutter();
const templater = new CookieCutter({ containerRunner });
await templater.run({
workspacePath: 'tempdir',
values,
logStream: stream,
dockerClient: mockDocker,
});
expect(runCommand).toHaveBeenCalledWith({
@@ -280,7 +259,7 @@ describe('CookieCutter Templater', () => {
.spyOn(fs, 'readdir')
.mockImplementationOnce(() => Promise.resolve([]));
const templater = new CookieCutter();
const templater = new CookieCutter({ containerRunner });
await expect(
templater.run({
workspacePath: 'tempdir',
@@ -292,7 +271,6 @@ describe('CookieCutter Templater', () => {
},
},
logStream: stream,
dockerClient: mockDocker,
}),
).rejects.toThrow(/No data generated by cookiecutter/);
});
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { runDockerContainer } from '@backstage/backend-common';
import { ContainerRunner } from '@backstage/backend-common';
import { JsonValue } from '@backstage/config';
import fs from 'fs-extra';
import path from 'path';
@@ -24,6 +24,12 @@ import { TemplaterBase, TemplaterRunOptions } from './types';
const commandExists = require('command-exists-promise');
export class CookieCutter implements TemplaterBase {
private readonly containerRunner: ContainerRunner;
constructor({ containerRunner }: { containerRunner: ContainerRunner }) {
this.containerRunner = containerRunner;
}
private async fetchTemplateCookieCutter(
directory: string,
): Promise<Record<string, JsonValue>> {
@@ -40,7 +46,6 @@ export class CookieCutter implements TemplaterBase {
public async run({
workspacePath,
dockerClient,
values,
logStream,
}: TemplaterRunOptions): Promise<void> {
@@ -74,23 +79,16 @@ export class CookieCutter implements TemplaterBase {
logStream,
});
} else {
await runDockerContainer({
await this.containerRunner.runContainer({
imageName: imageName || 'spotify/backstage-cookiecutter',
args: [
'cookiecutter',
'--no-input',
'-o',
'/output',
'/input',
'--verbose',
],
command: 'cookiecutter',
args: ['--no-input', '-o', '/output', '/input', '--verbose'],
mountDirs,
workingDir: '/input',
// Set the home directory inside the container as something that applications can
// write to, otherwise they will just fail trying to write to /
envVars: { HOME: '/tmp' },
logStream,
dockerClient,
});
}
@@ -14,7 +14,7 @@
* limitations under the License.
*/
import { runDockerContainer } from '@backstage/backend-common';
import { ContainerRunner } from '@backstage/backend-common';
import fs from 'fs-extra';
import path from 'path';
import * as yaml from 'yaml';
@@ -25,11 +25,16 @@ import { TemplaterBase, TemplaterRunOptions } from '../types';
const GITHUB_ACTIONS_ANNOTATION = 'github.com/project-slug';
export class CreateReactAppTemplater implements TemplaterBase {
private readonly containerRunner: ContainerRunner;
constructor({ containerRunner }: { containerRunner: ContainerRunner }) {
this.containerRunner = containerRunner;
}
public async run({
workspacePath,
values,
logStream,
dockerClient,
}: TemplaterRunOptions): Promise<void> {
const {
component_id: componentName,
@@ -46,23 +51,20 @@ export class CreateReactAppTemplater implements TemplaterBase {
[intermediateDir]: '/result',
};
await runDockerContainer({
await this.containerRunner.runContainer({
imageName: 'node:lts-alpine',
command: ['npx'],
args: [
'create-react-app',
componentName as string,
withTypescript ? ' --template typescript' : '',
],
mountDirs,
workingDir: '/result',
logStream: logStream,
dockerClient: dockerClient,
// Set the home directory inside the container as something that applications can
// write to, otherwise they will just fail trying to write to /
envVars: { HOME: '/tmp' },
createOptions: {
Entrypoint: ['npx'],
WorkingDir: '/result',
},
});
// if cookiecutter was successful, intermediateDir will contain
@@ -14,10 +14,15 @@
* limitations under the License.
*/
import { ContainerRunner } from '@backstage/backend-common';
import { CookieCutter } from './cookiecutter';
import { Templaters } from './templaters';
describe('Templaters', () => {
const containerRunner: jest.Mocked<ContainerRunner> = {
runContainer: jest.fn(),
};
it('should throw an error when the templater is not registered', () => {
const templaters = new Templaters();
@@ -29,7 +34,7 @@ describe('Templaters', () => {
});
it('should return the correct templater when the templater matches', () => {
const templaters = new Templaters();
const templater = new CookieCutter();
const templater = new CookieCutter({ containerRunner });
templaters.register('cookiecutter', templater);
@@ -13,9 +13,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type { Writable } from 'stream';
import Docker from 'dockerode';
import gitUrlParse from 'git-url-parse';
import type { Writable } from 'stream';
/**
* Currently the required template values. The owner
@@ -48,7 +48,6 @@ export type TemplaterRunOptions = {
workspacePath: string;
values: TemplaterValues;
logStream?: Writable;
dockerClient: Docker;
};
export type TemplaterBase = {
@@ -29,20 +29,17 @@ jest.doMock('fs-extra', () => ({
}));
import {
SingleConnectionDatabaseManager,
PluginDatabaseManager,
getVoidLogger,
PluginDatabaseManager,
SingleConnectionDatabaseManager,
UrlReaders,
} from '@backstage/backend-common';
import { CatalogApi } from '@backstage/catalog-client';
import { ConfigReader } from '@backstage/config';
import express from 'express';
import request from 'supertest';
import { Preparers, Publishers, Templaters } from '../scaffolder';
import { createRouter } from './router';
import { Templaters, Preparers, Publishers } from '../scaffolder';
import Docker from 'dockerode';
import { CatalogApi } from '@backstage/catalog-client';
jest.mock('dockerode');
const createCatalogClient = (templates: any[] = []) =>
({
@@ -115,7 +112,6 @@ describe('createRouter - working directory', () => {
templaters: new Templaters(),
publishers: new Publishers(),
config: new ConfigReader(workDirConfig('/path')),
dockerClient: new Docker(),
database: createDatabase(),
catalogClient: createCatalogClient([template]),
reader: mockUrlReader,
@@ -130,7 +126,6 @@ describe('createRouter - working directory', () => {
templaters: new Templaters(),
publishers: new Publishers(),
config: new ConfigReader(workDirConfig('/path')),
dockerClient: new Docker(),
database: createDatabase(),
catalogClient: createCatalogClient([template]),
reader: mockUrlReader,
@@ -160,7 +155,6 @@ describe('createRouter - working directory', () => {
templaters: new Templaters(),
publishers: new Publishers(),
config: new ConfigReader({}),
dockerClient: new Docker(),
database: createDatabase(),
catalogClient: createCatalogClient([template]),
reader: mockUrlReader,
@@ -234,7 +228,6 @@ describe('createRouter', () => {
templaters: new Templaters(),
publishers: new Publishers(),
config: new ConfigReader({}),
dockerClient: new Docker(),
database: createDatabase(),
catalogClient: createCatalogClient([template]),
reader: mockUrlReader,
@@ -15,7 +15,6 @@
*/
import { Config } from '@backstage/config';
import Docker from 'dockerode';
import express from 'express';
import { resolve as resolvePath, dirname } from 'path';
import Router from 'express-promise-router';
@@ -62,7 +61,6 @@ export interface RouterOptions {
logger: Logger;
config: Config;
reader: UrlReader;
dockerClient: Docker;
database: PluginDatabaseManager;
catalogClient: CatalogApi;
actions?: TemplateAction<any>[];
@@ -96,7 +94,6 @@ export async function createRouter(
logger: parentLogger,
config,
reader,
dockerClient,
database,
catalogClient,
actions,
@@ -124,13 +121,11 @@ export async function createRouter(
? actions
: [
...createLegacyActions({
dockerClient,
preparers,
publishers,
templaters,
}),
...createBuiltinActions({
dockerClient,
integrations,
catalogClient,
templaters,
@@ -243,7 +238,6 @@ export async function createRouter(
const templater = templaters.get(ctx.entity.spec.templater);
await templater.run({
workspacePath: ctx.workspacePath,
dockerClient,
logStream: ctx.logStream,
values: ctx.values,
});
+1 -1
View File
@@ -35,7 +35,6 @@
"@backstage/config": "^0.1.4",
"@backstage/errors": "^0.1.1",
"@backstage/techdocs-common": "^0.5.0",
"@types/dockerode": "^3.2.1",
"@types/express": "^4.17.6",
"cross-fetch": "^3.0.6",
"dockerode": "^3.2.1",
@@ -47,6 +46,7 @@
},
"devDependencies": {
"@backstage/cli": "^0.6.8",
"@types/dockerode": "^3.2.1",
"supertest": "^6.1.3"
},
"files": [
@@ -28,7 +28,6 @@ import {
PublisherBase,
UrlPreparer,
} from '@backstage/techdocs-common';
import Docker from 'dockerode';
import fs from 'fs-extra';
import os from 'os';
import path from 'path';
@@ -41,7 +40,6 @@ type DocsBuilderArguments = {
publisher: PublisherBase;
entity: Entity;
logger: Logger;
dockerClient: Docker;
};
export class DocsBuilder {
@@ -50,7 +48,6 @@ export class DocsBuilder {
private publisher: PublisherBase;
private entity: Entity;
private logger: Logger;
private dockerClient: Docker;
constructor({
preparers,
@@ -58,14 +55,12 @@ export class DocsBuilder {
publisher,
entity,
logger,
dockerClient,
}: DocsBuilderArguments) {
this.preparer = preparers.get(entity);
this.generator = generators.get(entity);
this.publisher = publisher;
this.entity = entity;
this.logger = logger;
this.dockerClient = dockerClient;
}
public async build(): Promise<void> {
@@ -157,7 +152,6 @@ export class DocsBuilder {
await this.generator.run({
inputDir: preparedDir,
outputDir,
dockerClient: this.dockerClient,
parsedLocationAnnotation,
etag: newEtag,
});
@@ -24,7 +24,6 @@ import {
PublisherBase,
} from '@backstage/techdocs-common';
import fetch from 'cross-fetch';
import Docker from 'dockerode';
import express from 'express';
import Router from 'express-promise-router';
import { Knex } from 'knex';
@@ -40,7 +39,6 @@ type RouterOptions = {
discovery: PluginEndpointDiscovery;
database?: Knex; // TODO: Make database required when we're implementing database stuff.
config: Config;
dockerClient: Docker;
};
export async function createRouter({
@@ -48,7 +46,6 @@ export async function createRouter({
generators,
publisher,
config,
dockerClient,
logger,
discovery,
}: RouterOptions): Promise<express.Router> {
@@ -165,7 +162,6 @@ export async function createRouter({
preparers,
generators,
publisher,
dockerClient,
logger,
entity,
});
@@ -16,21 +16,22 @@
import {
createServiceBuilder,
DockerContainerRunner,
SingleHostDiscovery,
UrlReader,
} from '@backstage/backend-common';
import { ConfigReader } from '@backstage/config';
import {
DirectoryPreparer,
Generators,
Preparers,
Publisher,
TechdocsGenerator,
} from '@backstage/techdocs-common';
import Docker from 'dockerode';
import { Server } from 'http';
import { Logger } from 'winston';
import { createRouter } from './router';
import Docker from 'dockerode';
import {
Preparers,
DirectoryPreparer,
Generators,
TechdocsGenerator,
Publisher,
} from '@backstage/techdocs-common';
import { ConfigReader } from '@backstage/config';
export interface ServerOptions {
port: number;
@@ -65,21 +66,25 @@ export async function startStandaloneServer(
);
preparers.register('dir', directoryPreparer);
const dockerClient = new Docker();
const containerRunner = new DockerContainerRunner({ dockerClient });
const generators = new Generators();
const techdocsGenerator = new TechdocsGenerator(logger, config);
const techdocsGenerator = new TechdocsGenerator({
logger,
containerRunner,
config,
});
generators.register('techdocs', techdocsGenerator);
const publisher = await Publisher.fromConfig(config, { logger, discovery });
const dockerClient = new Docker();
logger.debug('Starting application server...');
const router = await createRouter({
preparers,
generators,
logger,
publisher,
dockerClient,
config,
discovery,
});