paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 32 Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:backstage/backstage into RoadieHQ-5identity-api-client-interface

Browse Source
This commit is contained in:
Brian Fletcher
2022-09-01 10:51:52 +01:00
parent 861ab75dd0 af7594d587
commit 04370684e8
245 changed files with 65219 additions and 190669 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/scaffolder-backend/api-report.md
+13
View File
@@ -572,6 +572,19 @@ export class ScaffolderEntitiesProcessor implements CatalogProcessor {
validateEntityKind(entity: Entity): Promise<boolean>;
}
// @alpha
export const scaffolderPlugin: (
options: ScaffolderPluginOptions,
) => BackendFeature;
// @alpha
export type ScaffolderPluginOptions = {
actions?: TemplateAction<any>[];
taskWorkers?: number;
taskBroker?: TaskBroker;
additionalTemplateFilters?: Record<string, TemplateFilter>;
};
// @public
export type SerializedTask = {
id: string;
plugins/scaffolder-backend/package.json
+5 -6
View File
@@ -5,7 +5,6 @@
"main": "src/index.ts",
"types": "src/index.ts",
"license": "Apache-2.0",
"private": false,
"publishConfig": {
"access": "public",
"main": "dist/index.cjs.js",
@@ -36,6 +35,7 @@
},
"dependencies": {
"@backstage/backend-common": "^0.15.1-next.1",
"@backstage/backend-plugin-api": "^0.1.2-next.0",
"@backstage/catalog-client": "^1.0.5-next.0",
"@backstage/catalog-model": "^1.1.0",
"@backstage/config": "^1.0.1",
@@ -43,9 +43,8 @@
"@backstage/integration": "^1.3.1-next.0",
"@backstage/plugin-auth-node": "^0.2.5-next.1",
"@backstage/plugin-catalog-backend": "^1.4.0-next.1",
"@backstage/plugin-scaffolder-common": "^1.2.0-next.0",
"@backstage/backend-plugin-api": "^0.1.2-next.0",
"@backstage/plugin-catalog-node": "^1.0.2-next.0",
"@backstage/plugin-scaffolder-common": "^1.2.0-next.0",
"@backstage/types": "^1.0.0",
"@gitbeaker/core": "^35.6.0",
"@gitbeaker/node": "^35.1.0",
@@ -58,7 +57,7 @@
"express": "^4.17.1",
"express-promise-router": "^4.1.0",
"fs-extra": "10.1.0",
"git-url-parse": "^12.0.0",
"git-url-parse": "^13.0.0",
"globby": "^11.0.0",
"isbinaryfile": "^5.0.0",
"isomorphic-git": "^1.8.0",
@@ -73,9 +72,9 @@
"octokit-plugin-create-pull-request": "^3.10.0",
"p-limit": "^3.1.0",
"uuid": "^8.2.0",
"vm2": "^3.9.11",
"winston": "^3.2.1",
"yaml": "^2.0.0",
"vm2": "^3.9.11",
"zen-observable": "^0.8.15",
"zod": "^3.11.6"
},
@@ -92,7 +91,7 @@
"esbuild": "^0.14.1",
"jest-when": "^3.1.0",
"mock-fs": "^5.1.0",
"msw": "^0.45.0",
"msw": "^0.46.0",
"supertest": "^6.1.3",
"yaml": "^2.0.0"
},
plugins/scaffolder-backend/src/ScaffolderPlugin.ts
+138
View File
@@ -0,0 +1,138 @@
/*
* Copyright 2022 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
configServiceRef,
createBackendPlugin,
databaseServiceRef,
loggerServiceRef,
loggerToWinstonLogger,
permissionsServiceRef,
urlReaderServiceRef,
httpRouterServiceRef,
createExtensionPoint,
} from '@backstage/backend-plugin-api';
import { ScmIntegrations } from '@backstage/integration';
import { catalogServiceRef } from '@backstage/plugin-catalog-node';
import { TemplateFilter } from './lib';
import { createBuiltinActions, TaskBroker, TemplateAction } from './scaffolder';
import { createRouter } from './service/router';
/**
* Catalog plugin options
* @alpha
*/
export type ScaffolderPluginOptions = {
actions?: TemplateAction<any>[];
taskWorkers?: number;
taskBroker?: TaskBroker;
additionalTemplateFilters?: Record<string, TemplateFilter>;
};
/**
* @alpha
* TODO: MOVE to scaffolder-node.
*/
interface ScaffolderActionsExtensionPoint {
addActions(...actions: TemplateAction<any>[]): void;
}
class ScaffolderActionsExtensionPointImpl
implements ScaffolderActionsExtensionPoint
{
#actions = new Array<TemplateAction<any>>();
addActions(...actions: TemplateAction<any>[]): void {
this.#actions.push(...actions);
}
get actions() {
return this.#actions;
}
}
/**
* @alpha
* TODO: MOVE to scaffolder-node.
*/
export const scaffolderActionsExtensionPoint =
createExtensionPoint<ScaffolderActionsExtensionPoint>({
id: 'scaffolder.actions',
});
/**
* Catalog plugin
* @alpha
*/
export const scaffolderPlugin = createBackendPlugin({
id: 'scaffolder',
register(env, options: ScaffolderPluginOptions) {
const actionsExtensions = new ScaffolderActionsExtensionPointImpl();
env.registerExtensionPoint(
scaffolderActionsExtensionPoint,
actionsExtensions,
);
env.registerInit({
deps: {
logger: loggerServiceRef,
config: configServiceRef,
reader: urlReaderServiceRef,
permissions: permissionsServiceRef,
database: databaseServiceRef,
httpRouter: httpRouterServiceRef,
catalogClient: catalogServiceRef,
},
async init({
logger,
config,
reader,
database,
httpRouter,
catalogClient,
}) {
const { additionalTemplateFilters, taskBroker, taskWorkers } = options;
const log = loggerToWinstonLogger(logger);
const actions = options.actions || [
...actionsExtensions.actions,
...createBuiltinActions({
integrations: ScmIntegrations.fromConfig(config),
catalogClient,
reader,
config,
additionalTemplateFilters,
}),
];
const actionIds = actions.map(action => action.id).join(', ');
log.info(
`Starting scaffolder with the following actions enabled ${actionIds}`,
);
const router = await createRouter({
logger: log,
config,
database,
catalogClient,
reader,
actions,
taskBroker,
taskWorkers,
additionalTemplateFilters,
});
httpRouter.use(router);
},
});
},
});
plugins/scaffolder-backend/src/index.ts
+2
View File
@@ -25,3 +25,5 @@ export * from './service/router';
export * from './lib';
export * from './processor';
export * from './extension';
export { scaffolderPlugin } from './ScaffolderPlugin';
export type { ScaffolderPluginOptions } from './ScaffolderPlugin';
plugins/scaffolder-backend/src/lib/files/deserializeDirectoryContents.test.ts
+4
View File
@@ -41,6 +41,7 @@ describe('deserializeDirectoryContents', () => {
path: 'a.txt',
content: Buffer.from('a', 'utf8'),
executable: false,
symlink: false,
},
]);
});
@@ -65,16 +66,19 @@ describe('deserializeDirectoryContents', () => {
path: 'a.txt',
content: Buffer.from('a', 'utf8'),
executable: false,
symlink: false,
},
{
path: 'a/b.txt',
content: Buffer.from('b', 'utf8'),
executable: false,
symlink: false,
},
{
path: 'a/b/c.txt',
content: Buffer.from('c', 'utf8'),
executable: false,
symlink: false,
},
]);
});
plugins/scaffolder-backend/src/lib/files/serializeDirectoryContents.test.ts
+38
View File
@@ -28,21 +28,25 @@ describe('serializeDirectoryContents', () => {
{
path: 'index.ts',
executable: false,
symlink: false,
content: expect.any(Buffer),
},
{
path: 'types.ts',
executable: false,
symlink: false,
content: expect.any(Buffer),
},
{
path: 'serializeDirectoryContents.ts',
executable: false,
symlink: false,
content: expect.any(Buffer),
},
{
path: 'serializeDirectoryContents.test.ts',
executable: false,
symlink: false,
content: expect.any(Buffer),
},
]),
@@ -72,26 +76,31 @@ describe('serializeDirectoryContents', () => {
{
path: 'a.txt',
executable: false,
symlink: false,
content: Buffer.from('a', 'utf8'),
},
{
path: 'b/b1.txt',
executable: false,
symlink: false,
content: Buffer.from('b1', 'utf8'),
},
{
path: 'b/b2.txt',
executable: false,
symlink: false,
content: Buffer.from('b2', 'utf8'),
},
{
path: 'c/c1/c11.txt',
executable: false,
symlink: false,
content: Buffer.from('c11', 'utf8'),
},
{
path: 'c/c1/c11/c111.txt',
executable: false,
symlink: false,
content: Buffer.from('c111', 'utf8'),
},
]);
@@ -111,11 +120,31 @@ describe('serializeDirectoryContents', () => {
{
path: 'a.txt',
executable: false,
symlink: false,
content: Buffer.from('some text', 'utf8'),
},
]);
});
it('should pick up broken symlinks', async () => {
mockFs({
root: {
'b.txt': mockFs.symlink({
path: './a.txt',
}),
},
});
await expect(serializeDirectoryContents('root')).resolves.toEqual([
{
path: 'b.txt',
executable: false,
symlink: true,
content: Buffer.from('./a.txt', 'utf8'),
},
]);
});
it('should ignore symlinked folder files', async () => {
mockFs({
root: {
@@ -133,11 +162,13 @@ describe('serializeDirectoryContents', () => {
{
path: 'a.txt',
executable: false,
symlink: false,
content: Buffer.from('some text', 'utf8'),
},
{
path: 'linkme/b.txt',
executable: false,
symlink: false,
content: Buffer.from('lols', 'utf8'),
},
]);
@@ -160,11 +191,13 @@ describe('serializeDirectoryContents', () => {
{
path: '.gitignore',
executable: false,
symlink: false,
content: Buffer.from('*.txt', 'utf8'),
},
{
path: 'a.log',
executable: false,
symlink: false,
content: Buffer.from('a', 'utf8'),
},
]);
@@ -198,26 +231,31 @@ describe('serializeDirectoryContents', () => {
{
path: 'a.txt',
executable: false,
symlink: false,
content: Buffer.from('a', 'utf8'),
},
{
path: 'b/.b',
executable: false,
symlink: false,
content: Buffer.from('b', 'utf8'),
},
{
path: 'b/b.txt',
executable: false,
symlink: false,
content: Buffer.from('b', 'utf8'),
},
{
path: 'c/c.log',
executable: false,
symlink: false,
content: Buffer.from('c', 'utf8'),
},
{
path: 'c/c.txt',
executable: false,
symlink: false,
content: Buffer.from('c', 'utf8'),
},
]);
plugins/scaffolder-backend/src/lib/files/serializeDirectoryContents.ts
+38 -6
View File
@@ -14,11 +14,12 @@
* limitations under the License.
*/
import fs from 'fs-extra';
import { promises as fs } from 'fs';
import globby from 'globby';
import limiterFactory from 'p-limit';
import { join as joinPath } from 'path';
import { resolveSafeChildPath } from '@backstage/backend-common';
import { SerializedFile } from './types';
import { isError } from '@backstage/errors';
const DEFAULT_GLOB_PATTERNS = ['./**', '!.git'];
@@ -32,6 +33,14 @@ export const isExecutable = (fileMode: number | undefined) => {
return res > 0;
};
async function asyncFilter<T>(
array: T[],
callback: (value: T, index: number, array: T[]) => Promise<boolean>,
): Promise<T[]> {
const filterMap = await Promise.all(array.map(callback));
return array.filter((_value, index) => filterMap[index]);
}
export async function serializeDirectoryContents(
sourcePath: string,
options?: {
@@ -44,19 +53,42 @@ export async function serializeDirectoryContents(
dot: true,
gitignore: options?.gitignore,
followSymbolicLinks: false,
// In order to pick up 'broken' symlinks, we oxymoronically request files AND folders yet we filter out folders
// This is because broken symlinks aren't classed as files so we need to glob everything
onlyFiles: false,
objectMode: true,
stats: true,
});
const limiter = limiterFactory(10);
const valid = await asyncFilter(paths, async ({ dirent, path }) => {
if (dirent.isDirectory()) return false;
if (!dirent.isSymbolicLink()) return true;
const safePath = resolveSafeChildPath(sourcePath, path);
// we only want files that don't exist
try {
await fs.stat(safePath);
return false;
} catch (e) {
return isError(e) && e.code === 'ENOENT';
}
});
return Promise.all(
paths.map(async ({ path, stats }) => ({
valid.map(async ({ dirent, path, stats }) => ({
path,
content: await limiter(async () =>
fs.readFile(joinPath(sourcePath, path)),
),
content: await limiter(async () => {
const absFilePath = resolveSafeChildPath(sourcePath, path);
if (dirent.isSymbolicLink()) {
return fs.readlink(absFilePath, 'buffer');
}
return fs.readFile(absFilePath);
}),
executable: isExecutable(stats?.mode),
symlink: dirent.isSymbolicLink(),
})),
);
}
plugins/scaffolder-backend/src/lib/files/types.ts
+1
View File
@@ -18,4 +18,5 @@ export interface SerializedFile {
path: string;
content: Buffer;
executable?: boolean;
symlink?: boolean;
}
plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/template.test.ts
+14
View File
@@ -297,6 +297,9 @@ describe('fetch:template', () => {
symlink: mockFs.symlink({
path: 'a-binary-file.png',
}),
brokenSymlink: mockFs.symlink({
path: './not-a-real-file.txt',
}),
},
});
@@ -371,6 +374,17 @@ describe('fetch:template', () => {
fs.realpath(`${workspacePath}/target/symlink`),
).resolves.toBe(joinPath(workspacePath, 'target', 'a-binary-file.png'));
});
it('copies broken symlinks as-is without processing them', async () => {
await expect(
fs
.lstat(`${workspacePath}/target/brokenSymlink`)
.then(i => i.isSymbolicLink()),
).resolves.toBe(true);
await expect(
fs.readlink(`${workspacePath}/target/brokenSymlink`),
).resolves.toEqual('./not-a-real-file.txt');
});
});
});
plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.test.ts
+28 -1
View File
@@ -122,6 +122,7 @@ describe('publish:azure', () => {
it('should not throw if there is a token provided through ctx.input', async () => {
mockGitClient.createRepository.mockImplementation(() => ({
remoteUrl: 'http://google.com',
id: '709e891c-dee7-4f91-b963-534713c0737f',
}));
await action.handler({
@@ -148,6 +149,7 @@ describe('publish:azure', () => {
it('should throw if there is no remoteUrl returned', async () => {
mockGitClient.createRepository.mockImplementation(() => ({
remoteUrl: null,
id: '709e891c-dee7-4f91-b963-534713c0737f',
}));
await expect(
action.handler({
@@ -159,9 +161,25 @@ describe('publish:azure', () => {
).rejects.toThrow(/No remote URL returned/);
});
it('should throw if there is no repositoryId returned', async () => {
mockGitClient.createRepository.mockImplementation(() => ({
remoteUrl: 'http://google.com',
id: null,
}));
await expect(
action.handler({
...mockContext,
input: {
repoUrl: 'dev.azure.com?repo=bob&owner=owner&organization=org',
},
}),
).rejects.toThrow(/No Id returned/);
});
it('should call the azureApis with the correct values', async () => {
mockGitClient.createRepository.mockImplementation(() => ({
remoteUrl: 'http://google.com',
id: '709e891c-dee7-4f91-b963-534713c0737f',
}));
await action.handler(mockContext);
@@ -182,6 +200,7 @@ describe('publish:azure', () => {
it('should call initRepoAndPush with the correct values', async () => {
mockGitClient.createRepository.mockImplementation(() => ({
remoteUrl: 'https://dev.azure.com/organization/project/_git/repo',
id: '709e891c-dee7-4f91-b963-534713c0737f',
}));
await action.handler(mockContext);
@@ -200,6 +219,7 @@ describe('publish:azure', () => {
it('should call initRepoAndPush with the correct default branch', async () => {
mockGitClient.createRepository.mockImplementation(() => ({
remoteUrl: 'https://dev.azure.com/organization/project/_git/repo',
id: '709e891c-dee7-4f91-b963-534713c0737f',
}));
await action.handler({
@@ -246,6 +266,7 @@ describe('publish:azure', () => {
mockGitClient.createRepository.mockImplementation(() => ({
remoteUrl: 'https://dev.azure.com/organization/project/_git/repo',
id: '709e891c-dee7-4f91-b963-534713c0737f',
}));
await customAuthorAction.handler(mockContext);
@@ -283,6 +304,7 @@ describe('publish:azure', () => {
mockGitClient.createRepository.mockImplementation(() => ({
remoteUrl: 'https://dev.azure.com/organization/project/_git/repo',
id: '709e891c-dee7-4f91-b963-534713c0737f',
}));
await customAuthorAction.handler(mockContext);
@@ -298,9 +320,10 @@ describe('publish:azure', () => {
});
});
it('should call output with the remoteUrl and the repoContentsUrl', async () => {
it('should call output with the remoteUrl the repoContentsUrl and the repositoryId', async () => {
mockGitClient.createRepository.mockImplementation(() => ({
remoteUrl: 'https://dev.azure.com/organization/project/_git/repo',
id: '709e891c-dee7-4f91-b963-534713c0737f',
}));
await action.handler(mockContext);
@@ -313,5 +336,9 @@ describe('publish:azure', () => {
'repoContentsUrl',
'https://dev.azure.com/organization/project/_git/repo',
);
expect(mockContext.output).toHaveBeenCalledWith(
'repositoryId',
'709e891c-dee7-4f91-b963-534713c0737f',
);
});
});
plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/azure.ts
+10
View File
@@ -104,6 +104,10 @@ export function createPublishAzureAction(options: {
title: 'A URL to the root of the repository',
type: 'string',
},
repositoryId: {
title: 'The Id of the created repository',
type: 'string',
},
},
},
},
@@ -160,6 +164,11 @@ export function createPublishAzureAction(options: {
'No remote URL returned from create repository for Azure',
);
}
const repositoryId = returnedRepo.id;
if (!repositoryId) {
throw new InputError('No Id returned from create repository for Azure');
}
// blam: Repo contents is serialized into the path,
// so it's just the base path I think
@@ -191,6 +200,7 @@ export function createPublishAzureAction(options: {
ctx.output('remoteUrl', remoteUrl);
ctx.output('repoContentsUrl', repoContentsUrl);
ctx.output('repositoryId', repositoryId);
},
});
}
plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/githubPullRequest.test.ts
+54
View File
@@ -359,6 +359,60 @@ describe('createPublishGithubPullRequestAction', () => {
});
});
describe('with broken symlink', () => {
let input: GithubPullRequestActionInput;
let ctx: ActionContext<GithubPullRequestActionInput>;
beforeEach(() => {
input = {
repoUrl: 'github.com?owner=myorg&repo=myrepo',
title: 'Create my new app',
branchName: 'new-app',
description: 'This PR is really good',
};
mockFs({
[workspacePath]: {
Makefile: mockFs.symlink({
path: '../../nothing/yet',
}),
},
});
ctx = {
createTemporaryDirectory: jest.fn(),
output: jest.fn(),
logger: getRootLogger(),
logStream: new Writable(),
input,
workspacePath,
};
});
it('creates a pull request', async () => {
await instance.handler(ctx);
expect(fakeClient.createPullRequest).toHaveBeenCalledWith({
owner: 'myorg',
repo: 'myrepo',
title: 'Create my new app',
head: 'new-app',
body: 'This PR is really good',
changes: [
{
commit: 'Create my new app',
files: {
Makefile: {
content: Buffer.from('../../nothing/yet').toString('utf-8'),
encoding: 'utf-8',
mode: '120000',
},
},
},
],
});
});
});
describe('with executable file mode 755', () => {
let input: GithubPullRequestActionInput;
let ctx: ActionContext<GithubPullRequestActionInput>;
plugins/scaffolder-backend/src/scaffolder/actions/builtin/publish/githubPullRequest.ts
+21 -6
View File
@@ -26,7 +26,10 @@ import { InputError, CustomErrorBase } from '@backstage/errors';
import { createPullRequest } from 'octokit-plugin-create-pull-request';
import { resolveSafeChildPath } from '@backstage/backend-common';
import { getOctokitOptions } from '../github/helpers';
import { serializeDirectoryContents } from '../../../../lib/files';
import {
SerializedFile,
serializeDirectoryContents,
} from '../../../../lib/files';
import { Logger } from 'winston';
export type Encoding = 'utf-8' | 'base64';
@@ -249,21 +252,33 @@ export const createPublishGithubPullRequestAction = ({
const directoryContents = await serializeDirectoryContents(fileRoot, {
gitignore: true,
});
const determineFileMode = (file: SerializedFile): string => {
if (file.symlink) return '120000';
if (file.executable) return '100755';
return '100644';
};
const determineFileEncoding = (
file: SerializedFile,
): 'utf-8' | 'base64' => (file.symlink ? 'utf-8' : 'base64');
const files = Object.fromEntries(
directoryContents.map(file => [
targetPath ? path.posix.join(targetPath, file.path) : file.path,
{
// See the properties of tree items
// in https://docs.github.com/en/rest/reference/git#trees
mode: file.executable ? '100755' : '100644',
// Always use base64 encoding to avoid doubling a binary file in size
mode: determineFileMode(file),
// Always use base64 encoding where possible to avoid doubling a binary file in size
// due to interpreting a binary file as utf-8 and sending github
// the utf-8 encoded content.
// the utf-8 encoded content. Symlinks are kept as utf-8 to avoid them
// being formatted as a series of scrambled characters
//
// For example, the original gradle-wrapper.jar is 57.8k in https://github.com/kennethzfeng/pull-request-test/pull/5/files.
// Its size could be doubled to 98.3K (See https://github.com/kennethzfeng/pull-request-test/pull/4/files)
encoding: 'base64' as const,
content: file.content.toString('base64'),
encoding: determineFileEncoding(file),
content: file.content.toString(determineFileEncoding(file)),
},
]),
);