From 6fa98f941bf0b53fc22918f53862cf11f463359f Mon Sep 17 00:00:00 2001 From: blam Date: Wed, 24 Feb 2021 17:17:51 +0100 Subject: [PATCH] feat: new actions work nicely and publish to github as expected Signed-off-by: Johan Haals --- packages/backend/src/plugins/scaffolder.ts | 2 + .../test-template-v2/template.yaml | 27 +-- .../src/scaffolder/tasks/TaskWorker.ts | 2 + .../src/scaffolder/tasks/TemplateConverter.ts | 2 +- .../src/scaffolder/tasks/builtin.ts | 196 ++++++++++++++++-- .../scaffolder-backend/src/service/router.ts | 12 +- .../fields/RepoUrlPicker/RepoUrlPicker.tsx | 6 +- 7 files changed, 211 insertions(+), 36 deletions(-) diff --git a/packages/backend/src/plugins/scaffolder.ts b/packages/backend/src/plugins/scaffolder.ts index 1b97f735a0..9cab56d7f6 100644 --- a/packages/backend/src/plugins/scaffolder.ts +++ b/packages/backend/src/plugins/scaffolder.ts @@ -32,6 +32,7 @@ export default async function createPlugin({ logger, config, database, + reader, }: PluginEnvironment): Promise { const cookiecutterTemplater = new CookieCutter(); const craTemplater = new CreateReactAppTemplater(); @@ -57,5 +58,6 @@ export default async function createPlugin({ dockerClient, database, catalogClient, + urlReader: reader, }); } diff --git a/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml b/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml index 6b9e2142d9..27b744664a 100644 --- a/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml +++ b/plugins/scaffolder-backend/sample-templates/test-template-v2/template.yaml @@ -82,8 +82,8 @@ spec: type: service parameters: - - title: Fill in some parameters for the template - required: + - title: Fill in some steps + required: - name properties: name: @@ -93,7 +93,7 @@ spec: ui:autofocus: true ui:options: rows: 5 - - title: Choose destination + - title: Choose a location required: - repoUrl properties: @@ -104,7 +104,6 @@ spec: ui:options: allowedHosts: - github.com - steps: - id: fetch-base @@ -112,7 +111,6 @@ spec: action: fetch:cookiecutter parameters: url: ./template - targetDir: blob values: name: '{{ parameters.name }}' @@ -120,23 +118,26 @@ spec: name: Fetch Docs action: fetch:plain parameters: - url: https://github.com/aeothgiaetgh/aetkijhahte/tree/master/docs/template + targetPath: ./community + url: https://github.com/backstage/community/tree/main/backstage-community-sessions - id: publish name: Publish action: publish:github parameters: - allowedHosts: ['ghe.mycompany.com'] - name: '{{ parameters.name }}' - # storePath: github.com?owner=spotify&repo=name - storePath: '{{ parameters.storePath }}' + allowedHosts: ['github.com'] + description: 'This is {{ parameters.name }}' + # repoUrl: github.com?owner=spotify&repo=name + repoUrl: '{{ parameters.repoUrl }}' - id: register name: Register action: catalog:register parameters: - catalogInfoUrl: '{{ steps.publish.output.catalogInfoUrl }}' + repoContentsUrl: '{{ steps.publish.output.repoContentsUrl }}' + catalogInfoPath: '/catalog-info.yaml' + output: - catalogInfoUrl: '{{ steps.publish.output.catalogInfoUrl }}' - entityRef: '{{ steps.register.entityRef }}' + remoteUrl: '{{ steps.publish.output.remoteUrl }}' + entityRef: '{{ steps.register.output.entityRef }}' diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts index 1e038ba65e..8fd425fbdf 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TaskWorker.ts @@ -154,6 +154,7 @@ export class TaskWorker { } } + console.log('DEBUG: templateCtx =', JSON.stringify(templateCtx, 0, 2)); const output = JSON.parse( JSON.stringify(task.spec.output), (_key, value) => { @@ -168,6 +169,7 @@ export class TaskWorker { return value; }, ); + console.log('DEBUG: output =', output); await task.complete('completed', { output }); } catch (error) { diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts index f9e9abf925..dd9a4e25da 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/TemplateConverter.ts @@ -94,7 +94,7 @@ export function templateEntityToSpec( steps, output: { remoteUrl: '{{ steps.publish.output.remoteUrl }}', - catalogInfoUrl: '{{ steps.publish.output.catalogInfoUrl }}', + catalogInfoUrl: '{{ steps.register.output.catalogInfoUrl }}', entityRef: '{{ steps.register.output.entityRef }}', }, }; diff --git a/plugins/scaffolder-backend/src/scaffolder/tasks/builtin.ts b/plugins/scaffolder-backend/src/scaffolder/tasks/builtin.ts index a6c39a9044..214e7a8cef 100644 --- a/plugins/scaffolder-backend/src/scaffolder/tasks/builtin.ts +++ b/plugins/scaffolder-backend/src/scaffolder/tasks/builtin.ts @@ -20,10 +20,17 @@ import Docker from 'dockerode'; import { TemplaterBuilder, TemplaterValues } from '../stages/templater'; import { TemplateAction } from './types'; import { InputError, UrlReader } from '@backstage/backend-common'; -import { ScmIntegrations } from '@backstage/integration'; +import { + GithubCredentialsProvider, + ScmIntegrations, +} from '@backstage/integration'; import { JsonValue } from '@backstage/config'; import { CatalogApi } from '@backstage/catalog-client'; import { getEntityName } from '@backstage/catalog-model'; +import { PublisherBuilder } from '../stages'; +import { Octokit } from '@octokit/rest'; +import { Config } from '@backstage/config'; +import { initRepoAndPush } from '../stages/publish/helpers'; async function fetchContents({ urlReader, @@ -44,8 +51,16 @@ async function fetchContents({ ); } + let fetchUrlIsAbsolute = false; + try { + new URL(fetchUrl); + fetchUrlIsAbsolute = true; + } catch { + /* ignored */ + } + // We handle both file locations and url ones - if (baseUrl?.startsWith('file://')) { + if (!fetchUrlIsAbsolute && baseUrl?.startsWith('file://')) { const basePath = baseUrl.slice('file://'.length); if (isAbsolute(fetchUrl)) { throw new InputError( @@ -57,28 +72,27 @@ async function fetchContents({ } else { let readUrl; - if (baseUrl) { + if (fetchUrlIsAbsolute) { + readUrl = fetchUrl; + } else if (baseUrl) { const integration = integrations.byUrl(baseUrl); if (!integration) { throw new InputError(`No integration found for location ${baseUrl}`); } + readUrl = integration.resolveUrl({ url: fetchUrl, base: baseUrl, }); } else { - // If we don't have a baseUrl, check if our provided fetch url is absolute - try { - readUrl = new URL(fetchUrl).toString(); - } catch { - throw new InputError( - `Failed to fetch, template location could not be determined and the fetch URL is relative, ${fetchUrl}`, - ); - } - - const res = await urlReader.readTree(readUrl); - await res.dir({ targetDir: outputPath }); + throw new InputError( + `Failed to fetch, template location could not be determined and the fetch URL is relative, ${fetchUrl}`, + ); } + + const res = await urlReader.readTree(readUrl); + await fs.ensureDir(outputPath); + await res.dir({ targetDir: outputPath }); } } @@ -177,24 +191,174 @@ export function createFetchCookiecutterAction(options: { }; } +export function createPublishGithubAction(options: { + integrations: ScmIntegrations; + repoVisibility: 'private' | 'internal' | 'public'; +}): TemplateAction { + const { integrations, repoVisibility } = options; + + const credentialsProviders = new Map( + integrations.github.list().map(integration => { + const provider = GithubCredentialsProvider.create(integration.config); + return [integration.config.host, provider]; + }), + ); + + return { + id: 'publish:github', + async handler(ctx) { + const { repoUrl, description, access } = ctx.parameters; + + if (typeof repoUrl !== 'string') { + throw new Error( + `Invalid repo URL passed to publish:github, got ${typeof repoUrl}`, + ); + } + let parsed; + try { + parsed = new URL(`https://${repoUrl}`); + } catch (error) { + throw new InputError( + `Invalid repo URL passed to publish:github, got ${repoUrl}, ${error}`, + ); + } + const host = parsed.host; + const owner = parsed.searchParams.get('owner'); + if (!owner) { + throw new InputError( + `Invalid repo URL passed to publish:github, missing owner`, + ); + } + const repo = parsed.searchParams.get('repo'); + if (!repo) { + throw new InputError( + `Invalid repo URL passed to publish:github, missing repo`, + ); + } + + const credentialsProvider = credentialsProviders.get(host); + const integrationConfig = integrations.github.byHost(host); + + if (!credentialsProvider || !integrationConfig) { + throw new InputError( + `No matching integration configuration for host ${host}, please check your Integrations config`, + ); + } + + const { token } = await credentialsProvider.getCredentials({ + url: `${host}/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}`, + }); + + if (!token) { + throw new InputError( + `No token available for host: ${host}, with owner ${owner}, and repo ${repo}`, + ); + } + + const client = new Octokit({ + auth: token, + baseUrl: integrationConfig.config.apiBaseUrl, + }); + + const user = await client.users.getByUsername({ + username: owner, + }); + + const repoCreationPromise = + user.data.type === 'Organization' + ? client.repos.createInOrg({ + name: repo, + org: owner, + private: repoVisibility !== 'public', + visibility: repoVisibility, + description: description as string, + }) + : client.repos.createForAuthenticatedUser({ + name: repo, + private: repoVisibility === 'private', + description: description as string, + }); + + const { data } = await repoCreationPromise; + const accessString = access as string; + if (accessString?.startsWith(`${owner}/`)) { + const [, team] = accessString.split('/'); + await client.teams.addOrUpdateRepoPermissionsInOrg({ + org: owner, + team_slug: team, + owner, + repo, + permission: 'admin', + }); + // no need to add accessString if it's the person who own's the personal account + } else if (accessString && accessString !== owner) { + await client.repos.addCollaborator({ + owner, + repo, + username: accessString, + permission: 'admin', + }); + } + + const remoteUrl = data?.clone_url; + const repoContentsUrl = data?.html_url + ? `${data?.html_url}/blob/master` + : undefined; + + await initRepoAndPush({ + dir: ctx.workspacePath, + remoteUrl, + auth: { + username: 'x-access-token', + password: token, + }, + logger: ctx.logger, + }); + + ctx.output('remoteUrl', remoteUrl); + ctx.output('repoContentsUrl', repoContentsUrl); + }, + }; +} + export function createCatalogRegisterAction(options: { catalogClient: CatalogApi; + integrations: ScmIntegrations; }): TemplateAction { - const { catalogClient } = options; + const { catalogClient, integrations } = options; return { id: 'catalog:register', async handler(ctx) { - const { catalogInfoUrl } = ctx.parameters; + const { + repoContentsUrl, + catalogInfoPath = '/catalog-info.yaml', + } = ctx.parameters; + + let { catalogInfoUrl } = ctx.parameters; + if (!catalogInfoUrl) { + const integration = integrations.byUrl(repoContentsUrl as string); + if (!integration) { + throw new InputError('No integration found for host'); + } + + catalogInfoUrl = integration.resolveUrl({ + base: repoContentsUrl as string, + url: catalogInfoPath as string, + }); + } + ctx.logger.info(`Registering ${catalogInfoUrl} in the catalog`); const result = await catalogClient.addLocation({ type: 'url', target: catalogInfoUrl as string, }); + console.log('DEBUG: result.entities =', result.entities); if (result.entities.length >= 1) { const { kind, name, namespace } = getEntityName(result.entities[0]); ctx.output('entityRef', `${kind}:${namespace}/${name}`); + ctx.output('catalogInfoUrl', catalogInfoUrl); } }, }; diff --git a/plugins/scaffolder-backend/src/service/router.ts b/plugins/scaffolder-backend/src/service/router.ts index 71619de3bc..3cc8d01238 100644 --- a/plugins/scaffolder-backend/src/service/router.ts +++ b/plugins/scaffolder-backend/src/service/router.ts @@ -57,6 +57,7 @@ import { import { createFetchPlainAction, createFetchCookiecutterAction, + createPublishGithubAction, createCatalogRegisterAction, } from '../scaffolder/tasks/builtin'; import { ScmIntegrations } from '@backstage/integration'; @@ -145,7 +146,16 @@ export async function createRouter( templaters, }), ); - actionRegistry.register(createCatalogRegisterAction({ catalogClient })); + actionRegistry.register( + createPublishGithubAction({ + integrations, + repoVisibility: 'public', + }), + ); + + actionRegistry.register( + createCatalogRegisterAction({ catalogClient, integrations }), + ); worker.start(); diff --git a/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx b/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx index 49c4213251..90d53fce85 100644 --- a/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx +++ b/plugins/scaffolder/src/components/fields/RepoUrlPicker/RepoUrlPicker.tsx @@ -69,11 +69,7 @@ export const RepoUrlPicker: Field = ({ formData, }) => { const api = useApi(scaffolderApiRef); - // const allowedHosts = uiSchema['ui:options']?.allowedHosts as string[]; - const allowedHosts = React.useMemo( - () => ['github.com', 'gitlab.com'] as string[], - [], - ); + const allowedHosts = uiSchema['ui:options']?.allowedHosts as string[]; const { value: integrations, loading } = useAsync(async () => { return await api.getIntegrationsList({ allowedHosts });