diff --git a/.changeset/fuzzy-wombats-search.md b/.changeset/fuzzy-wombats-search.md new file mode 100644 index 0000000000..ca1754acc9 --- /dev/null +++ b/.changeset/fuzzy-wombats-search.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder-backend': minor +--- + +Added `fetch:plain:file` action to fetch a single file diff --git a/plugins/scaffolder-backend/api-report.md b/plugins/scaffolder-backend/api-report.md index f0eb714ab3..a569ff8232 100644 --- a/plugins/scaffolder-backend/api-report.md +++ b/plugins/scaffolder-backend/api-report.md @@ -105,6 +105,15 @@ export function createFetchPlainAction(options: { targetPath?: string | undefined; }>; +// @public +export function createFetchPlainFileAction(options: { + reader: UrlReader; + integrations: ScmIntegrations; +}): TemplateAction_2<{ + url: string; + targetPath: string; +}>; + // @public export function createFetchTemplateAction(options: { reader: UrlReader; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts index d4c8de3604..747b5cdaed 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts @@ -31,7 +31,11 @@ import { import { TemplateFilter, TemplateGlobal } from '../../../lib'; import { createDebugLogAction, createWaitAction } from './debug'; -import { createFetchPlainAction, createFetchTemplateAction } from './fetch'; +import { + createFetchPlainAction, + createFetchPlainFileAction, + createFetchTemplateAction, +} from './fetch'; import { createFilesystemDeleteAction, createFilesystemRenameAction, @@ -112,6 +116,10 @@ export const createBuiltinActions = ( reader, integrations, }), + createFetchPlainFileAction({ + reader, + integrations, + }), createFetchTemplateAction({ integrations, reader, diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.test.ts index 86d5f17803..2740d16efb 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.test.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.test.ts @@ -21,7 +21,7 @@ import { resolve as resolvePath } from 'path'; import { UrlReader } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; import { ScmIntegrations } from '@backstage/integration'; -import { fetchContents } from './helpers'; +import { fetchContents, fetchFile } from './helpers'; import os from 'os'; describe('fetchContent helper', () => { @@ -37,9 +37,10 @@ describe('fetchContent helper', () => { }), ); + const readUrl = jest.fn(); const readTree = jest.fn(); const reader: UrlReader = { - readUrl: jest.fn(), + readUrl, readTree, search: jest.fn(), }; @@ -50,7 +51,7 @@ describe('fetchContent helper', () => { outputPath: os.tmpdir(), }; - it('should reject absolute file locations', async () => { + it('fetch contents should reject absolute file locations', async () => { await expect( fetchContents({ ...options, @@ -62,7 +63,7 @@ describe('fetchContent helper', () => { ); }); - it('should reject relative file locations that exit the baseUrl', async () => { + it('fetch contents should reject relative file locations that exit the baseUrl', async () => { await expect( fetchContents({ ...options, @@ -74,7 +75,7 @@ describe('fetchContent helper', () => { ); }); - it('should copy file to outputpath', async () => { + it('fetch contents should copy file to outputpath', async () => { await fetchContents({ ...options, baseUrl: 'file:///some/path', @@ -84,7 +85,7 @@ describe('fetchContent helper', () => { expect(fs.copy).toHaveBeenCalledWith(resolvePath('/some/foo'), 'somepath'); }); - it('should reject if no integration matches location', async () => { + it('fetch contents should reject if no integration matches location', async () => { await expect( fetchContents({ ...options, @@ -95,7 +96,7 @@ describe('fetchContent helper', () => { ); }); - it('should reject if fetch url is relative and no base url is specified', async () => { + it('fetch contents should reject if fetch url is relative and no base url is specified', async () => { await expect( fetchContents({ ...options, @@ -106,7 +107,7 @@ describe('fetchContent helper', () => { ); }); - it('should fetch url contents', async () => { + it('fetch contents should fetch url contents', async () => { const dirFunction = jest.fn(); readTree.mockResolvedValue({ dir: dirFunction, @@ -116,7 +117,92 @@ describe('fetchContent helper', () => { outputPath: 'foo', fetchUrl: 'https://github.com/backstage/foo', }); - expect(fs.ensureDir).toHaveBeenCalled(); + expect(fs.ensureDir).toHaveBeenCalledWith('foo'); expect(dirFunction).toHaveBeenCalledWith({ targetDir: 'foo' }); }); + + it('fetch file should reject absolute file locations', async () => { + await expect( + fetchFile({ + ...options, + baseUrl: 'file:///some/path', + fetchUrl: '/etc/passwd', + }), + ).rejects.toThrow( + 'Relative path is not allowed to refer to a directory outside its parent', + ); + }); + + it('fetch file should reject relative file locations that exit the baseUrl', async () => { + await expect( + fetchFile({ + ...options, + baseUrl: 'file:///some/path', + fetchUrl: '../test', + }), + ).rejects.toThrow( + 'Relative path is not allowed to refer to a directory outside its parent', + ); + }); + + it('fetch file should copy file to outputpath', async () => { + await fetchFile({ + ...options, + baseUrl: 'file:///some/path', + fetchUrl: 'foo', + outputPath: 'somepath', + }); + expect(fs.copyFile).toHaveBeenCalledWith( + resolvePath('/some/foo'), + 'somepath', + ); + }); + + it('fetch file should reject if no integration matches location', async () => { + await expect( + fetchFile({ + ...options, + baseUrl: 'http://example.com/some/folder', + }), + ).rejects.toThrow( + 'No integration found for location http://example.com/some/folder', + ); + }); + + it('fetch file should reject if fetch url is relative and no base url is specified', async () => { + await expect( + fetchFile({ + ...options, + fetchUrl: 'foo', + }), + ).rejects.toThrow( + 'Failed to fetch, template location could not be determined and the fetch URL is relative, foo', + ); + }); + + it('fetch file should fetch content from url', async () => { + readUrl.mockResolvedValue({ + buffer: () => Buffer.from('test', 'utf8'), + }); + await fetchFile({ + ...options, + outputPath: 'foo', + fetchUrl: 'https://github.com/backstage/foo', + }); + expect(fs.ensureDir).toHaveBeenCalledWith('.'); + expect(fs.outputFile).toHaveBeenCalledWith('foo', 'test'); + }); + + it('fetch file should fetch content from url into directory', async () => { + readUrl.mockResolvedValue({ + buffer: () => Buffer.from('test', 'utf8'), + }); + await fetchFile({ + ...options, + outputPath: 'mydir/foo', + fetchUrl: 'https://github.com/backstage/foo', + }); + expect(fs.ensureDir).toHaveBeenCalledWith('mydir'); + expect(fs.outputFile).toHaveBeenCalledWith('mydir/foo', 'test'); + }); }); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.ts index 4b82595cfd..ac44f0a8a5 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.ts @@ -35,6 +35,55 @@ export async function fetchContents(options: { }) { const { reader, integrations, baseUrl, fetchUrl = '.', outputPath } = options; + const fetchUrlIsAbsolute = isFetchUrlAbsolute(fetchUrl); + + // We handle both file locations and url ones + if (!fetchUrlIsAbsolute && baseUrl?.startsWith('file://')) { + const basePath = baseUrl.slice('file://'.length); + const srcDir = resolveSafeChildPath(path.dirname(basePath), fetchUrl); + await fs.copy(srcDir, outputPath); + } else { + const readUrl = getReadUrl(fetchUrl, baseUrl, integrations); + + const res = await reader.readTree(readUrl); + await fs.ensureDir(outputPath); + await res.dir({ targetDir: outputPath }); + } +} + +/** + * A helper function that reads the content of a single file from the given URL. + * Can be used in your own actions, and also used behind fetch:plain:file + * + * @public + */ +export async function fetchFile(options: { + reader: UrlReader; + integrations: ScmIntegrations; + baseUrl?: string; + fetchUrl?: string; + outputPath: string; +}) { + const { reader, integrations, baseUrl, fetchUrl = '.', outputPath } = options; + + const fetchUrlIsAbsolute = isFetchUrlAbsolute(fetchUrl); + + // We handle both file locations and url ones + if (!fetchUrlIsAbsolute && baseUrl?.startsWith('file://')) { + const basePath = baseUrl.slice('file://'.length); + const src = resolveSafeChildPath(path.dirname(basePath), fetchUrl); + await fs.copyFile(src, outputPath); + } else { + const readUrl = getReadUrl(fetchUrl, baseUrl, integrations); + + const res = await reader.readUrl(readUrl); + await fs.ensureDir(path.dirname(outputPath)); + const buffer = await res.buffer(); + await fs.outputFile(outputPath, buffer.toString()); + } +} + +function isFetchUrlAbsolute(fetchUrl: string) { let fetchUrlIsAbsolute = false; try { // eslint-disable-next-line no-new @@ -43,35 +92,28 @@ export async function fetchContents(options: { } catch { /* ignored */ } + return fetchUrlIsAbsolute; +} - // We handle both file locations and url ones - if (!fetchUrlIsAbsolute && baseUrl?.startsWith('file://')) { - const basePath = baseUrl.slice('file://'.length); - const srcDir = resolveSafeChildPath(path.dirname(basePath), fetchUrl); - await fs.copy(srcDir, outputPath); - } else { - let readUrl; - - if (fetchUrlIsAbsolute) { - readUrl = fetchUrl; - } else if (baseUrl) { - const integration = integrations.byUrl(baseUrl); - if (!integration) { - throw new InputError(`No integration found for location ${baseUrl}`); - } - - readUrl = integration.resolveUrl({ - url: fetchUrl, - base: baseUrl, - }); - } else { - throw new InputError( - `Failed to fetch, template location could not be determined and the fetch URL is relative, ${fetchUrl}`, - ); +function getReadUrl( + fetchUrl: string, + baseUrl: string | undefined, + integrations: ScmIntegrations, +) { + if (isFetchUrlAbsolute(fetchUrl)) { + return fetchUrl; + } else if (baseUrl) { + const integration = integrations.byUrl(baseUrl); + if (!integration) { + throw new InputError(`No integration found for location ${baseUrl}`); } - const res = await reader.readTree(readUrl); - await fs.ensureDir(outputPath); - await res.dir({ targetDir: outputPath }); + return integration.resolveUrl({ + url: fetchUrl, + base: baseUrl, + }); } + throw new InputError( + `Failed to fetch, template location could not be determined and the fetch URL is relative, ${fetchUrl}`, + ); } diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts index c9a16c274e..14a40f53c3 100644 --- a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts @@ -15,5 +15,6 @@ */ export { createFetchPlainAction } from './plain'; +export { createFetchPlainFileAction } from './plainFile'; export { createFetchTemplateAction } from './template'; export { fetchContents } from './helpers'; diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plainFile.test.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plainFile.test.ts new file mode 100644 index 0000000000..5d9e99fc7d --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plainFile.test.ts @@ -0,0 +1,85 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +jest.mock('./helpers'); + +import os from 'os'; +import { resolve as resolvePath } from 'path'; +import { getVoidLogger, UrlReader } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; +import { ScmIntegrations } from '@backstage/integration'; +import { createFetchPlainFileAction } from './plainFile'; +import { PassThrough } from 'stream'; +import { fetchFile } from './helpers'; + +describe('fetch:plain:file', () => { + const integrations = ScmIntegrations.fromConfig( + new ConfigReader({ + integrations: { + github: [{ host: 'github.com', token: 'token' }], + }, + }), + ); + const reader: UrlReader = { + readUrl: jest.fn(), + readTree: jest.fn(), + search: jest.fn(), + }; + + beforeEach(() => { + jest.resetAllMocks(); + }); + + const action = createFetchPlainFileAction({ integrations, reader }); + const mockContext = { + workspacePath: os.tmpdir(), + logger: getVoidLogger(), + logStream: new PassThrough(), + output: jest.fn(), + createTemporaryDirectory: jest.fn(), + }; + + it('should disallow a target path outside working directory', async () => { + await expect( + action.handler({ + ...mockContext, + input: { + url: 'https://github.com/backstage/community/tree/main/backstage-community-sessions/assets/Backstage%20Community%20Sessions.png', + targetPath: '/foobar', + }, + }), + ).rejects.toThrow( + /Relative path is not allowed to refer to a directory outside its parent/, + ); + }); + + it('should fetch plain', async () => { + await action.handler({ + ...mockContext, + input: { + url: 'https://github.com/backstage/community/tree/main/backstage-community-sessions/assets/Backstage%20Community%20Sessions.png', + targetPath: 'lol', + }, + }); + expect(fetchFile).toHaveBeenCalledWith( + expect.objectContaining({ + outputPath: resolvePath(mockContext.workspacePath, 'lol'), + fetchUrl: + 'https://github.com/backstage/community/tree/main/backstage-community-sessions/assets/Backstage%20Community%20Sessions.png', + }), + ); + }); +}); diff --git a/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plainFile.ts b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plainFile.ts new file mode 100644 index 0000000000..7b33ad93b4 --- /dev/null +++ b/plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plainFile.ts @@ -0,0 +1,75 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { UrlReader, resolveSafeChildPath } from '@backstage/backend-common'; +import { ScmIntegrations } from '@backstage/integration'; +import { fetchFile } from './helpers'; +import { createTemplateAction } from '@backstage/plugin-scaffolder-node'; + +/** + * Downloads content and places it in the workspace, or optionally + * in a subdirectory specified by the 'targetPath' input option. + * @public + */ +export function createFetchPlainFileAction(options: { + reader: UrlReader; + integrations: ScmIntegrations; +}) { + const { reader, integrations } = options; + + return createTemplateAction<{ url: string; targetPath: string }>({ + id: 'fetch:plain:file', + description: 'Downloads single file and places it in the workspace.', + schema: { + input: { + type: 'object', + required: ['url', 'targetPath'], + properties: { + url: { + title: 'Fetch URL', + description: + 'Relative path or absolute URL pointing to the single file to fetch.', + type: 'string', + }, + targetPath: { + title: 'Target Path', + description: + 'Target path within the working directory to download the file as.', + type: 'string', + }, + }, + }, + }, + supportsDryRun: true, + async handler(ctx) { + ctx.logger.info('Fetching plain content from remote URL'); + + // Finally move the template result into the task workspace + const outputPath = resolveSafeChildPath( + ctx.workspacePath, + ctx.input.targetPath, + ); + + await fetchFile({ + reader, + integrations, + baseUrl: ctx.templateInfo?.baseUrl, + fetchUrl: ctx.input.url, + outputPath, + }); + }, + }); +}