Added fetch:plain:file action to fetch a single file

Fixes #15388

Signed-off-by: Arthur Gavlyukovskiy <agavlyukovskiy@gmail.com>

.changeset/fuzzy-wombats-search.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-scaffolder-backend': minor
+---
+
+Added `fetch:plain:file` action to fetch a single file

plugins/scaffolder-backend/api-report.md

@@ -105,6 +105,15 @@ export function createFetchPlainAction(options: {
   targetPath?: string | undefined;
 }>;
 
+// @public
+export function createFetchPlainFileAction(options: {
+  reader: UrlReader;
+  integrations: ScmIntegrations;
+}): TemplateAction_2<{
+  url: string;
+  targetPath: string;
+}>;
+
 // @public
 export function createFetchTemplateAction(options: {
   reader: UrlReader;

plugins/scaffolder-backend/src/scaffolder/actions/builtin/createBuiltinActions.ts

@@ -31,7 +31,11 @@ import {
 
 import { TemplateFilter, TemplateGlobal } from '../../../lib';
 import { createDebugLogAction, createWaitAction } from './debug';
-import { createFetchPlainAction, createFetchTemplateAction } from './fetch';
+import {
+  createFetchPlainAction,
+  createFetchPlainFileAction,
+  createFetchTemplateAction,
+} from './fetch';
 import {
   createFilesystemDeleteAction,
   createFilesystemRenameAction,
@@ -112,6 +116,10 @@ export const createBuiltinActions = (
       reader,
       integrations,
     }),
+    createFetchPlainFileAction({
+      reader,
+      integrations,
+    }),
     createFetchTemplateAction({
       integrations,
       reader,

plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.test.ts

@@ -21,7 +21,7 @@ import { resolve as resolvePath } from 'path';
 import { UrlReader } from '@backstage/backend-common';
 import { ConfigReader } from '@backstage/config';
 import { ScmIntegrations } from '@backstage/integration';
-import { fetchContents } from './helpers';
+import { fetchContents, fetchFile } from './helpers';
 import os from 'os';
 
 describe('fetchContent helper', () => {
@@ -37,9 +37,10 @@ describe('fetchContent helper', () => {
     }),
   );
 
+  const readUrl = jest.fn();
   const readTree = jest.fn();
   const reader: UrlReader = {
-    readUrl: jest.fn(),
+    readUrl,
     readTree,
     search: jest.fn(),
   };
@@ -50,7 +51,7 @@ describe('fetchContent helper', () => {
     outputPath: os.tmpdir(),
   };
 
-  it('should reject absolute file locations', async () => {
+  it('fetch contents should reject absolute file locations', async () => {
     await expect(
       fetchContents({
         ...options,
@@ -62,7 +63,7 @@ describe('fetchContent helper', () => {
     );
   });
 
-  it('should reject relative file locations that exit the baseUrl', async () => {
+  it('fetch contents should reject relative file locations that exit the baseUrl', async () => {
     await expect(
       fetchContents({
         ...options,
@@ -74,7 +75,7 @@ describe('fetchContent helper', () => {
     );
   });
 
-  it('should copy file to outputpath', async () => {
+  it('fetch contents should copy file to outputpath', async () => {
     await fetchContents({
       ...options,
       baseUrl: 'file:///some/path',
@@ -84,7 +85,7 @@ describe('fetchContent helper', () => {
     expect(fs.copy).toHaveBeenCalledWith(resolvePath('/some/foo'), 'somepath');
   });
 
-  it('should reject if no integration matches location', async () => {
+  it('fetch contents should reject if no integration matches location', async () => {
     await expect(
       fetchContents({
         ...options,
@@ -95,7 +96,7 @@ describe('fetchContent helper', () => {
     );
   });
 
-  it('should reject if fetch url is relative and no base url is specified', async () => {
+  it('fetch contents should reject if fetch url is relative and no base url is specified', async () => {
     await expect(
       fetchContents({
         ...options,
@@ -106,7 +107,7 @@ describe('fetchContent helper', () => {
     );
   });
 
-  it('should fetch url contents', async () => {
+  it('fetch contents should fetch url contents', async () => {
     const dirFunction = jest.fn();
     readTree.mockResolvedValue({
       dir: dirFunction,
@@ -116,7 +117,92 @@ describe('fetchContent helper', () => {
       outputPath: 'foo',
       fetchUrl: 'https://github.com/backstage/foo',
     });
-    expect(fs.ensureDir).toHaveBeenCalled();
+    expect(fs.ensureDir).toHaveBeenCalledWith('foo');
     expect(dirFunction).toHaveBeenCalledWith({ targetDir: 'foo' });
   });
+
+  it('fetch file should reject absolute file locations', async () => {
+    await expect(
+      fetchFile({
+        ...options,
+        baseUrl: 'file:///some/path',
+        fetchUrl: '/etc/passwd',
+      }),
+    ).rejects.toThrow(
+      'Relative path is not allowed to refer to a directory outside its parent',
+    );
+  });
+
+  it('fetch file should reject relative file locations that exit the baseUrl', async () => {
+    await expect(
+      fetchFile({
+        ...options,
+        baseUrl: 'file:///some/path',
+        fetchUrl: '../test',
+      }),
+    ).rejects.toThrow(
+      'Relative path is not allowed to refer to a directory outside its parent',
+    );
+  });
+
+  it('fetch file should copy file to outputpath', async () => {
+    await fetchFile({
+      ...options,
+      baseUrl: 'file:///some/path',
+      fetchUrl: 'foo',
+      outputPath: 'somepath',
+    });
+    expect(fs.copyFile).toHaveBeenCalledWith(
+      resolvePath('/some/foo'),
+      'somepath',
+    );
+  });
+
+  it('fetch file should reject if no integration matches location', async () => {
+    await expect(
+      fetchFile({
+        ...options,
+        baseUrl: 'http://example.com/some/folder',
+      }),
+    ).rejects.toThrow(
+      'No integration found for location http://example.com/some/folder',
+    );
+  });
+
+  it('fetch file should reject if fetch url is relative and no base url is specified', async () => {
+    await expect(
+      fetchFile({
+        ...options,
+        fetchUrl: 'foo',
+      }),
+    ).rejects.toThrow(
+      'Failed to fetch, template location could not be determined and the fetch URL is relative, foo',
+    );
+  });
+
+  it('fetch file should fetch content from url', async () => {
+    readUrl.mockResolvedValue({
+      buffer: () => Buffer.from('test', 'utf8'),
+    });
+    await fetchFile({
+      ...options,
+      outputPath: 'foo',
+      fetchUrl: 'https://github.com/backstage/foo',
+    });
+    expect(fs.ensureDir).toHaveBeenCalledWith('.');
+    expect(fs.outputFile).toHaveBeenCalledWith('foo', 'test');
+  });
+
+  it('fetch file should fetch content from url into directory', async () => {
+    readUrl.mockResolvedValue({
+      buffer: () => Buffer.from('test', 'utf8'),
+    });
+    await fetchFile({
+      ...options,
+      outputPath: 'mydir/foo',
+      fetchUrl: 'https://github.com/backstage/foo',
+    });
+    expect(fs.ensureDir).toHaveBeenCalledWith('mydir');
+    expect(fs.outputFile).toHaveBeenCalledWith('mydir/foo', 'test');
+  });
 });

plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/helpers.ts

@@ -35,6 +35,55 @@ export async function fetchContents(options: {
 }) {
   const { reader, integrations, baseUrl, fetchUrl = '.', outputPath } = options;
 
+  const fetchUrlIsAbsolute = isFetchUrlAbsolute(fetchUrl);
+
+  // We handle both file locations and url ones
+  if (!fetchUrlIsAbsolute && baseUrl?.startsWith('file://')) {
+    const basePath = baseUrl.slice('file://'.length);
+    const srcDir = resolveSafeChildPath(path.dirname(basePath), fetchUrl);
+    await fs.copy(srcDir, outputPath);
+  } else {
+    const readUrl = getReadUrl(fetchUrl, baseUrl, integrations);
+
+    const res = await reader.readTree(readUrl);
+    await fs.ensureDir(outputPath);
+    await res.dir({ targetDir: outputPath });
+  }
+}
+
+/**
+ * A helper function that reads the content of a single file from the given URL.
+ * Can be used in your own actions, and also used behind fetch:plain:file
+ *
+ * @public
+ */
+export async function fetchFile(options: {
+  reader: UrlReader;
+  integrations: ScmIntegrations;
+  baseUrl?: string;
+  fetchUrl?: string;
+  outputPath: string;
+}) {
+  const { reader, integrations, baseUrl, fetchUrl = '.', outputPath } = options;
+
+  const fetchUrlIsAbsolute = isFetchUrlAbsolute(fetchUrl);
+
+  // We handle both file locations and url ones
+  if (!fetchUrlIsAbsolute && baseUrl?.startsWith('file://')) {
+    const basePath = baseUrl.slice('file://'.length);
+    const src = resolveSafeChildPath(path.dirname(basePath), fetchUrl);
+    await fs.copyFile(src, outputPath);
+  } else {
+    const readUrl = getReadUrl(fetchUrl, baseUrl, integrations);
+
+    const res = await reader.readUrl(readUrl);
+    await fs.ensureDir(path.dirname(outputPath));
+    const buffer = await res.buffer();
+    await fs.outputFile(outputPath, buffer.toString());
+  }
+}
+
+function isFetchUrlAbsolute(fetchUrl: string) {
   let fetchUrlIsAbsolute = false;
   try {
     // eslint-disable-next-line no-new
@@ -43,35 +92,28 @@ export async function fetchContents(options: {
   } catch {
     /* ignored */
   }
+  return fetchUrlIsAbsolute;
+}
 
-  // We handle both file locations and url ones
-  if (!fetchUrlIsAbsolute && baseUrl?.startsWith('file://')) {
-    const basePath = baseUrl.slice('file://'.length);
-    const srcDir = resolveSafeChildPath(path.dirname(basePath), fetchUrl);
-    await fs.copy(srcDir, outputPath);
-  } else {
-    let readUrl;
-
-    if (fetchUrlIsAbsolute) {
-      readUrl = fetchUrl;
-    } else if (baseUrl) {
-      const integration = integrations.byUrl(baseUrl);
-      if (!integration) {
-        throw new InputError(`No integration found for location ${baseUrl}`);
-      }
-
-      readUrl = integration.resolveUrl({
-        url: fetchUrl,
-        base: baseUrl,
-      });
-    } else {
-      throw new InputError(
-        `Failed to fetch, template location could not be determined and the fetch URL is relative, ${fetchUrl}`,
-      );
+function getReadUrl(
+  fetchUrl: string,
+  baseUrl: string | undefined,
+  integrations: ScmIntegrations,
+) {
+  if (isFetchUrlAbsolute(fetchUrl)) {
+    return fetchUrl;
+  } else if (baseUrl) {
+    const integration = integrations.byUrl(baseUrl);
+    if (!integration) {
+      throw new InputError(`No integration found for location ${baseUrl}`);
     }
 
-    const res = await reader.readTree(readUrl);
-    await fs.ensureDir(outputPath);
-    await res.dir({ targetDir: outputPath });
+    return integration.resolveUrl({
+      url: fetchUrl,
+      base: baseUrl,
+    });
   }
+  throw new InputError(
+    `Failed to fetch, template location could not be determined and the fetch URL is relative, ${fetchUrl}`,
+  );
 }

plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/index.ts

@@ -15,5 +15,6 @@
  */
 
 export { createFetchPlainAction } from './plain';
+export { createFetchPlainFileAction } from './plainFile';
 export { createFetchTemplateAction } from './template';
 export { fetchContents } from './helpers';

plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plainFile.test.ts

@@ -0,0 +1,85 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+jest.mock('./helpers');
+
+import os from 'os';
+import { resolve as resolvePath } from 'path';
+import { getVoidLogger, UrlReader } from '@backstage/backend-common';
+import { ConfigReader } from '@backstage/config';
+import { ScmIntegrations } from '@backstage/integration';
+import { createFetchPlainFileAction } from './plainFile';
+import { PassThrough } from 'stream';
+import { fetchFile } from './helpers';
+
+describe('fetch:plain:file', () => {
+  const integrations = ScmIntegrations.fromConfig(
+    new ConfigReader({
+      integrations: {
+        github: [{ host: 'github.com', token: 'token' }],
+      },
+    }),
+  );
+  const reader: UrlReader = {
+    readUrl: jest.fn(),
+    readTree: jest.fn(),
+    search: jest.fn(),
+  };
+
+  beforeEach(() => {
+    jest.resetAllMocks();
+  });
+
+  const action = createFetchPlainFileAction({ integrations, reader });
+  const mockContext = {
+    workspacePath: os.tmpdir(),
+    logger: getVoidLogger(),
+    logStream: new PassThrough(),
+    output: jest.fn(),
+    createTemporaryDirectory: jest.fn(),
+  };
+
+  it('should disallow a target path outside working directory', async () => {
+    await expect(
+      action.handler({
+        ...mockContext,
+        input: {
+          url: 'https://github.com/backstage/community/tree/main/backstage-community-sessions/assets/Backstage%20Community%20Sessions.png',
+          targetPath: '/foobar',
+        },
+      }),
+    ).rejects.toThrow(
+      /Relative path is not allowed to refer to a directory outside its parent/,
+    );
+  });
+
+  it('should fetch plain', async () => {
+    await action.handler({
+      ...mockContext,
+      input: {
+        url: 'https://github.com/backstage/community/tree/main/backstage-community-sessions/assets/Backstage%20Community%20Sessions.png',
+        targetPath: 'lol',
+      },
+    });
+    expect(fetchFile).toHaveBeenCalledWith(
+      expect.objectContaining({
+        outputPath: resolvePath(mockContext.workspacePath, 'lol'),
+        fetchUrl:
+          'https://github.com/backstage/community/tree/main/backstage-community-sessions/assets/Backstage%20Community%20Sessions.png',
+      }),
+    );
+  });
+});

plugins/scaffolder-backend/src/scaffolder/actions/builtin/fetch/plainFile.ts

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { UrlReader, resolveSafeChildPath } from '@backstage/backend-common';
+import { ScmIntegrations } from '@backstage/integration';
+import { fetchFile } from './helpers';
+import { createTemplateAction } from '@backstage/plugin-scaffolder-node';
+
+/**
+ * Downloads content and places it in the workspace, or optionally
+ * in a subdirectory specified by the 'targetPath' input option.
+ * @public
+ */
+export function createFetchPlainFileAction(options: {
+  reader: UrlReader;
+  integrations: ScmIntegrations;
+}) {
+  const { reader, integrations } = options;
+
+  return createTemplateAction<{ url: string; targetPath: string }>({
+    id: 'fetch:plain:file',
+    description: 'Downloads single file and places it in the workspace.',
+    schema: {
+      input: {
+        type: 'object',
+        required: ['url', 'targetPath'],
+        properties: {
+          url: {
+            title: 'Fetch URL',
+            description:
+              'Relative path or absolute URL pointing to the single file to fetch.',
+            type: 'string',
+          },
+          targetPath: {
+            title: 'Target Path',
+            description:
+              'Target path within the working directory to download the file as.',
+            type: 'string',
+          },
+        },
+      },
+    },
+    supportsDryRun: true,
+    async handler(ctx) {
+      ctx.logger.info('Fetching plain content from remote URL');
+
+      // Finally move the template result into the task workspace
+      const outputPath = resolveSafeChildPath(
+        ctx.workspacePath,
+        ctx.input.targetPath,
+      );
+
+      await fetchFile({
+        reader,
+        integrations,
+        baseUrl: ctx.templateInfo?.baseUrl,
+        fetchUrl: ctx.input.url,
+        outputPath,
+      });
+    },
+  });
+}