backend-common: add top-level archive filtering to ArchiveResponse + fix files()

packages/backend-common/src/reading/tree/ArchiveResponse.test.ts

@@ -36,10 +36,51 @@ describe('ArchiveResponse', () => {
     mockFs.restore();
   });
 
+  it('should read files', async () => {
+    const stream = fs.createReadStream('/test-archive.tar.gz');
+
+    const res = new ArchiveResponse(stream, 'mock-repo/', '/tmp');
+    const files = await res.files();
+
+    expect(files).toEqual([
+      {
+        path: 'mkdocs.yml',
+        content: expect.any(Function),
+      },
+      {
+        path: 'docs/index.md',
+        content: expect.any(Function),
+      },
+    ]);
+    const contents = await Promise.all(files.map(f => f.content()));
+    expect(contents.map(c => c.toString('utf8').trim())).toEqual([
+      'site_name: Test',
+      '# Test',
+    ]);
+  });
+
+  it('should read files with filter', async () => {
+    const stream = fs.createReadStream('/test-archive.tar.gz');
+
+    const res = new ArchiveResponse(stream, 'mock-repo/', '/tmp', path =>
+      path.endsWith('.yml'),
+    );
+    const files = await res.files();
+
+    expect(files).toEqual([
+      {
+        path: 'mkdocs.yml',
+        content: expect.any(Function),
+      },
+    ]);
+    const content = await files[0].content();
+    expect(content.toString('utf8').trim()).toEqual('site_name: Test');
+  });
+
   it('should read as archive and files', async () => {
     const stream = fs.createReadStream('/test-archive.tar.gz');
 
-    const res = new ArchiveResponse(stream, 'mock-repo', '/tmp');
+    const res = new ArchiveResponse(stream, 'mock-repo/', '/tmp');
     const buffer = await res.archive();
 
     await expect(res.archive()).rejects.toThrow(
@@ -51,11 +92,11 @@ describe('ArchiveResponse', () => {
 
     expect(files).toEqual([
       {
-        path: './mkdocs.yml',
+        path: 'mkdocs.yml',
         content: expect.any(Function),
       },
       {
-        path: './docs/index.md',
+        path: 'docs/index.md',
         content: expect.any(Function),
       },
     ]);
@@ -83,11 +124,29 @@ describe('ArchiveResponse', () => {
   it('should extract archive into directory with a subpath', async () => {
     const stream = fs.createReadStream('/test-archive.tar.gz');
 
-    const res = new ArchiveResponse(stream, 'mock-repo/docs', '/tmp');
+    const res = new ArchiveResponse(stream, 'mock-repo/docs/', '/tmp');
     const dir = await res.dir();
 
+    expect(dir).toMatch(/^\/tmp\/.*$/);
     await expect(
       fs.readFile(resolvePath(dir, 'index.md'), 'utf8'),
     ).resolves.toBe('# Test\n');
   });
+
+  it('should extract archive into directory with a subpath and filter', async () => {
+    const stream = fs.createReadStream('/test-archive.tar.gz');
+
+    const res = new ArchiveResponse(stream, 'mock-repo/', '/tmp', path =>
+      path.endsWith('.yml'),
+    );
+    const dir = await res.dir({ targetDir: '/tmp' });
+
+    expect(dir).toBe('/tmp');
+    await expect(fs.pathExists(resolvePath(dir, 'mkdocs.yml'))).resolves.toBe(
+      true,
+    );
+    await expect(
+      fs.pathExists(resolvePath(dir, 'docs/index.md')),
+    ).resolves.toBe(false);
+  });
 });

packages/backend-common/src/reading/tree/ArchiveResponse.ts

@@ -21,7 +21,7 @@ import fs from 'fs-extra';
 import { Readable, pipeline as pipelineCb } from 'stream';
 import { promisify } from 'util';
 import concatStream from 'concat-stream';
-import { ReadTreeResponse, File } from '../types';
+import { ReadTreeResponse, File, ReadTreeResponseDirOptions } from '../types';
 
 // Tar types for `Parse` is not a proper constructor, but it should be
 const TarParseStream = (Parse as unknown) as { new (): ParseStream };
@@ -38,7 +38,17 @@ export class ArchiveResponse implements ReadTreeResponse {
     private readonly stream: Readable,
     private readonly subPath: string,
     private readonly workDir: string = os.tmpdir(),
-  ) {}
+    private readonly filter?: (path: string) => boolean,
+  ) {
+    if (subPath) {
+      if (!subPath.endsWith('/')) {
+        throw new TypeError('ArchiveResponse subPath must end with a /');
+      }
+      if (subPath.startsWith('/')) {
+        throw new TypeError('ArchiveResponse subPath must not start with a /');
+      }
+    }
+  }
 
   // Make sure the input stream is only read once
   private onlyOnce() {
@@ -60,14 +70,28 @@ export class ArchiveResponse implements ReadTreeResponse {
         return;
       }
 
-      const contentPromise = new Promise<Buffer>(async resolve => {
+      if (this.subPath) {
+        if (!entry.path.startsWith(this.subPath)) {
+          entry.resume();
+          return;
+        }
+      }
+
+      const path = this.subPath
+        ? entry.path.replace(this.subPath, '')
+        : entry.path;
+      if (this.filter) {
+        if (!this.filter(path)) {
+          entry.resume();
+          return;
+        }
+      }
+
+      const content = new Promise<Buffer>(async resolve => {
         await pipeline(entry, concatStream(resolve));
       });
 
-      files.push({
-        path: entry.path,
-        content: () => contentPromise,
-      });
+      files.push({ path, content: () => content });
 
       entry.resume();
     });
@@ -93,7 +117,7 @@ export class ArchiveResponse implements ReadTreeResponse {
     try {
       return await new Promise(async resolve => {
         await pipeline(
-          tar.create({ cwd: tmpDir }, ['.']),
+          tar.create({ cwd: tmpDir }, ['']),
           concatStream(resolve),
         );
       });
@@ -102,20 +126,30 @@ export class ArchiveResponse implements ReadTreeResponse {
     }
   }
 
-  async dir(outDir?: string): Promise<string> {
+  async dir(options?: ReadTreeResponseDirOptions): Promise<string> {
     this.onlyOnce();
 
     const dir =
-      outDir ?? (await fs.mkdtemp(path.join(this.workDir, 'backstage-')));
+      options?.targetDir ??
+      (await fs.mkdtemp(path.join(this.workDir, 'backstage-')));
 
-    const strip = this.subPath ? this.subPath.split('/').length : 0;
+    const strip = this.subPath ? this.subPath.split('/').length - 1 : 0;
 
     await pipeline(
       this.stream,
       tar.extract({
         strip,
         cwd: dir,
-        filter: path => path.startsWith(this.subPath),
+        filter: path => {
+          if (this.subPath && !path.startsWith(this.subPath)) {
+            return false;
+          }
+          if (this.filter) {
+            const innerPath = path.split('/').slice(strip).join('/');
+            return this.filter(innerPath);
+          }
+          return true;
+        },
       }),
     );
 

packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts

@@ -25,6 +25,8 @@ type FromArchiveOptions = {
   stream: Readable;
   // If set, root of the tree will be set to the given path. Should not have a trailing `/`.
   path?: string;
+  // Filter passed on from the ReadTreeOptions
+  filter?: (path: string) => boolean;
 };
 
 export class ReadTreeResponseFactory {
@@ -42,6 +44,7 @@ export class ReadTreeResponseFactory {
       options.stream,
       options.path ?? '',
       this.workDir,
+      options.filter,
     );
   }
 }