Introduce ZipArchiveResponse

2020-11-16 16:08:05 +01:00
parent c6cf38c6a4
commit 9367dde00d
5 changed files with 512 additions and 5 deletions
@@ -36,6 +36,7 @@
    "@backstage/test-utils": "^0.1.3",
    "@types/cors": "^2.8.6",
    "@types/express": "^4.17.6",
+    "archiver": "^5.0.2",
    "compression": "^1.7.4",
    "concat-stream": "^2.0.0",
    "cors": "^2.8.5",
@@ -55,6 +56,7 @@
    "selfsigned": "^1.10.7",
    "stoppable": "^1.1.0",
    "tar": "^6.0.5",
+    "unzipper": "^0.10.11",
    "winston": "^3.2.1"
  },
  "peerDependencies": {
@@ -67,6 +69,7 @@
  },
  "devDependencies": {
    "@backstage/cli": "^0.3.0",
+    "@types/archiver": "^3.1.1",
    "@types/compression": "^1.7.0",
    "@types/concat-stream": "^1.6.0",
    "@types/fs-extra": "^9.0.3",
@@ -78,6 +81,7 @@
    "@types/stoppable": "^1.1.0",
    "@types/supertest": "^2.0.8",
    "@types/tar": "^4.0.3",
+    "@types/unzipper": "^0.10.3",
    "@types/webpack-env": "^1.15.2",
    "@types/yaml": "^1.9.7",
    "get-port": "^5.1.1",
@@ -0,0 +1,151 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs-extra';
+import mockFs from 'mock-fs';
+import { resolve as resolvePath } from 'path';
+import { ZipArchiveResponse } from './ZipArchiveResponse';
+
+const archiveData = fs.readFileSync(
+  resolvePath(__filename, '../../__fixtures__/repo.zip'),
+);
+
+describe('ZipArchiveResponse', () => {
+  beforeEach(() => {
+    mockFs({
+      '/test-archive.zip': archiveData,
+      '/tmp': mockFs.directory(),
+    });
+  });
+
+  afterEach(() => {
+    mockFs.restore();
+  });
+
+  it('should read files', async () => {
+    const stream = fs.createReadStream('/test-archive.zip');
+
+    const res = new ZipArchiveResponse(stream, 'mock-repo/', '/tmp');
+    const files = await res.files();
+
+    expect(files).toEqual([
+      {
+        path: 'docs/index.md',
+        content: expect.any(Function),
+      },
+      {
+        path: 'mkdocs.yml',
+        content: expect.any(Function),
+      },
+    ]);
+    const contents = await Promise.all(files.map(f => f.content()));
+    expect(contents.map(c => c.toString('utf8').trim())).toEqual([
+      '# Test',
+      'site_name: Test',
+    ]);
+  });
+
+  it('should read files with filter', async () => {
+    const stream = fs.createReadStream('/test-archive.zip');
+
+    const res = new ZipArchiveResponse(stream, 'mock-repo/', '/tmp', path =>
+      path.endsWith('.yml'),
+    );
+    const files = await res.files();
+
+    expect(files).toEqual([
+      {
+        path: 'mkdocs.yml',
+        content: expect.any(Function),
+      },
+    ]);
+    const content = await files[0].content();
+    expect(content.toString('utf8').trim()).toEqual('site_name: Test');
+  });
+
+  it('should read as archive and files', async () => {
+    const stream = fs.createReadStream('/test-archive.zip');
+
+    const res = new ZipArchiveResponse(stream, 'mock-repo/', '/tmp');
+    const buffer = await res.archive();
+
+    await expect(res.archive()).rejects.toThrow(
+      'Response has already been read',
+    );
+
+    const res2 = new ZipArchiveResponse(buffer, '', '/tmp');
+    const files = await res2.files();
+
+    expect(files).toEqual([
+      {
+        path: 'docs/index.md',
+        content: expect.any(Function),
+      },
+      {
+        path: 'mkdocs.yml',
+        content: expect.any(Function),
+      },
+    ]);
+    const contents = await Promise.all(files.map(f => f.content()));
+    expect(contents.map(c => c.toString('utf8').trim())).toEqual([
+      '# Test',
+      'site_name: Test',
+    ]);
+  });
+
+  it('should extract entire archive into directory', async () => {
+    const stream = fs.createReadStream('/test-archive.zip');
+
+    const res = new ZipArchiveResponse(stream, '', '/tmp');
+    const dir = await res.dir();
+
+    await expect(
+      fs.readFile(resolvePath(dir, 'mock-repo/mkdocs.yml'), 'utf8'),
+    ).resolves.toBe('site_name: Test\n');
+    await expect(
+      fs.readFile(resolvePath(dir, 'mock-repo/docs/index.md'), 'utf8'),
+    ).resolves.toBe('# Test\n');
+  });
+
+  it('should extract archive into directory with a subpath', async () => {
+    const stream = fs.createReadStream('/test-archive.zip');
+
+    const res = new ZipArchiveResponse(stream, 'mock-repo/docs/', '/tmp');
+    const dir = await res.dir();
+
+    expect(dir).toMatch(/^\/tmp\/.*$/);
+    await expect(
+      fs.readFile(resolvePath(dir, 'index.md'), 'utf8'),
+    ).resolves.toBe('# Test\n');
+  });
+
+  it('should extract archive into directory with a subpath and filter', async () => {
+    const stream = fs.createReadStream('/test-archive.zip');
+
+    const res = new ZipArchiveResponse(stream, 'mock-repo/', '/tmp', path =>
+      path.endsWith('.yml'),
+    );
+    const dir = await res.dir({ targetDir: '/tmp' });
+
+    expect(dir).toBe('/tmp');
+    await expect(fs.pathExists(resolvePath(dir, 'mkdocs.yml'))).resolves.toBe(
+      true,
+    );
+    await expect(
+      fs.pathExists(resolvePath(dir, 'docs/index.md')),
+    ).resolves.toBe(false);
+  });
+});
@@ -0,0 +1,154 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import path from 'path';
+import fs from 'fs-extra';
+import unzipper, { Entry } from 'unzipper';
+import archiver from 'archiver';
+import { Readable } from 'stream';
+import {
+  ReadTreeResponse,
+  ReadTreeResponseFile,
+  ReadTreeResponseDirOptions,
+} from '../types';
+
+/**
+ * Wraps a zip archive stream into a tree response reader.
+ */
+export class ZipArchiveResponse implements ReadTreeResponse {
+  private read = false;
+
+  constructor(
+    private readonly stream: Readable,
+    private readonly subPath: string,
+    private readonly workDir: string,
+    private readonly filter?: (path: string) => boolean,
+  ) {
+    if (subPath) {
+      if (!subPath.endsWith('/')) {
+        this.subPath += '/';
+      }
+      if (subPath.startsWith('/')) {
+        throw new TypeError(
+          `ZipArchiveResponse subPath must not start with a /, got '${subPath}'`,
+        );
+      }
+    }
+  }
+
+  // Make sure the input stream is only read once
+  private onlyOnce() {
+    if (this.read) {
+      throw new Error('Response has already been read');
+    }
+    this.read = true;
+  }
+
+  private getPath(entry: Entry): string {
+    return entry.path.slice(this.subPath.length);
+  }
+
+  private shouldBeIncluded(entry: Entry): boolean {
+    if (this.subPath) {
+      if (!entry.path.startsWith(this.subPath)) {
+        return false;
+      }
+    }
+    if (this.filter) {
+      return this.filter(this.getPath(entry));
+    }
+    return true;
+  }
+
+  async files(): Promise<ReadTreeResponseFile[]> {
+    this.onlyOnce();
+
+    const files = Array<ReadTreeResponseFile>();
+
+    await this.stream
+      .pipe(unzipper.Parse())
+      .on('entry', (entry: Entry) => {
+        if (entry.type === 'Directory') {
+          entry.resume();
+          return;
+        }
+
+        if (this.shouldBeIncluded(entry)) {
+          files.push({
+            path: this.getPath(entry),
+            content: () => entry.buffer(),
+          });
+        } else {
+          entry.autodrain();
+        }
+      })
+      .promise();
+
+    return files;
+  }
+
+  async archive(): Promise<Readable> {
+    this.onlyOnce();
+
+    if (!this.subPath) {
+      return this.stream;
+    }
+
+    const archive = archiver('zip');
+    await this.stream
+      .pipe(unzipper.Parse())
+      .on('entry', (entry: Entry) => {
+        if (entry.type === 'File' && this.shouldBeIncluded(entry)) {
+          archive.append(entry, { name: this.getPath(entry) });
+        } else {
+          entry.autodrain();
+        }
+      })
+      .promise();
+    archive.finalize();
+
+    return archive;
+  }
+
+  async dir(options?: ReadTreeResponseDirOptions): Promise<string> {
+    this.onlyOnce();
+
+    const dir =
+      options?.targetDir ??
+      (await fs.mkdtemp(path.join(this.workDir, 'backstage-')));
+
+    await this.stream
+      .pipe(unzipper.Parse())
+      .on('entry', (entry: Entry) => {
+        if (this.shouldBeIncluded(entry)) {
+          if (entry.type === 'Directory') {
+            const directoryPath = this.getPath(entry);
+            if (directoryPath) {
+              fs.mkdirSync(path.join(dir, this.getPath(entry)));
+            }
+            entry.resume();
+            return;
+          }
+          entry.pipe(fs.createWriteStream(path.join(dir, this.getPath(entry))));
+        } else {
+          entry.autodrain();
+        }
+      })
+      .promise();
+
+    return dir;
+  }
+}