backend-common: add archive tree reading utilities

2020-11-10 10:47:24 +01:00
parent e74422b4d8
commit 1b632b5aa7
4 changed files with 281 additions and 0 deletions
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs-extra';
+import mockFs from 'mock-fs';
+import { Readable } from 'stream';
+import { resolve as resolvePath } from 'path';
+import { ArchiveResponse } from './ArchiveResponse';
+
+const archiveData = fs.readFileSync(
+  resolvePath(__filename, '../../__fixtures__/repo.tar.gz'),
+);
+
+describe('ArchiveResponse', () => {
+  beforeEach(() => {
+    mockFs({
+      '/test-archive.tar.gz': archiveData,
+      '/tmp': mockFs.directory(),
+    });
+  });
+
+  afterEach(() => {
+    mockFs.restore();
+  });
+
+  it('should read as archive and files', async () => {
+    const stream = fs.createReadStream('/test-archive.tar.gz');
+
+    const res = new ArchiveResponse(stream, 'mock-repo', '/tmp');
+    const buffer = await res.archive();
+
+    await expect(res.archive()).rejects.toThrow(
+      'Response has already been read',
+    );
+
+    const res2 = new ArchiveResponse(Readable.from(buffer), '', '/tmp');
+    const files = await res2.files();
+
+    expect(files).toEqual([
+      {
+        path: './mkdocs.yml',
+        content: expect.any(Function),
+      },
+      {
+        path: './docs/index.md',
+        content: expect.any(Function),
+      },
+    ]);
+    const contents = await Promise.all(files.map(f => f.content()));
+    expect(contents.map(c => c.toString('utf8').trim())).toEqual([
+      'site_name: Test',
+      '# Test',
+    ]);
+  });
+
+  it('should extract entire archive into directory', async () => {
+    const stream = fs.createReadStream('/test-archive.tar.gz');
+
+    const res = new ArchiveResponse(stream, '', '/tmp');
+    const dir = await res.dir();
+
+    await expect(
+      fs.readFile(resolvePath(dir, 'mock-repo/mkdocs.yml'), 'utf8'),
+    ).resolves.toBe('site_name: Test\n');
+    await expect(
+      fs.readFile(resolvePath(dir, 'mock-repo/docs/index.md'), 'utf8'),
+    ).resolves.toBe('# Test\n');
+  });
+
+  it('should extract archive into directory with a subpath', async () => {
+    const stream = fs.createReadStream('/test-archive.tar.gz');
+
+    const res = new ArchiveResponse(stream, 'mock-repo/docs', '/tmp');
+    const dir = await res.dir();
+
+    await expect(
+      fs.readFile(resolvePath(dir, 'index.md'), 'utf8'),
+    ).resolves.toBe('# Test\n');
+  });
+});
@@ -0,0 +1,124 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import os from 'os';
+import tar, { Parse, ParseStream, ReadEntry } from 'tar';
+import path from 'path';
+import fs from 'fs-extra';
+import { Readable, pipeline as pipelineCb } from 'stream';
+import { promisify } from 'util';
+import concatStream from 'concat-stream';
+import { ReadTreeResponse, File } from '../types';
+
+// Tar types for `Parse` is not a proper constructor, but it should be
+const TarParseStream = (Parse as unknown) as { new (): ParseStream };
+
+const pipeline = promisify(pipelineCb);
+
+/**
+ * Wraps a tar archive stream into a tree response reader.
+ */
+export class ArchiveResponse implements ReadTreeResponse {
+  private read = false;
+
+  constructor(
+    private readonly stream: Readable,
+    private readonly subPath: string,
+    private readonly workDir: string = os.tmpdir(),
+  ) {}
+
+  // Make sure the input stream is only read once
+  private onlyOnce() {
+    if (this.read) {
+      throw new Error('Response has already been read');
+    }
+    this.read = true;
+  }
+
+  async files(): Promise<File[]> {
+    this.onlyOnce();
+
+    const files: File[] = [];
+    const parser = new TarParseStream();
+
+    parser.on('entry', (entry: ReadEntry & Readable) => {
+      if (entry.type === 'Directory') {
+        entry.resume();
+        return;
+      }
+
+      const contentPromise = new Promise<Buffer>(async resolve => {
+        await pipeline(entry, concatStream(resolve));
+      });
+
+      files.push({
+        path: entry.path,
+        content: () => contentPromise,
+      });
+
+      entry.resume();
+    });
+
+    await pipeline(this.stream, parser);
+
+    return files;
+  }
+
+  async archive(): Promise<Buffer> {
+    if (!this.subPath) {
+      this.onlyOnce();
+
+      return new Promise(resolve =>
+        pipeline(this.stream, concatStream(resolve)),
+      );
+    }
+
+    // TODO(Rugvip): method for repacking a tar with a subpath is to simply extract into a
+    //               tmp dir and recreate the archive. Would be nicer to stream things instead.
+    const tmpDir = await this.dir();
+
+    try {
+      return await new Promise(async resolve => {
+        await pipeline(
+          tar.create({ cwd: tmpDir }, ['.']),
+          concatStream(resolve),
+        );
+      });
+    } finally {
+      await fs.remove(tmpDir);
+    }
+  }
+
+  async dir(outDir?: string): Promise<string> {
+    this.onlyOnce();
+
+    const dir =
+      outDir ?? (await fs.mkdtemp(path.join(this.workDir, 'backstage-')));
+
+    const strip = this.subPath ? this.subPath.split('/').length : 0;
+
+    await pipeline(
+      this.stream,
+      tar.extract({
+        strip,
+        cwd: dir,
+        filter: path => path.startsWith(this.subPath),
+      }),
+    );
+
+    return dir;
+  }
+}
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import os from 'os';
+import { Readable } from 'stream';
+import { Config } from '@backstage/config';
+import { ReadTreeResponse } from '../types';
+import { ArchiveResponse } from './ArchiveResponse';
+
+type FromArchiveOptions = {
+  // A binary stream of a tar archive.
+  stream: Readable;
+  // If set, root of the tree will be set to the given path. Should not have a trailing `/`.
+  path?: string;
+};
+
+export class ReadTreeResponseFactory {
+  static create(options: { config: Config }): ReadTreeResponseFactory {
+    return new ReadTreeResponseFactory(
+      options.config.getOptionalString('backend.workingDirectory') ??
+        os.tmpdir(),
+    );
+  }
+
+  constructor(private readonly workDir: string) {}
+
+  async fromArchive(options: FromArchiveOptions): Promise<ReadTreeResponse> {
+    return new ArchiveResponse(
+      options.stream,
+      options.path ?? '',
+      this.workDir,
+    );
+  }
+}
@@ -0,0 +1,17 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export { ReadTreeResponseFactory } from './ReadTreeResponseFactory';