From 1b632b5aa71f695a3d265be6ec6327c4158babb5 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 10 Nov 2020 10:47:24 +0100 Subject: [PATCH] backend-common: add archive tree reading utilities --- .../src/reading/tree/ArchiveResponse.test.ts | 93 +++++++++++++ .../src/reading/tree/ArchiveResponse.ts | 124 ++++++++++++++++++ .../reading/tree/ReadTreeResponseFactory.ts | 47 +++++++ .../backend-common/src/reading/tree/index.ts | 17 +++ 4 files changed, 281 insertions(+) create mode 100644 packages/backend-common/src/reading/tree/ArchiveResponse.test.ts create mode 100644 packages/backend-common/src/reading/tree/ArchiveResponse.ts create mode 100644 packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts create mode 100644 packages/backend-common/src/reading/tree/index.ts diff --git a/packages/backend-common/src/reading/tree/ArchiveResponse.test.ts b/packages/backend-common/src/reading/tree/ArchiveResponse.test.ts new file mode 100644 index 0000000000..57a490650c --- /dev/null +++ b/packages/backend-common/src/reading/tree/ArchiveResponse.test.ts @@ -0,0 +1,93 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import mockFs from 'mock-fs'; +import { Readable } from 'stream'; +import { resolve as resolvePath } from 'path'; +import { ArchiveResponse } from './ArchiveResponse'; + +const archiveData = fs.readFileSync( + resolvePath(__filename, '../../__fixtures__/repo.tar.gz'), +); + +describe('ArchiveResponse', () => { + beforeEach(() => { + mockFs({ + '/test-archive.tar.gz': archiveData, + '/tmp': mockFs.directory(), + }); + }); + + afterEach(() => { + mockFs.restore(); + }); + + it('should read as archive and files', async () => { + const stream = fs.createReadStream('/test-archive.tar.gz'); + + const res = new ArchiveResponse(stream, 'mock-repo', '/tmp'); + const buffer = await res.archive(); + + await expect(res.archive()).rejects.toThrow( + 'Response has already been read', + ); + + const res2 = new ArchiveResponse(Readable.from(buffer), '', '/tmp'); + const files = await res2.files(); + + expect(files).toEqual([ + { + path: './mkdocs.yml', + content: expect.any(Function), + }, + { + path: './docs/index.md', + content: expect.any(Function), + }, + ]); + const contents = await Promise.all(files.map(f => f.content())); + expect(contents.map(c => c.toString('utf8').trim())).toEqual([ + 'site_name: Test', + '# Test', + ]); + }); + + it('should extract entire archive into directory', async () => { + const stream = fs.createReadStream('/test-archive.tar.gz'); + + const res = new ArchiveResponse(stream, '', '/tmp'); + const dir = await res.dir(); + + await expect( + fs.readFile(resolvePath(dir, 'mock-repo/mkdocs.yml'), 'utf8'), + ).resolves.toBe('site_name: Test\n'); + await expect( + fs.readFile(resolvePath(dir, 'mock-repo/docs/index.md'), 'utf8'), + ).resolves.toBe('# Test\n'); + }); + + it('should extract archive into directory with a subpath', async () => { + const stream = fs.createReadStream('/test-archive.tar.gz'); + + const res = new ArchiveResponse(stream, 'mock-repo/docs', '/tmp'); + const dir = await res.dir(); + + await expect( + fs.readFile(resolvePath(dir, 'index.md'), 'utf8'), + ).resolves.toBe('# Test\n'); + }); +}); diff --git a/packages/backend-common/src/reading/tree/ArchiveResponse.ts b/packages/backend-common/src/reading/tree/ArchiveResponse.ts new file mode 100644 index 0000000000..65ad8c18fe --- /dev/null +++ b/packages/backend-common/src/reading/tree/ArchiveResponse.ts @@ -0,0 +1,124 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import os from 'os'; +import tar, { Parse, ParseStream, ReadEntry } from 'tar'; +import path from 'path'; +import fs from 'fs-extra'; +import { Readable, pipeline as pipelineCb } from 'stream'; +import { promisify } from 'util'; +import concatStream from 'concat-stream'; +import { ReadTreeResponse, File } from '../types'; + +// Tar types for `Parse` is not a proper constructor, but it should be +const TarParseStream = (Parse as unknown) as { new (): ParseStream }; + +const pipeline = promisify(pipelineCb); + +/** + * Wraps a tar archive stream into a tree response reader. + */ +export class ArchiveResponse implements ReadTreeResponse { + private read = false; + + constructor( + private readonly stream: Readable, + private readonly subPath: string, + private readonly workDir: string = os.tmpdir(), + ) {} + + // Make sure the input stream is only read once + private onlyOnce() { + if (this.read) { + throw new Error('Response has already been read'); + } + this.read = true; + } + + async files(): Promise { + this.onlyOnce(); + + const files: File[] = []; + const parser = new TarParseStream(); + + parser.on('entry', (entry: ReadEntry & Readable) => { + if (entry.type === 'Directory') { + entry.resume(); + return; + } + + const contentPromise = new Promise(async resolve => { + await pipeline(entry, concatStream(resolve)); + }); + + files.push({ + path: entry.path, + content: () => contentPromise, + }); + + entry.resume(); + }); + + await pipeline(this.stream, parser); + + return files; + } + + async archive(): Promise { + if (!this.subPath) { + this.onlyOnce(); + + return new Promise(resolve => + pipeline(this.stream, concatStream(resolve)), + ); + } + + // TODO(Rugvip): method for repacking a tar with a subpath is to simply extract into a + // tmp dir and recreate the archive. Would be nicer to stream things instead. + const tmpDir = await this.dir(); + + try { + return await new Promise(async resolve => { + await pipeline( + tar.create({ cwd: tmpDir }, ['.']), + concatStream(resolve), + ); + }); + } finally { + await fs.remove(tmpDir); + } + } + + async dir(outDir?: string): Promise { + this.onlyOnce(); + + const dir = + outDir ?? (await fs.mkdtemp(path.join(this.workDir, 'backstage-'))); + + const strip = this.subPath ? this.subPath.split('/').length : 0; + + await pipeline( + this.stream, + tar.extract({ + strip, + cwd: dir, + filter: path => path.startsWith(this.subPath), + }), + ); + + return dir; + } +} diff --git a/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts new file mode 100644 index 0000000000..0cdf926720 --- /dev/null +++ b/packages/backend-common/src/reading/tree/ReadTreeResponseFactory.ts @@ -0,0 +1,47 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import os from 'os'; +import { Readable } from 'stream'; +import { Config } from '@backstage/config'; +import { ReadTreeResponse } from '../types'; +import { ArchiveResponse } from './ArchiveResponse'; + +type FromArchiveOptions = { + // A binary stream of a tar archive. + stream: Readable; + // If set, root of the tree will be set to the given path. Should not have a trailing `/`. + path?: string; +}; + +export class ReadTreeResponseFactory { + static create(options: { config: Config }): ReadTreeResponseFactory { + return new ReadTreeResponseFactory( + options.config.getOptionalString('backend.workingDirectory') ?? + os.tmpdir(), + ); + } + + constructor(private readonly workDir: string) {} + + async fromArchive(options: FromArchiveOptions): Promise { + return new ArchiveResponse( + options.stream, + options.path ?? '', + this.workDir, + ); + } +} diff --git a/packages/backend-common/src/reading/tree/index.ts b/packages/backend-common/src/reading/tree/index.ts new file mode 100644 index 0000000000..858cf15877 --- /dev/null +++ b/packages/backend-common/src/reading/tree/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { ReadTreeResponseFactory } from './ReadTreeResponseFactory';