paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 38 Packages Projects Releases Wiki Activity

Introduce ZipArchiveResponse

Browse Source
This commit is contained in:
Mattias Frinnström
2020-11-16 16:08:05 +01:00
parent c6cf38c6a4
commit 9367dde00d
5 changed files with 512 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/backend-common/package.json
+4
View File
@@ -36,6 +36,7 @@
"@backstage/test-utils": "^0.1.3",
"@types/cors": "^2.8.6",
"@types/express": "^4.17.6",
"archiver": "^5.0.2",
"compression": "^1.7.4",
"concat-stream": "^2.0.0",
"cors": "^2.8.5",
@@ -55,6 +56,7 @@
"selfsigned": "^1.10.7",
"stoppable": "^1.1.0",
"tar": "^6.0.5",
"unzipper": "^0.10.11",
"winston": "^3.2.1"
},
"peerDependencies": {
@@ -67,6 +69,7 @@
},
"devDependencies": {
"@backstage/cli": "^0.3.0",
"@types/archiver": "^3.1.1",
"@types/compression": "^1.7.0",
"@types/concat-stream": "^1.6.0",
"@types/fs-extra": "^9.0.3",
@@ -78,6 +81,7 @@
"@types/stoppable": "^1.1.0",
"@types/supertest": "^2.0.8",
"@types/tar": "^4.0.3",
"@types/unzipper": "^0.10.3",
"@types/webpack-env": "^1.15.2",
"@types/yaml": "^1.9.7",
"get-port": "^5.1.1",
packages/backend-common/src/reading/__fixtures__/repo.zip
BIN
View File
Binary file not shown.
packages/backend-common/src/reading/tree/ZipArchiveResponse.test.ts
+151
View File
@@ -0,0 +1,151 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs-extra';
import mockFs from 'mock-fs';
import { resolve as resolvePath } from 'path';
import { ZipArchiveResponse } from './ZipArchiveResponse';
const archiveData = fs.readFileSync(
resolvePath(__filename, '../../__fixtures__/repo.zip'),
);
describe('ZipArchiveResponse', () => {
beforeEach(() => {
mockFs({
'/test-archive.zip': archiveData,
'/tmp': mockFs.directory(),
});
});
afterEach(() => {
mockFs.restore();
});
it('should read files', async () => {
const stream = fs.createReadStream('/test-archive.zip');
const res = new ZipArchiveResponse(stream, 'mock-repo/', '/tmp');
const files = await res.files();
expect(files).toEqual([
{
path: 'docs/index.md',
content: expect.any(Function),
},
{
path: 'mkdocs.yml',
content: expect.any(Function),
},
]);
const contents = await Promise.all(files.map(f => f.content()));
expect(contents.map(c => c.toString('utf8').trim())).toEqual([
'# Test',
'site_name: Test',
]);
});
it('should read files with filter', async () => {
const stream = fs.createReadStream('/test-archive.zip');
const res = new ZipArchiveResponse(stream, 'mock-repo/', '/tmp', path =>
path.endsWith('.yml'),
);
const files = await res.files();
expect(files).toEqual([
{
path: 'mkdocs.yml',
content: expect.any(Function),
},
]);
const content = await files[0].content();
expect(content.toString('utf8').trim()).toEqual('site_name: Test');
});
it('should read as archive and files', async () => {
const stream = fs.createReadStream('/test-archive.zip');
const res = new ZipArchiveResponse(stream, 'mock-repo/', '/tmp');
const buffer = await res.archive();
await expect(res.archive()).rejects.toThrow(
'Response has already been read',
);
const res2 = new ZipArchiveResponse(buffer, '', '/tmp');
const files = await res2.files();
expect(files).toEqual([
{
path: 'docs/index.md',
content: expect.any(Function),
},
{
path: 'mkdocs.yml',
content: expect.any(Function),
},
]);
const contents = await Promise.all(files.map(f => f.content()));
expect(contents.map(c => c.toString('utf8').trim())).toEqual([
'# Test',
'site_name: Test',
]);
});
it('should extract entire archive into directory', async () => {
const stream = fs.createReadStream('/test-archive.zip');
const res = new ZipArchiveResponse(stream, '', '/tmp');
const dir = await res.dir();
await expect(
fs.readFile(resolvePath(dir, 'mock-repo/mkdocs.yml'), 'utf8'),
).resolves.toBe('site_name: Test\n');
await expect(
fs.readFile(resolvePath(dir, 'mock-repo/docs/index.md'), 'utf8'),
).resolves.toBe('# Test\n');
});
it('should extract archive into directory with a subpath', async () => {
const stream = fs.createReadStream('/test-archive.zip');
const res = new ZipArchiveResponse(stream, 'mock-repo/docs/', '/tmp');
const dir = await res.dir();
expect(dir).toMatch(/^\/tmp\/.*$/);
await expect(
fs.readFile(resolvePath(dir, 'index.md'), 'utf8'),
).resolves.toBe('# Test\n');
});
it('should extract archive into directory with a subpath and filter', async () => {
const stream = fs.createReadStream('/test-archive.zip');
const res = new ZipArchiveResponse(stream, 'mock-repo/', '/tmp', path =>
path.endsWith('.yml'),
);
const dir = await res.dir({ targetDir: '/tmp' });
expect(dir).toBe('/tmp');
await expect(fs.pathExists(resolvePath(dir, 'mkdocs.yml'))).resolves.toBe(
true,
);
await expect(
fs.pathExists(resolvePath(dir, 'docs/index.md')),
).resolves.toBe(false);
});
});
packages/backend-common/src/reading/tree/ZipArchiveResponse.ts
+154
View File
@@ -0,0 +1,154 @@
/*
* Copyright 2020 Spotify AB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import path from 'path';
import fs from 'fs-extra';
import unzipper, { Entry } from 'unzipper';
import archiver from 'archiver';
import { Readable } from 'stream';
import {
ReadTreeResponse,
ReadTreeResponseFile,
ReadTreeResponseDirOptions,
} from '../types';
/**
* Wraps a zip archive stream into a tree response reader.
*/
export class ZipArchiveResponse implements ReadTreeResponse {
private read = false;
constructor(
private readonly stream: Readable,
private readonly subPath: string,
private readonly workDir: string,
private readonly filter?: (path: string) => boolean,
) {
if (subPath) {
if (!subPath.endsWith('/')) {
this.subPath += '/';
}
if (subPath.startsWith('/')) {
throw new TypeError(
`ZipArchiveResponse subPath must not start with a /, got '${subPath}'`,
);
}
}
}
// Make sure the input stream is only read once
private onlyOnce() {
if (this.read) {
throw new Error('Response has already been read');
}
this.read = true;
}
private getPath(entry: Entry): string {
return entry.path.slice(this.subPath.length);
}
private shouldBeIncluded(entry: Entry): boolean {
if (this.subPath) {
if (!entry.path.startsWith(this.subPath)) {
return false;
}
}
if (this.filter) {
return this.filter(this.getPath(entry));
}
return true;
}
async files(): Promise<ReadTreeResponseFile[]> {
this.onlyOnce();
const files = Array<ReadTreeResponseFile>();
await this.stream
.pipe(unzipper.Parse())
.on('entry', (entry: Entry) => {
if (entry.type === 'Directory') {
entry.resume();
return;
}
if (this.shouldBeIncluded(entry)) {
files.push({
path: this.getPath(entry),
content: () => entry.buffer(),
});
} else {
entry.autodrain();
}
})
.promise();
return files;
}
async archive(): Promise<Readable> {
this.onlyOnce();
if (!this.subPath) {
return this.stream;
}
const archive = archiver('zip');
await this.stream
.pipe(unzipper.Parse())
.on('entry', (entry: Entry) => {
if (entry.type === 'File' && this.shouldBeIncluded(entry)) {
archive.append(entry, { name: this.getPath(entry) });
} else {
entry.autodrain();
}
})
.promise();
archive.finalize();
return archive;
}
async dir(options?: ReadTreeResponseDirOptions): Promise<string> {
this.onlyOnce();
const dir =
options?.targetDir ??
(await fs.mkdtemp(path.join(this.workDir, 'backstage-')));
await this.stream
.pipe(unzipper.Parse())
.on('entry', (entry: Entry) => {
if (this.shouldBeIncluded(entry)) {
if (entry.type === 'Directory') {
const directoryPath = this.getPath(entry);
if (directoryPath) {
fs.mkdirSync(path.join(dir, this.getPath(entry)));
}
entry.resume();
return;
}
entry.pipe(fs.createWriteStream(path.join(dir, this.getPath(entry))));
} else {
entry.autodrain();
}
})
.promise();
return dir;
}
}