paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 43 Packages Projects Releases Wiki Activity

chore: made some progress moving over to a new library for this

Browse Source 
Signed-off-by: blam <ben@blam.sh>
This commit is contained in:
blam
2022-07-07 11:55:40 +02:00
parent 6cd6946674
commit 2b59c8ed4d
4 changed files with 164 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/backend-common/package.json
+3 -1
View File
@@ -41,6 +41,7 @@
"@backstage/integration": "^1.2.2-next.3",
"@backstage/types": "^1.0.0",
"@google-cloud/storage": "^6.0.0",
"@keyv/redis": "^2.2.3",
"@manypkg/get-packages": "^1.1.3",
"@octokit/rest": "^19.0.3",
"@types/cors": "^2.8.6",
@@ -48,6 +49,7 @@
"@types/express": "^4.17.6",
"@types/luxon": "^2.0.4",
"@types/webpack-env": "^1.15.2",
"@types/yauzl": "^2.10.0",
"archiver": "^5.0.2",
"aws-sdk": "^2.840.0",
"base64-stream": "^1.0.0",
@@ -64,7 +66,6 @@
"jose": "^4.6.0",
"keyv": "^4.0.3",
"keyv-memcache": "^1.2.5",
"@keyv/redis": "^2.2.3",
"knex": "^2.0.0",
"lodash": "^4.17.21",
"logform": "^2.3.2",
@@ -80,6 +81,7 @@
"tar": "^6.1.2",
"unzipper": "^0.10.11",
"winston": "^3.2.1",
"yauzl": "^2.10.0",
"yn": "^4.0.0"
},
"peerDependencies": {
packages/backend-common/src/reading/tree/ZipArchiveResponse.ts
+78 -57
View File
@@ -15,25 +15,27 @@
*/
import archiver from 'archiver';
import unzipper2, { Entry } from 'yauzl';
import fs from 'fs-extra';
import platformPath from 'path';
import { Readable } from 'stream';
import unzipper, { Entry } from 'unzipper';
// import unzipper, { Entry } from 'unzipper';
import {
ReadTreeResponse,
ReadTreeResponseDirOptions,
ReadTreeResponseFile,
} from '../types';
import { streamToTimeoutPromise } from './util';
import { zip } from 'lodash';
const guardCorruptZipStream = (stream: Readable) =>
streamToTimeoutPromise(stream, {
eventName: 'entry',
timeoutMs: 3000,
getError: (entry: Entry) =>
new Error(`Timed out while unzipping ${entry.type}: ${entry.path}`),
const streamToBuffer = async (stream: Readable): Promise<Buffer> => {
const buffers: Buffer[] = [];
return new Promise((resolve, reject) => {
stream.on('data', (data: Buffer) => buffers.push(data));
stream.on('error', reject);
stream.on('end', () => resolve(Buffer.concat(buffers)));
});
};
/**
* Wraps a zip archive stream into a tree response reader.
*/
@@ -76,15 +78,13 @@ export class ZipArchiveResponse implements ReadTreeResponse {
private shouldBeIncluded(entry: Entry): boolean {
if (this.subPath) {
if (!entry.path.startsWith(this.subPath)) {
if (!entry.fileName.startsWith(this.subPath)) {
return false;
}
}
if (this.filter) {
return this.filter(this.getInnerPath(entry.path), {
size:
(entry.vars as { uncompressedSize?: number }).uncompressedSize ??
entry.vars.compressedSize,
return this.filter(this.getInnerPath(entry.fileName), {
size: entry.uncompressedSize,
});
}
return true;
@@ -92,29 +92,36 @@ export class ZipArchiveResponse implements ReadTreeResponse {
async files(): Promise<ReadTreeResponseFile[]> {
this.onlyOnce();
const files = Array<ReadTreeResponseFile>();
const parseStream = this.stream
.pipe(unzipper.Parse())
.on('entry', (entry: Entry) => {
if (entry.type === 'Directory') {
entry.resume();
const buffer = await streamToBuffer(this.stream);
return await new Promise((resolve, reject) => {
unzipper2.fromBuffer(buffer, { lazyEntries: false }, (err, zipfile) => {
if (err) {
reject(err);
return;
}
zipfile.on('entry', async (entry: Entry) => {
// If it's not a directory, and it's included, then grab the contents of the file from the buffer
if (!/\/$/.test(entry.fileName) && this.shouldBeIncluded(entry)) {
files.push({
path: this.getInnerPath(entry.fileName),
content: () =>
new Promise<Buffer>((cResolve, cReject) => {
zipfile.openReadStream(entry, async (cError, readStream) => {
if (cError) {
return cReject(cError);
}
return cResolve(await streamToBuffer(readStream));
});
}),
});
}
});
if (this.shouldBeIncluded(entry)) {
files.push({
path: this.getInnerPath(entry.path),
content: () => entry.buffer(),
});
} else {
entry.autodrain();
}
zipfile.once('end', () => resolve(files));
});
await guardCorruptZipStream(parseStream);
return files;
});
}
async archive(): Promise<Readable> {
@@ -124,17 +131,32 @@ export class ZipArchiveResponse implements ReadTreeResponse {
return this.stream;
}
const buffer = await streamToBuffer(this.stream);
const archive = archiver('zip');
const parseStream = this.stream
.pipe(unzipper.Parse())
.on('entry', (entry: Entry) => {
if (entry.type === 'File' && this.shouldBeIncluded(entry)) {
archive.append(entry, { name: this.getInnerPath(entry.path) });
} else {
entry.autodrain();
await new Promise<void>((resolve, reject) => {
unzipper2.fromBuffer(buffer, { lazyEntries: false }, (err, zipfile) => {
if (err) {
reject(err);
return;
}
zipfile.on('entry', async (entry: Entry) => {
// If it's not a directory, and it's included, then grab the contents of the file from the buffer
if (!/\/$/.test(entry.fileName) && this.shouldBeIncluded(entry)) {
zipfile.openReadStream(entry, async (err2, readStream) => {
if (err2) {
reject(err2);
return;
}
archive.append(await streamToBuffer(readStream), {
name: this.getInnerPath(entry.fileName),
});
});
}
});
zipfile.once('end', () => resolve());
});
await guardCorruptZipStream(parseStream);
});
archive.finalize();
@@ -143,29 +165,28 @@ export class ZipArchiveResponse implements ReadTreeResponse {
async dir(options?: ReadTreeResponseDirOptions): Promise<string> {
this.onlyOnce();
const dir =
options?.targetDir ??
(await fs.mkdtemp(platformPath.join(this.workDir, 'backstage-')));
const parseStream = this.stream
.pipe(unzipper.Parse())
.on('entry', async (entry: Entry) => {
// Ignore directory entries since we handle that with the file entries
// as a zip can have files with directories without directory entries
if (entry.type === 'File' && this.shouldBeIncluded(entry)) {
const entryPath = this.getInnerPath(entry.path);
const dirname = platformPath.dirname(entryPath);
if (dirname) {
await fs.mkdirp(platformPath.join(dir, dirname));
return new Promise((resolve, reject) => {
const parseStream = this.stream
.pipe(unzipper.Parse())
.on('entry', async (entry: Entry) => {
// Ignore directory entries since we handle that with the file entries
// as a zip can have files with directories without directory entries
if (entry.type === 'File' && this.shouldBeIncluded(entry)) {
const entryPath = this.getInnerPath(entry.path);
const dirname = platformPath.dirname(entryPath);
if (dirname) {
await fs.mkdirp(platformPath.join(dir, dirname));
}
entry.pipe(fs.createWriteStream(platformPath.join(dir, entryPath)));
} else {
entry.autodrain();
}
entry.pipe(fs.createWriteStream(platformPath.join(dir, entryPath)));
} else {
entry.autodrain();
}
});
await guardCorruptZipStream(parseStream);
return dir;
})
.on('finish', () => resolve(dir));
});
}
}
plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml
+75 -21
View File
@@ -8,28 +8,82 @@ spec:
type: website
owner: team-a
parameters:
- name: Enter some stuff
description: Enter some stuff
- name: Repositories
description: Some repo
properties:
inputString:
appRepoUrl:
type: string
title: string input test
inputObject:
type: object
title: object input test
description: a little nested thing never hurt anyone right?
properties:
first:
type: string
title: first
second:
type: number
title: second
title: First Repository
ui:field: RepoUrlPicker
serviceRepoUrl:
type: string
title: First Repository
ui:field: RepoUrlPicker
steps:
- id: debug
if: ${{ true === true }}
name: Debug
action: debug:log
# First get the app template folder, and template into ./app
- id: app_template
name: Fetch Skeleton + Template
action: fetch:template
input:
message: ${{ parameters.inputString }}
extra: ${{ parameters.inputObject }}
url: ./skeleton
targetPath: ./app
copyWithoutRender:
- .github/*
values:
component_id: ${{ parameters.component_id }}
description: ${{ parameters.description }}
services_app_port: ${{ parameters.services_app_port }}
owner: ${{ parameters.owner }}
destination: ${{ parameters.appRepoUrl | parseRepoUrl }}
# First then service into ./service
- id: app_service_config_template
name: Fetch App Servcie Config. Skeleton + Template
action: fetch:template
input:
url: https://github.com/my-org/helm-values-template
targetPath: ./service
copyWithoutRender:
- .github/*
values:
component_id: ${{ parameters.component_id }}
description: ${{ parameters.description }}
services_app_port: ${{ parameters.services_app_port }}
owner: ${{ parameters.owner }}
destination: ${{ parameters.serviceRepoUrl | parseRepoUrl }}
# Publish the app
- id: publish_app
name: Publish App
action: publish:github
input:
sourcePath: ./app
allowedHosts: ['github.com']
description: This is ${{ parameters.component_id }}
repoUrl: ${{ parameters.appRepoUrl }}
# Publish the service
- id: publish_service
name: Publish Service
action: publish:github
input:
sourcePath: ./service
allowedHosts: ['github.com']
description: This is ${{ parameters.component_id }}
repoUrl: ${{ parameters.serviceRepoUrl }}
# Register the app
- id: register_app
name: Register Application
action: catalog:register
input:
repoContentsUrl: ${{ steps.publish_app.output.repoContentsUrl }}
catalogInfoPath: '/catalog-info.yaml'
# Register the service
- id: register_service
name: Register Application
action: catalog:register
input:
repoContentsUrl: ${{ steps.publish_service.output.repoContentsUrl }}
catalogInfoPath: '/catalog-info.yaml'
yarn.lock
+8 -1
View File
@@ -7320,6 +7320,13 @@
resolved "https://registry.npmjs.org/@types/yarnpkg__lockfile/-/yarnpkg__lockfile-1.1.5.tgz#9639020e1fb65120a2f4387db8f1e8b63efdf229"
integrity sha512-8NYnGOctzsI4W0ApsP/BIHD/LnxpJ6XaGf2AZmz4EyDYJMxtprN4279dLNI1CPZcwC9H18qYcaFv4bXi0wmokg==
"@types/yauzl@^2.10.0":
version "2.10.0"
resolved "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.0.tgz#b3248295276cf8c6f153ebe6a9aba0c988cb2599"
integrity sha512-Cn6WYCm0tXv8p6k+A8PvbDG763EDpBoTzHdA+Q/MF6H3sapGjCm9NzoaJncJS9tUKSuCoDs9XHxYYsQDgxR6kw==
dependencies:
"@types/node" "*"
"@types/yauzl@^2.9.1":
version "2.9.2"
resolved "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.9.2.tgz#c48e5d56aff1444409e39fa164b0b4d4552a7b7a"
@@ -26322,7 +26329,7 @@ yarn-lock-check@^1.0.5:
yauzl@^2.10.0:
version "2.10.0"
resolved "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz#c7eb17c93e112cb1086fa6d8e51fb0667b79a5f9"
integrity sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk=
integrity sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==
dependencies:
buffer-crc32 "~0.2.3"
fd-slicer "~1.1.0"