diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index cb2f653277..40beb3deca 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -41,6 +41,7 @@ "@backstage/integration": "^1.2.2-next.3", "@backstage/types": "^1.0.0", "@google-cloud/storage": "^6.0.0", + "@keyv/redis": "^2.2.3", "@manypkg/get-packages": "^1.1.3", "@octokit/rest": "^19.0.3", "@types/cors": "^2.8.6", @@ -48,6 +49,7 @@ "@types/express": "^4.17.6", "@types/luxon": "^2.0.4", "@types/webpack-env": "^1.15.2", + "@types/yauzl": "^2.10.0", "archiver": "^5.0.2", "aws-sdk": "^2.840.0", "base64-stream": "^1.0.0", @@ -64,7 +66,6 @@ "jose": "^4.6.0", "keyv": "^4.0.3", "keyv-memcache": "^1.2.5", - "@keyv/redis": "^2.2.3", "knex": "^2.0.0", "lodash": "^4.17.21", "logform": "^2.3.2", @@ -80,6 +81,7 @@ "tar": "^6.1.2", "unzipper": "^0.10.11", "winston": "^3.2.1", + "yauzl": "^2.10.0", "yn": "^4.0.0" }, "peerDependencies": { diff --git a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts index 501113d9f1..6a8e5ee621 100644 --- a/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/ZipArchiveResponse.ts @@ -15,25 +15,27 @@ */ import archiver from 'archiver'; +import unzipper2, { Entry } from 'yauzl'; import fs from 'fs-extra'; import platformPath from 'path'; import { Readable } from 'stream'; -import unzipper, { Entry } from 'unzipper'; +// import unzipper, { Entry } from 'unzipper'; import { ReadTreeResponse, ReadTreeResponseDirOptions, ReadTreeResponseFile, } from '../types'; import { streamToTimeoutPromise } from './util'; +import { zip } from 'lodash'; -const guardCorruptZipStream = (stream: Readable) => - streamToTimeoutPromise(stream, { - eventName: 'entry', - timeoutMs: 3000, - getError: (entry: Entry) => - new Error(`Timed out while unzipping ${entry.type}: ${entry.path}`), +const streamToBuffer = async (stream: Readable): Promise => { + const buffers: Buffer[] = []; + return new Promise((resolve, reject) => { + stream.on('data', (data: Buffer) => buffers.push(data)); + stream.on('error', reject); + stream.on('end', () => resolve(Buffer.concat(buffers))); }); - +}; /** * Wraps a zip archive stream into a tree response reader. */ @@ -76,15 +78,13 @@ export class ZipArchiveResponse implements ReadTreeResponse { private shouldBeIncluded(entry: Entry): boolean { if (this.subPath) { - if (!entry.path.startsWith(this.subPath)) { + if (!entry.fileName.startsWith(this.subPath)) { return false; } } if (this.filter) { - return this.filter(this.getInnerPath(entry.path), { - size: - (entry.vars as { uncompressedSize?: number }).uncompressedSize ?? - entry.vars.compressedSize, + return this.filter(this.getInnerPath(entry.fileName), { + size: entry.uncompressedSize, }); } return true; @@ -92,29 +92,36 @@ export class ZipArchiveResponse implements ReadTreeResponse { async files(): Promise { this.onlyOnce(); - const files = Array(); - const parseStream = this.stream - .pipe(unzipper.Parse()) - .on('entry', (entry: Entry) => { - if (entry.type === 'Directory') { - entry.resume(); + const buffer = await streamToBuffer(this.stream); + return await new Promise((resolve, reject) => { + unzipper2.fromBuffer(buffer, { lazyEntries: false }, (err, zipfile) => { + if (err) { + reject(err); return; } + zipfile.on('entry', async (entry: Entry) => { + // If it's not a directory, and it's included, then grab the contents of the file from the buffer + if (!/\/$/.test(entry.fileName) && this.shouldBeIncluded(entry)) { + files.push({ + path: this.getInnerPath(entry.fileName), + content: () => + new Promise((cResolve, cReject) => { + zipfile.openReadStream(entry, async (cError, readStream) => { + if (cError) { + return cReject(cError); + } + return cResolve(await streamToBuffer(readStream)); + }); + }), + }); + } + }); - if (this.shouldBeIncluded(entry)) { - files.push({ - path: this.getInnerPath(entry.path), - content: () => entry.buffer(), - }); - } else { - entry.autodrain(); - } + zipfile.once('end', () => resolve(files)); }); - await guardCorruptZipStream(parseStream); - - return files; + }); } async archive(): Promise { @@ -124,17 +131,32 @@ export class ZipArchiveResponse implements ReadTreeResponse { return this.stream; } + const buffer = await streamToBuffer(this.stream); const archive = archiver('zip'); - const parseStream = this.stream - .pipe(unzipper.Parse()) - .on('entry', (entry: Entry) => { - if (entry.type === 'File' && this.shouldBeIncluded(entry)) { - archive.append(entry, { name: this.getInnerPath(entry.path) }); - } else { - entry.autodrain(); + + await new Promise((resolve, reject) => { + unzipper2.fromBuffer(buffer, { lazyEntries: false }, (err, zipfile) => { + if (err) { + reject(err); + return; } + zipfile.on('entry', async (entry: Entry) => { + // If it's not a directory, and it's included, then grab the contents of the file from the buffer + if (!/\/$/.test(entry.fileName) && this.shouldBeIncluded(entry)) { + zipfile.openReadStream(entry, async (err2, readStream) => { + if (err2) { + reject(err2); + return; + } + archive.append(await streamToBuffer(readStream), { + name: this.getInnerPath(entry.fileName), + }); + }); + } + }); + zipfile.once('end', () => resolve()); }); - await guardCorruptZipStream(parseStream); + }); archive.finalize(); @@ -143,29 +165,28 @@ export class ZipArchiveResponse implements ReadTreeResponse { async dir(options?: ReadTreeResponseDirOptions): Promise { this.onlyOnce(); - const dir = options?.targetDir ?? (await fs.mkdtemp(platformPath.join(this.workDir, 'backstage-'))); - const parseStream = this.stream - .pipe(unzipper.Parse()) - .on('entry', async (entry: Entry) => { - // Ignore directory entries since we handle that with the file entries - // as a zip can have files with directories without directory entries - if (entry.type === 'File' && this.shouldBeIncluded(entry)) { - const entryPath = this.getInnerPath(entry.path); - const dirname = platformPath.dirname(entryPath); - if (dirname) { - await fs.mkdirp(platformPath.join(dir, dirname)); + return new Promise((resolve, reject) => { + const parseStream = this.stream + .pipe(unzipper.Parse()) + .on('entry', async (entry: Entry) => { + // Ignore directory entries since we handle that with the file entries + // as a zip can have files with directories without directory entries + if (entry.type === 'File' && this.shouldBeIncluded(entry)) { + const entryPath = this.getInnerPath(entry.path); + const dirname = platformPath.dirname(entryPath); + if (dirname) { + await fs.mkdirp(platformPath.join(dir, dirname)); + } + entry.pipe(fs.createWriteStream(platformPath.join(dir, entryPath))); + } else { + entry.autodrain(); } - entry.pipe(fs.createWriteStream(platformPath.join(dir, entryPath))); - } else { - entry.autodrain(); - } - }); - await guardCorruptZipStream(parseStream); - - return dir; + }) + .on('finish', () => resolve(dir)); + }); } } diff --git a/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml b/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml index 10d4a39ff7..c56e5a0d79 100644 --- a/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml +++ b/plugins/scaffolder-backend/fixtures/test-v1beta3/template.yaml @@ -8,28 +8,82 @@ spec: type: website owner: team-a parameters: - - name: Enter some stuff - description: Enter some stuff + - name: Repositories + description: Some repo properties: - inputString: + appRepoUrl: type: string - title: string input test - inputObject: - type: object - title: object input test - description: a little nested thing never hurt anyone right? - properties: - first: - type: string - title: first - second: - type: number - title: second + title: First Repository + ui:field: RepoUrlPicker + serviceRepoUrl: + type: string + title: First Repository + ui:field: RepoUrlPicker steps: - - id: debug - if: ${{ true === true }} - name: Debug - action: debug:log + # First get the app template folder, and template into ./app + - id: app_template + name: Fetch Skeleton + Template + action: fetch:template input: - message: ${{ parameters.inputString }} - extra: ${{ parameters.inputObject }} + url: ./skeleton + targetPath: ./app + copyWithoutRender: + - .github/* + values: + component_id: ${{ parameters.component_id }} + description: ${{ parameters.description }} + services_app_port: ${{ parameters.services_app_port }} + owner: ${{ parameters.owner }} + destination: ${{ parameters.appRepoUrl | parseRepoUrl }} + + # First then service into ./service + - id: app_service_config_template + name: Fetch App Servcie Config. Skeleton + Template + action: fetch:template + input: + url: https://github.com/my-org/helm-values-template + targetPath: ./service + copyWithoutRender: + - .github/* + values: + component_id: ${{ parameters.component_id }} + description: ${{ parameters.description }} + services_app_port: ${{ parameters.services_app_port }} + owner: ${{ parameters.owner }} + destination: ${{ parameters.serviceRepoUrl | parseRepoUrl }} + + # Publish the app + - id: publish_app + name: Publish App + action: publish:github + input: + sourcePath: ./app + allowedHosts: ['github.com'] + description: This is ${{ parameters.component_id }} + repoUrl: ${{ parameters.appRepoUrl }} + + # Publish the service + - id: publish_service + name: Publish Service + action: publish:github + input: + sourcePath: ./service + allowedHosts: ['github.com'] + description: This is ${{ parameters.component_id }} + repoUrl: ${{ parameters.serviceRepoUrl }} + + # Register the app + - id: register_app + name: Register Application + action: catalog:register + input: + repoContentsUrl: ${{ steps.publish_app.output.repoContentsUrl }} + catalogInfoPath: '/catalog-info.yaml' + + # Register the service + - id: register_service + name: Register Application + action: catalog:register + input: + repoContentsUrl: ${{ steps.publish_service.output.repoContentsUrl }} + catalogInfoPath: '/catalog-info.yaml' diff --git a/yarn.lock b/yarn.lock index cd607fcc68..9dfb3ea4ad 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7320,6 +7320,13 @@ resolved "https://registry.npmjs.org/@types/yarnpkg__lockfile/-/yarnpkg__lockfile-1.1.5.tgz#9639020e1fb65120a2f4387db8f1e8b63efdf229" integrity sha512-8NYnGOctzsI4W0ApsP/BIHD/LnxpJ6XaGf2AZmz4EyDYJMxtprN4279dLNI1CPZcwC9H18qYcaFv4bXi0wmokg== +"@types/yauzl@^2.10.0": + version "2.10.0" + resolved "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.0.tgz#b3248295276cf8c6f153ebe6a9aba0c988cb2599" + integrity sha512-Cn6WYCm0tXv8p6k+A8PvbDG763EDpBoTzHdA+Q/MF6H3sapGjCm9NzoaJncJS9tUKSuCoDs9XHxYYsQDgxR6kw== + dependencies: + "@types/node" "*" + "@types/yauzl@^2.9.1": version "2.9.2" resolved "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.9.2.tgz#c48e5d56aff1444409e39fa164b0b4d4552a7b7a" @@ -26322,7 +26329,7 @@ yarn-lock-check@^1.0.5: yauzl@^2.10.0: version "2.10.0" resolved "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz#c7eb17c93e112cb1086fa6d8e51fb0667b79a5f9" - integrity sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk= + integrity sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g== dependencies: buffer-crc32 "~0.2.3" fd-slicer "~1.1.0"