From f059eaf7550e642c8ca9e5ab1d7e1a5617ec9608 Mon Sep 17 00:00:00 2001 From: Sebastian Qvarfordt Date: Mon, 9 Nov 2020 16:28:51 +0100 Subject: [PATCH] readTree method on UrlReader (#2925) * Basic implementation of readTree on github and use it in techdocs * Added a UrlPreparer and cleaned up some minor things * Download entire archive and filter out the wanted files in readTree * Added test for dir output of tree reader * Fixed formatting * Fixed formatting * Some cleanups and fixes * Fix typing issues * Fix prettier * Fix prettier * Removed unused dependency * Fixed comments on PR * Prettier * Moved @types/fs-extra to devDeps * Fixed another PR comment * Added a test for getDocFilesFromRepository * Prettier --- packages/backend-common/package.json | 10 ++ .../src/reading/GithubUrlReader.test.ts | 80 ++++++++++++ .../src/reading/GithubUrlReader.ts | 93 +++++++++++++- .../src/reading/UrlReaderPredicateMux.ts | 29 ++++- .../src/reading/__fixtures__/repo.tar.gz | Bin 0 -> 232 bytes packages/backend-common/src/reading/index.ts | 2 +- packages/backend-common/src/reading/types.ts | 16 +++ packages/backend/src/plugins/techdocs.ts | 8 +- plugins/techdocs-backend/src/helpers.test.ts | 117 ++++++++++++++++++ plugins/techdocs-backend/src/helpers.ts | 34 ++++- .../techdocs-backend/src/service/helpers.ts | 2 +- .../src/techdocs/stages/prepare/index.ts | 1 + .../src/techdocs/stages/prepare/types.ts | 8 +- .../src/techdocs/stages/prepare/url.ts | 42 +++++++ yarn.lock | 16 ++- 15 files changed, 450 insertions(+), 8 deletions(-) create mode 100644 packages/backend-common/src/reading/__fixtures__/repo.tar.gz create mode 100644 plugins/techdocs-backend/src/helpers.test.ts create mode 100644 plugins/techdocs-backend/src/techdocs/stages/prepare/url.ts diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index c004eaac28..f76cb1757a 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -36,11 +36,13 @@ "@types/cors": "^2.8.6", "@types/express": "^4.17.6", "compression": "^1.7.4", + "concat-stream": "^2.0.0", "cors": "^2.8.5", "cross-fetch": "^3.0.6", "express": "^4.17.1", "express-prom-bundle": "^6.1.0", "express-promise-router": "^3.0.3", + "fs-extra": "^9.0.1", "git-url-parse": "^11.4.0", "helmet": "^4.0.0", "knex": "^0.21.6", @@ -51,6 +53,7 @@ "prom-client": "^12.0.0", "selfsigned": "^1.10.7", "stoppable": "^1.1.0", + "tar": "^6.0.5", "winston": "^3.2.1" }, "peerDependencies": { @@ -64,17 +67,24 @@ "devDependencies": { "@backstage/cli": "^0.2.0", "@types/compression": "^1.7.0", + "@types/concat-stream": "^1.6.0", + "@types/fs-extra": "^9.0.3", "@types/http-errors": "^1.6.3", "@types/minimist": "^1.2.0", + "@types/mock-fs": "^4.13.0", "@types/morgan": "^1.9.0", + "@types/recursive-readdir": "^2.2.0", "@types/stoppable": "^1.1.0", "@types/supertest": "^2.0.8", + "@types/tar": "^4.0.3", "@types/webpack-env": "^1.15.2", "@types/yaml": "^1.9.7", "get-port": "^5.1.1", "http-errors": "^1.7.3", "jest": "^26.0.1", + "mock-fs": "^4.13.0", "msw": "^0.21.2", + "recursive-readdir": "^2.2.2", "supertest": "^4.0.2" }, "files": [ diff --git a/packages/backend-common/src/reading/GithubUrlReader.test.ts b/packages/backend-common/src/reading/GithubUrlReader.test.ts index 8df464db9f..1e7326af2b 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.test.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.test.ts @@ -15,6 +15,9 @@ */ import { ConfigReader } from '@backstage/config'; +import { setupServer } from 'msw/node'; +import { msw } from '@backstage/test-utils'; +import { rest } from 'msw'; import { getApiRequestOptions, getApiUrl, @@ -24,6 +27,10 @@ import { ProviderConfig, readConfig, } from './GithubUrlReader'; +import fs from 'fs'; +import path from 'path'; +import mockfs from 'mock-fs'; +import recursive from 'recursive-readdir'; describe('GithubUrlReader', () => { describe('getApiRequestOptions', () => { @@ -230,4 +237,77 @@ describe('GithubUrlReader', () => { ); }); }); + + describe('readTree', () => { + const worker = setupServer(); + + msw.setupDefaultHandlers(worker); + + const repoBuffer = fs.readFileSync( + path.resolve('src', 'reading', '__fixtures__', 'repo.tar.gz'), + ); + + beforeEach(() => { + worker.use( + rest.get( + 'https://github.com/spotify/mock/archive/repo.tar.gz', + (_, res, ctx) => + res( + ctx.status(200), + ctx.set('Content-Type', 'application/x-gzip'), + ctx.body(repoBuffer), + ), + ), + ); + }); + + it('returns the wanted files from an archive', async () => { + const processor = new GithubUrlReader({ + host: 'github.com', + }); + + const response = await processor.readTree( + 'https://github.com/spotify/mock', + 'repo', + ['mkdocs.yml', 'docs'], + ); + + const files = response.files(); + + const mkDocsFile = await files[0].content(); + const indexMarkdownFile = await files[1].content(); + + expect(mkDocsFile.toString()).toBe('site_name: Test\n'); + expect(indexMarkdownFile.toString()).toBe('# Test\n'); + }); + + it('returns a folder path from an archive', async () => { + const processor = new GithubUrlReader({ + host: 'github.com', + }); + + const response = await processor.readTree( + 'https://github.com/spotify/mock', + 'repo', + ['mkdocs.yml', 'docs'], + ); + + mockfs(); + const directory = await response.dir('/tmp/fs'); + + const writtenToDirectory = fs.existsSync(directory); + const paths = await recursive(directory); + mockfs.restore(); + + expect(writtenToDirectory).toBe(true); + expect(paths.sort()).toEqual( + [ + '/tmp/fs/mock-repo/docs/index.md', + '/tmp/fs/mock-repo/mkdocs.yml', + ].sort(), + ); + + worker.resetHandlers(); + }); + }); }); diff --git a/packages/backend-common/src/reading/GithubUrlReader.ts b/packages/backend-common/src/reading/GithubUrlReader.ts index d12344ec91..906076a3c2 100644 --- a/packages/backend-common/src/reading/GithubUrlReader.ts +++ b/packages/backend-common/src/reading/GithubUrlReader.ts @@ -18,7 +18,12 @@ import { Config } from '@backstage/config'; import parseGitUri from 'git-url-parse'; import fetch from 'cross-fetch'; import { NotFoundError } from '../errors'; -import { ReaderFactory, UrlReader } from './types'; +import { ReaderFactory, ReadTreeResponse, UrlReader, File } from './types'; +import tar from 'tar'; +import fs from 'fs-extra'; +import concatStream from 'concat-stream'; +import path from 'path'; +import os from 'os'; /** * The configuration parameters for a single GitHub API provider. @@ -229,6 +234,92 @@ export class GithubUrlReader implements UrlReader { throw new Error(message); } + private async getRepositoryArchive( + repoUrl: string, + branchName: string, + ): Promise { + return fetch(new URL(`${repoUrl}/archive/${branchName}.tar.gz`).toString()); + } + + private async writeBufferToFile( + filePath: string, + content: Buffer, + ): Promise { + await fs.outputFile(filePath, content.toString()); + } + + async readTree( + repoUrl: string, + branchName: string, + paths: Array, + ): Promise { + const { name: repoName } = parseGitUri(repoUrl); + + const repoArchive = await this.getRepositoryArchive(repoUrl, branchName); + + const files: File[] = []; + return new Promise(resolve => { + const parser = new (tar.Parse as any)({ + filter: (path: string) => + !!paths.filter(file => { + return path.startsWith(`${repoName}-${branchName}/${file}`); + }).length, + onentry: (entry: tar.ReadEntry) => { + if (entry.type === 'Directory') { + entry.resume(); + return; + } + + const contentPromise: Promise = new Promise(res => { + entry.pipe(concatStream(res)); + }); + + files.push({ + path: entry.path, + content: () => contentPromise, + }); + + entry.resume(); + }, + }); + + // @ts-ignore Typescript doesn't consider .pipe a method on ReadableStream. Don't know why. + repoArchive.body?.pipe(parser).on('finish', () => { + resolve({ + files: () => { + return files; + }, + archive: () => { + return new Promise(resolve => + resolve(Buffer.from('Archive is not yet implemented')), + ); + }, + dir: (outDir: string | undefined) => { + const targetDirectory = + outDir || fs.mkdtempSync(path.join(os.tmpdir(), 'backstage-')); + + return new Promise((res, rej) => { + Promise.all( + files.map(async file => { + return this.writeBufferToFile( + `${targetDirectory}/${file.path}`, + await file.content(), + ); + }), + ) + .then(() => { + res(`${targetDirectory}/${repoName}-${branchName}`); + }) + .catch(err => { + rej(err); + }); + }); + }, + }); + }); + }); + } + toString() { const { host, token } = this.config; return `github{host=${host},authed=${Boolean(token)}}`; diff --git a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts index 7654cc8aac..74f94f0295 100644 --- a/packages/backend-common/src/reading/UrlReaderPredicateMux.ts +++ b/packages/backend-common/src/reading/UrlReaderPredicateMux.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { UrlReader, UrlReaderPredicateTuple } from './types'; +import { ReadTreeResponse, UrlReader, UrlReaderPredicateTuple } from './types'; type Options = { // UrlReader to fall back to if no other reader is matched @@ -53,6 +53,33 @@ export class UrlReaderPredicateMux implements UrlReader { throw new Error(`No reader found that could handle '${url}'`); } + readTree( + repoUrl: string, + branchName: string, + paths: Array, + ): Promise { + const parsed = new URL(repoUrl); + + for (const { predicate, reader } of this.readers) { + if (predicate(parsed)) { + if (reader.readTree) return reader.readTree(repoUrl, branchName, paths); + throw new Error( + `Trying to call readTree on UrlReader which does not support the feature.`, + ); + } + } + + if (this.fallback) { + if (this.fallback.readTree) + return this.fallback.readTree(repoUrl, branchName, paths); + throw new Error( + `Trying to call readTree on UrlReader which does not support the feature.`, + ); + } + + throw new Error(`No reader found that could handle '${repoUrl}'`); + } + toString() { return `predicateMux{readers=${this.readers .map(t => t.reader) diff --git a/packages/backend-common/src/reading/__fixtures__/repo.tar.gz b/packages/backend-common/src/reading/__fixtures__/repo.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..7a8e9902a24232a5f7130637a05304134abc3fec GIT binary patch literal 232 zcmb2|=3oE==C@Zbay1!XSu&B+beL(LRxe2^Idk!eV446aOJ+o+S-o>~??20b^`}#YbLB3%Cll?@@AuK( zf9m={{U?{EUwv`;^MMVwzTe;Tuda?M|Lgvh`?I~>7XNQAm|j0i{htM|=koGBZ|a}E f`_yk@`Q^XBu@$utzd*@``2}nqTB%DJG#D5FAn; + readTree?( + repoUrl: string, + branchName: string, + paths: Array, + ): Promise; }; export type UrlReaderPredicateTuple = { @@ -37,3 +42,14 @@ export type ReaderFactory = (options: { config: Config; logger: Logger; }) => UrlReaderPredicateTuple[]; + +export type File = { + path: string; + content(): Promise; +}; + +export type ReadTreeResponse = { + files(): File[]; + archive(): Promise; + dir(outDir?: string): Promise; +}; diff --git a/packages/backend/src/plugins/techdocs.ts b/packages/backend/src/plugins/techdocs.ts index dea31c763b..a9d6293c33 100644 --- a/packages/backend/src/plugins/techdocs.ts +++ b/packages/backend/src/plugins/techdocs.ts @@ -22,6 +22,7 @@ import { LocalPublish, TechdocsGenerator, CommonGitPreparer, + UrlPreparer, } from '@backstage/plugin-techdocs-backend'; import { PluginEnvironment } from '../types'; import Docker from 'dockerode'; @@ -30,20 +31,25 @@ export default async function createPlugin({ logger, config, discovery, + reader, }: PluginEnvironment) { const generators = new Generators(); const techdocsGenerator = new TechdocsGenerator(logger, config); generators.register('techdocs', techdocsGenerator); const preparers = new Preparers(); - const commonGitPreparer = new CommonGitPreparer(logger); const directoryPreparer = new DirectoryPreparer(logger); preparers.register('dir', directoryPreparer); + + const commonGitPreparer = new CommonGitPreparer(logger); preparers.register('github', commonGitPreparer); preparers.register('gitlab', commonGitPreparer); preparers.register('azure/api', commonGitPreparer); + const urlPreparer = new UrlPreparer(reader, logger); + preparers.register('url', urlPreparer); + const publisher = new LocalPublish(logger); const dockerClient = new Docker(); diff --git a/plugins/techdocs-backend/src/helpers.test.ts b/plugins/techdocs-backend/src/helpers.test.ts new file mode 100644 index 0000000000..8d4d69ba26 --- /dev/null +++ b/plugins/techdocs-backend/src/helpers.test.ts @@ -0,0 +1,117 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getDocFilesFromRepository } from './helpers'; +import { UrlReader, ReadTreeResponse } from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; + +describe('getDocFilesFromRepository', () => { + it('should take the directory from UrlReader.readTree and add the docs path when mkdocs.yml is in root', async () => { + class MockUrlReader implements UrlReader { + async read() { + return Buffer.from('mock'); + } + + async readTree(): Promise { + return { + dir: async () => { + return '/tmp/testfolder'; + }, + files: () => { + return []; + }, + archive: async () => { + return Buffer.from(''); + }, + }; + } + } + + const mockEntity: Entity = { + metadata: { + namespace: 'default', + annotations: { + 'backstage.io/techdocs-ref': + 'url:https://github.com/backstage/backstage/blob/master/mkdocs.yml', + }, + name: 'mytestcomponent', + description: 'A component for testing', + }, + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + spec: { + type: 'documentation', + lifecycle: 'experimental', + owner: 'testuser', + }, + }; + + const output = await getDocFilesFromRepository( + new MockUrlReader(), + mockEntity, + ); + + expect(output).toBe('/tmp/testfolder/.'); + }); + + it('should take the directory from UrlReader.readTree and add the docs path when mkdocs.yml is in a subfolder', async () => { + class MockUrlReader implements UrlReader { + async read() { + return Buffer.from('mock'); + } + + async readTree(): Promise { + return { + dir: async () => { + return '/tmp/testfolder'; + }, + files: () => { + return []; + }, + archive: async () => { + return Buffer.from(''); + }, + }; + } + } + + const mockEntity: Entity = { + metadata: { + namespace: 'default', + annotations: { + 'backstage.io/techdocs-ref': + 'url:https://github.com/backstage/backstage/blob/master/subfolder/mkdocs.yml', + }, + name: 'mytestcomponent', + description: 'A component for testing', + }, + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + spec: { + type: 'documentation', + lifecycle: 'experimental', + owner: 'testuser', + }, + }; + + const output = await getDocFilesFromRepository( + new MockUrlReader(), + mockEntity, + ); + + expect(output).toBe('/tmp/testfolder/subfolder'); + }); +}); diff --git a/plugins/techdocs-backend/src/helpers.ts b/plugins/techdocs-backend/src/helpers.ts index 4cbf46516f..f861d4375e 100644 --- a/plugins/techdocs-backend/src/helpers.ts +++ b/plugins/techdocs-backend/src/helpers.ts @@ -22,7 +22,7 @@ import fs from 'fs-extra'; import { getDefaultBranch } from './default-branch'; import { getGitRepoType, getTokenForGitRepo } from './git-auth'; import { Entity } from '@backstage/catalog-model'; -import { InputError } from '@backstage/backend-common'; +import { InputError, UrlReader } from '@backstage/backend-common'; import { RemoteProtocol } from './techdocs/stages/prepare/types'; import { Logger } from 'winston'; @@ -78,6 +78,7 @@ export const getLocationForEntity = ( case 'github': case 'gitlab': case 'azure/api': + case 'url': return { type, target }; case 'dir': if (path.isAbsolute(target)) return { type, target }; @@ -168,3 +169,34 @@ export const getLastCommitTimestamp = async ( return commit.date().getTime(); }; + +export const getDocFilesFromRepository = async ( + reader: UrlReader, + entity: Entity, +): Promise => { + const { target } = parseReferenceAnnotation( + 'backstage.io/techdocs-ref', + entity, + ); + + const { ref, filepath: mkdocsPath } = parseGitUrl(target); + + const docsRootPath = path.dirname(mkdocsPath); + const docsFolderPath = path.join(docsRootPath, 'docs'); + + if (reader.readTree) { + const readTreeResponse = await reader.readTree( + parseGitUrl(target).toString(), + ref, + [mkdocsPath, docsFolderPath], + ); + + const tmpDir = await readTreeResponse.dir(); + + return `${tmpDir}/${docsRootPath}`; + } + + throw new Error( + `No readTree method available on the UrlReader for ${target}`, + ); +}; diff --git a/plugins/techdocs-backend/src/service/helpers.ts b/plugins/techdocs-backend/src/service/helpers.ts index b308dc28af..3017a68461 100644 --- a/plugins/techdocs-backend/src/service/helpers.ts +++ b/plugins/techdocs-backend/src/service/helpers.ts @@ -103,7 +103,7 @@ export class DocsBuilder { const { type, target } = getLocationForEntity(this.entity); // Unless docs are stored locally - const nonAgeCheckTypes = ['dir', 'file']; + const nonAgeCheckTypes = ['dir', 'file', 'url']; if (!nonAgeCheckTypes.includes(type)) { const lastCommit = await getLastCommitTimestamp(target, this.logger); const storageTimeStamp = buildMetadataStorage.getTimestamp(); diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/index.ts b/plugins/techdocs-backend/src/techdocs/stages/prepare/index.ts index 46e1d84c69..6c73725a3a 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/prepare/index.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/prepare/index.ts @@ -15,5 +15,6 @@ */ export { DirectoryPreparer } from './dir'; export { CommonGitPreparer } from './commonGit'; +export { UrlPreparer } from './url'; export { Preparers } from './preparers'; export type { PreparerBuilder, PreparerBase } from './types'; diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/types.ts b/plugins/techdocs-backend/src/techdocs/stages/prepare/types.ts index 1dd091e107..97e75306c3 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/prepare/types.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/prepare/types.ts @@ -30,4 +30,10 @@ export type PreparerBuilder = { get(entity: Entity): PreparerBase; }; -export type RemoteProtocol = 'dir' | 'github' | 'gitlab' | 'file' | 'azure/api'; +export type RemoteProtocol = + | 'dir' + | 'github' + | 'gitlab' + | 'file' + | 'azure/api' + | 'url'; diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/url.ts b/plugins/techdocs-backend/src/techdocs/stages/prepare/url.ts new file mode 100644 index 0000000000..330db05aa4 --- /dev/null +++ b/plugins/techdocs-backend/src/techdocs/stages/prepare/url.ts @@ -0,0 +1,42 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Entity } from '@backstage/catalog-model'; +import { PreparerBase } from './types'; +import { getDocFilesFromRepository } from '../../../helpers'; + +import { Logger } from 'winston'; +import { UrlReader } from '@backstage/backend-common'; + +export class UrlPreparer implements PreparerBase { + private readonly logger: Logger; + private readonly reader: UrlReader; + + constructor(reader: UrlReader, logger: Logger) { + this.logger = logger; + this.reader = reader; + } + + async prepare(entity: Entity): Promise { + try { + return getDocFilesFromRepository(this.reader, entity); + } catch (error) { + this.logger.debug( + `Unable to fetch files for building docs ${error.message}`, + ); + throw error; + } + } +} diff --git a/yarn.lock b/yarn.lock index 83b96cebdb..391b71d8c9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4916,6 +4916,13 @@ dependencies: "@types/express" "*" +"@types/concat-stream@^1.6.0": + version "1.6.0" + resolved "https://registry.npmjs.org/@types/concat-stream/-/concat-stream-1.6.0.tgz#394dbe0bb5fee46b38d896735e8b68ef2390d00d" + integrity sha1-OU2+C7X+5Gs42JZzXoto7yOQ0A0= + dependencies: + "@types/node" "*" + "@types/connect-history-api-fallback@*": version "1.3.3" resolved "https://registry.npmjs.org/@types/connect-history-api-fallback/-/connect-history-api-fallback-1.3.3.tgz#4772b79b8b53185f0f4c9deab09236baf76ee3b4" @@ -5109,6 +5116,13 @@ dependencies: "@types/node" "*" +"@types/fs-extra@^9.0.3": + version "9.0.3" + resolved "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.3.tgz#9996e5cce993508c32325380b429f04a1327523e" + integrity sha512-NKdGoXLTFTRED3ENcfCsH8+ekV4gbsysanx2OPbstXVV6fZMgUCqTxubs6I9r7pbOJbFgVq1rpFtLURjKCZWUw== + dependencies: + "@types/node" "*" + "@types/git-url-parse@^9.0.0": version "9.0.0" resolved "https://registry.npmjs.org/@types/git-url-parse/-/git-url-parse-9.0.0.tgz#aac1315a44fa4ed5a52c3820f6c3c2fb79cbd12d" @@ -22456,7 +22470,7 @@ tar@^4, tar@^4.4.10, tar@^4.4.12, tar@^4.4.8: safe-buffer "^5.1.2" yallist "^3.0.3" -tar@^6.0.1, tar@^6.0.2: +tar@^6.0.1, tar@^6.0.2, tar@^6.0.5: version "6.0.5" resolved "https://registry.npmjs.org/tar/-/tar-6.0.5.tgz#bde815086e10b39f1dcd298e89d596e1535e200f" integrity sha512-0b4HOimQHj9nXNEAA7zWwMM91Zhhba3pspja6sQbgTpynOJf+bkjBnfybNYzbpLbnwXnbyB4LOREvlyXLkCHSg==