GerritUrlReader: Implemented "readTree"

"readTree" has been implemented for the "GerritUrlReader". Gerrit have
a REST API's to download repo contents but there are a number of
limitations that makes it unusable.

This implementation works as follows:
* The project and branch is parsed from the url.
* The current revision is fetched from the Gerrit REST API.
* The revision string is used as "etag".
* If the etag has changed a temporary directory is created.
* The project is cloned to the temporary directory.
* The cloned content is read into a "Readable Stream".
* The temporary directory is removed.
* "readTree" returns a response using "fromTarArchive" as read from the
  temporary directory.

Also added an option to specify the base "cloneUrl" has been added to the
gerrit integration config.

Signed-off-by: Niklas Aronsson <niklasar@axis.com>

packages/backend-common/api-report.md

@@ -302,13 +302,19 @@ export type FromReadableArrayOptions = Array<{
 
 // @public
 export class GerritUrlReader implements UrlReader {
-  constructor(integration: GerritIntegration);
+  constructor(
+    integration: GerritIntegration,
+    deps: {
+      treeResponseFactory: ReadTreeResponseFactory;
+    },
+    workDir: string,
+  );
   // (undocumented)
   static factory: ReaderFactory;
   // (undocumented)
   read(url: string): Promise<Buffer>;
   // (undocumented)
-  readTree(): Promise<ReadTreeResponse>;
+  readTree(url: string, options?: ReadTreeOptions): Promise<ReadTreeResponse>;
   // (undocumented)
   readUrl(url: string, options?: ReadUrlOptions): Promise<ReadUrlResponse>;
   // (undocumented)

packages/backend-common/src/reading/GerritUrlReader.test.ts

@@ -14,24 +14,36 @@
  * limitations under the License.
  */
 
-import { rest } from 'msw';
-import { setupServer } from 'msw/node';
 import { setupRequestMockHandlers } from '@backstage/backend-test-utils';
 import { ConfigReader } from '@backstage/config';
-import { JsonObject } from '@backstage/types';
-import { getVoidLogger } from '../logging';
-import { DefaultReadTreeResponseFactory } from './tree';
-import { UrlReaderPredicateTuple } from './types';
+import { NotModifiedError, NotFoundError } from '@backstage/errors';
 import {
   GerritIntegration,
   readGerritIntegrationConfig,
 } from '@backstage/integration';
+import { JsonObject } from '@backstage/types';
+import { rest } from 'msw';
+import { setupServer } from 'msw/node';
+import mockFs from 'mock-fs';
+import fs from 'fs-extra';
+import path from 'path';
+import { getVoidLogger } from '../logging';
+import { UrlReaderPredicateTuple } from './types';
+import { DefaultReadTreeResponseFactory } from './tree';
 import { GerritUrlReader } from './GerritUrlReader';
 
 const treeResponseFactory = DefaultReadTreeResponseFactory.create({
   config: new ConfigReader({}),
 });
 
+jest.mock('../scm', () => ({
+  Git: {
+    fromAuth: () => ({
+      clone: jest.fn(() => Promise.resolve({})),
+    }),
+  },
+}));
+
 const gerritProcessor = new GerritUrlReader(
   new GerritIntegration(
     readGerritIntegrationConfig(
@@ -40,6 +52,8 @@ const gerritProcessor = new GerritUrlReader(
       }),
     ),
   ),
+  { treeResponseFactory },
+  '/tmp',
 );
 
 const createReader = (config: JsonObject): UrlReaderPredicateTuple[] => {
@@ -54,6 +68,10 @@ describe('GerritUrlReader', () => {
   const worker = setupServer();
   setupRequestMockHandlers(worker);
 
+  afterAll(() => {
+    jest.clearAllMocks();
+  });
+
   describe('reader factory', () => {
     it('creates a reader.', () => {
       const readers = createReader({
@@ -103,7 +121,7 @@ describe('GerritUrlReader', () => {
       expect(predicate(new URL('https://gerrit-review.com/path'))).toBe(false);
     });
 
-    it('returns false for  host.', () => {
+    it('returns false for host.', () => {
       expect(predicate(new URL('https://gerrit.com/path'))).toBe(true);
     });
   });
@@ -170,4 +188,104 @@ describe('GerritUrlReader', () => {
       );
     });
   });
+
+  describe('readTree', () => {
+    const branchAPIUrl =
+      'https://gerrit.com/projects/app%2Fweb/branches/master';
+    const branchAPIresponse = fs.readFileSync(
+      path.resolve(__dirname, '__fixtures__/gerrit/branch-info-response.txt'),
+    );
+    const treeUrl = 'https://gerrit.com/app/web/+/refs/heads/master/';
+    const etag = '52432507a70b677b5674b019c9a46b2e9f29d0a1';
+    const mkdocsContent = 'great content';
+    const mdContent = 'doc';
+
+    beforeEach(() => {
+      mockFs({
+        '/tmp/': mockFs.directory(),
+        '/tmp/gerrit-clone-123abc/repo/mkdocs.yml': mkdocsContent,
+        '/tmp/gerrit-clone-123abc/repo/docs/first.md': mdContent,
+      });
+      const spy = jest.spyOn(fs, 'mkdtemp');
+      spy.mockImplementation(() => '/tmp/gerrit-clone-123abc');
+    });
+
+    afterEach(() => {
+      mockFs.restore();
+      jest.clearAllMocks();
+    });
+
+    it('reads the wanted files correctly.', async () => {
+      worker.use(
+        rest.get(branchAPIUrl, (_, res, ctx) => {
+          return res(ctx.status(200), ctx.body(branchAPIresponse));
+        }),
+      );
+
+      const response = await gerritProcessor.readTree(treeUrl);
+
+      expect(response.etag).toBe(etag);
+
+      const files = await response.files();
+      expect(files.length).toBe(2);
+
+      const docsYaml = await files[0].content();
+      expect(docsYaml.toString()).toBe(mkdocsContent);
+
+      const mdFile = await files[1].content();
+      expect(mdFile.toString()).toBe(mdContent);
+    });
+
+    it('throws NotModifiedError for matching etags.', async () => {
+      worker.use(
+        rest.get(branchAPIUrl, (_, res, ctx) => {
+          return res(ctx.status(200), ctx.body(branchAPIresponse));
+        }),
+      );
+
+      await expect(gerritProcessor.readTree(treeUrl, { etag })).rejects.toThrow(
+        NotModifiedError,
+      );
+    });
+
+    it('throws NotFoundError if branch info not found.', async () => {
+      worker.use(
+        rest.get(branchAPIUrl, (_, res, ctx) => {
+          return res(ctx.status(404, 'Not found.'));
+        }),
+      );
+
+      await expect(gerritProcessor.readTree(treeUrl)).rejects.toThrow(
+        NotFoundError,
+      );
+    });
+
+    it('should throw on failures while getting branch info.', async () => {
+      worker.use(
+        rest.get(branchAPIUrl, (_, res, ctx) => {
+          return res(ctx.status(500, 'Error'));
+        }),
+      );
+
+      await expect(gerritProcessor.readTree(treeUrl)).rejects.toThrow(Error);
+    });
+
+    it('should returns wanted files with a subpath', async () => {
+      worker.use(
+        rest.get(branchAPIUrl, (_, res, ctx) => {
+          return res(ctx.status(200), ctx.body(branchAPIresponse));
+        }),
+      );
+
+      const response = await gerritProcessor.readTree(`${treeUrl}/docs`);
+
+      expect(response.etag).toBe(etag);
+
+      const files = await response.files();
+      expect(files.length).toBe(1);
+
+      const mdFile = await files[0].content();
+      expect(mdFile.toString()).toBe(mdContent);
+    });
+  });
 });

packages/backend-common/src/reading/GerritUrlReader.ts

@@ -14,16 +14,30 @@
  * limitations under the License.
  */
 
-import { NotFoundError } from '@backstage/errors';
+import { Git } from '../scm';
+import { NotFoundError, NotModifiedError } from '@backstage/errors';
 import {
   GerritIntegration,
+  getGerritCloneRepoUrl,
+  getGerritBranchApiUrl,
   getGerritFileContentsApiUrl,
   getGerritRequestOptions,
+  parseGerritJsonResponse,
+  parseGerritGitilesUrl,
 } from '@backstage/integration';
+import concatStream from 'concat-stream';
+import fs from 'fs-extra';
 import fetch, { Response } from 'node-fetch';
+import os from 'os';
+import { join as joinPath } from 'path';
+import tar from 'tar';
+import { pipeline as pipelineCb, Readable } from 'stream';
+import { promisify } from 'util';
 import {
   ReaderFactory,
+  ReadTreeOptions,
   ReadTreeResponse,
+  ReadTreeResponseFactory,
   ReadUrlOptions,
   ReadUrlResponse,
   SearchResponse,
@@ -31,6 +45,11 @@ import {
 } from './types';
 import { ScmIntegrations } from '@backstage/integration';
 
+const pipeline = promisify(pipelineCb);
+
+const createTemporaryDirectory = async (workDir: string): Promise<string> =>
+  await fs.mkdtemp(joinPath(workDir, '/gerrit-clone-'));
+
 /**
  * Implements a {@link UrlReader} for files in Gerrit.
  *
@@ -50,16 +69,22 @@ import { ScmIntegrations } from '@backstage/integration';
  * @public
  */
 export class GerritUrlReader implements UrlReader {
-  static factory: ReaderFactory = ({ config }) => {
+  static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
     const integrations = ScmIntegrations.fromConfig(config);
     if (!integrations.gerrit) {
       return [];
     }
+    const workDir =
+      config.getOptionalString('backend.workingDirectory') ?? os.tmpdir();
     return integrations.gerrit.list().map(integration => {
-      const reader = new GerritUrlReader(integration);
+      const reader = new GerritUrlReader(
+        integration,
+        { treeResponseFactory },
+        workDir,
+      );
       const predicate = (url: URL) => {
         const gitilesUrl = new URL(integration.config.gitilesBaseUrl!);
-        // If gitilesUrl is not specfified it will default to
+        // If gitilesUrl is not specified it will default to
         // "integration.config.host".
         return url.host === gitilesUrl.host;
       };
@@ -67,7 +92,11 @@ export class GerritUrlReader implements UrlReader {
     });
   };
 
-  constructor(private readonly integration: GerritIntegration) {}
+  constructor(
+    private readonly integration: GerritIntegration,
+    private readonly deps: { treeResponseFactory: ReadTreeResponseFactory },
+    private readonly workDir: string,
+  ) {}
 
   async read(url: string): Promise<Buffer> {
     const response = await this.readUrl(url);
@@ -109,8 +138,72 @@ export class GerritUrlReader implements UrlReader {
     );
   }
 
-  async readTree(): Promise<ReadTreeResponse> {
-    throw new Error('GerritReader does not implement readTree');
+  async readTree(
+    url: string,
+    options?: ReadTreeOptions,
+  ): Promise<ReadTreeResponse> {
+    const { filePath } = parseGerritGitilesUrl(this.integration.config, url);
+    const apiUrl = getGerritBranchApiUrl(this.integration.config, url);
+    let response: Response;
+    try {
+      response = await fetch(apiUrl, {
+        method: 'GET',
+        ...getGerritRequestOptions(this.integration.config),
+      });
+    } catch (e) {
+      throw new Error(`Unable to read branch state ${url}, ${e}`);
+    }
+
+    if (response.status === 404) {
+      throw new NotFoundError(`Not found: ${url}`);
+    }
+
+    if (!response.ok) {
+      throw new Error(
+        `${url} could not be read as ${apiUrl}, ${response.status} ${response.statusText}`,
+      );
+    }
+    const branchInfo = (await parseGerritJsonResponse(response as any)) as {
+      revision: string;
+    };
+    if (options?.etag === branchInfo.revision) {
+      throw new NotModifiedError();
+    }
+
+    const git = Git.fromAuth({
+      username: this.integration.config.username,
+      password: this.integration.config.password,
+    });
+    const tempDir = await createTemporaryDirectory(this.workDir);
+    const cloneUrl = getGerritCloneRepoUrl(this.integration.config, url);
+    try {
+      // The "fromTarArchive" function will strip the top level directory so
+      // an additional directory level is created when we clone.
+      await git.clone({
+        url: cloneUrl,
+        dir: joinPath(tempDir, 'repo'),
+        ref: branchInfo.revision,
+        depth: 1,
+      });
+
+      const data = await new Promise<Buffer>(async resolve => {
+        await pipeline(
+          tar.create({ cwd: tempDir }, ['']),
+          concatStream(resolve),
+        );
+      });
+      const tarArchive = Readable.from(data);
+      return await this.deps.treeResponseFactory.fromTarArchive({
+        stream: tarArchive as unknown as Readable,
+        subpath: filePath === '/' ? undefined : filePath,
+        etag: branchInfo.revision,
+        filter: options?.filter,
+      });
+    } catch (error) {
+      throw new Error(`Could not clone ${cloneUrl}: ${error}`);
+    } finally {
+      await fs.rm(tempDir, { recursive: true, force: true });
+    }
   }
 
   async search(): Promise<SearchResponse> {

packages/backend-common/src/reading/__fixtures__/gerrit/branch-info-response.txt

@@ -0,0 +1,2 @@
+)]}'
+{"web_links":[{"name":"browse","url":"https://gerrit.googlesource.com/app/web/+/refs/heads/master","target":"_blank"}],"ref":"refs/heads/master","revision":"52432507a70b677b5674b019c9a46b2e9f29d0a1"}