Merge branch 'master' of github.com:backstage/backstage into blam/isomorphic-git

* 'master' of github.com:backstage/backstage: (269 commits) address comments catalog-backend: flesh out and tweak the config schema Update chilled-pigs-destroy.md catalog-backend: start warning about usage of deprecated location types scripts: add script for migrating away from deprecated location types chore: fixing changeset prettyness Create good-hairs-sniff.md Update IconLinkVertical.tsx Update violet-sloths-reply.md Delete fuzzy-windows-cry.md Create violet-sloths-reply.md Added changeset build(deps): bump archiver from 5.0.2 to 5.1.0 build(deps): bump typescript-json-schema from 0.45.0 to 0.45.1 docs: Update lighthouse docs fix(app): listen to app.listen.host scaffolder-backend: clarify type detection error message docs: Update techdocs architecture features status scaffolder-backend: gitlab preparer uses integrations token core-api: ensure that routable extension components are discovered at boot ...
2020-12-21 16:31:15 +01:00
parent 54a19d4444 241bdb6215
commit ff0f7f89a2
551 changed files with 10869 additions and 6442 deletions
@@ -1,5 +1,110 @@
 # @backstage/backend-common

+## 0.4.1
+
+### Patch Changes
+
+- 1d1c2860f: Implement readTree on BitBucketUrlReader and getBitbucketDownloadUrl
+- 4eafdec4a: Introduce readTree method for GitLab URL Reader
+- Updated dependencies [1d1c2860f]
+- Updated dependencies [4eafdec4a]
+- Updated dependencies [178e09323]
+  - @backstage/integration@0.1.4
+
+## 0.4.0
+
+### Minor Changes
+
+- 12bbd748c: Removes the Prometheus integration from `backend-common`.
+
+  Rational behind this change is to keep the metrics integration of Backstage
+  generic. Instead of directly relying on Prometheus, Backstage will expose
+  metrics in a generic way. Integrators can then export the metrics in their
+  desired format. For example using Prometheus.
+
+  To keep the existing behavior, you need to integrate Prometheus in your
+  backend:
+
+  First, add a dependency on `express-prom-bundle` and `prom-client` to your backend.
+
+  ```diff
+  // packages/backend/package.json
+    "dependencies": {
+  +   "express-prom-bundle": "^6.1.0",
+  +   "prom-client": "^12.0.0",
+  ```
+
+  Then, add a handler for metrics and a simple instrumentation for the endpoints.
+
+  ```typescript
+  // packages/backend/src/metrics.ts
+  import { useHotCleanup } from '@backstage/backend-common';
+  import { RequestHandler } from 'express';
+  import promBundle from 'express-prom-bundle';
+  import prom from 'prom-client';
+  import * as url from 'url';
+
+  const rootRegEx = new RegExp('^/([^/]*)/.*');
+  const apiRegEx = new RegExp('^/api/([^/]*)/.*');
+
+  export function normalizePath(req: any): string {
+    const path = url.parse(req.originalUrl || req.url).pathname || '/';
+
+    // Capture /api/ and the plugin name
+    if (apiRegEx.test(path)) {
+      return path.replace(apiRegEx, '/api/$1');
+    }
+
+    // Only the first path segment at root level
+    return path.replace(rootRegEx, '/$1');
+  }
+
+  /**
+   * Adds a /metrics endpoint, register default runtime metrics and instrument the router.
+   */
+  export function metricsHandler(): RequestHandler {
+    // We can only initialize the metrics once and have to clean them up between hot reloads
+    useHotCleanup(module, () => prom.register.clear());
+
+    return promBundle({
+      includeMethod: true,
+      includePath: true,
+      // Using includePath alone is problematic, as it will include path labels with high
+      // cardinality (e.g. path params). Instead we would have to template them. However, this
+      // is difficult, as every backend plugin might use different routes. Instead we only take
+      // the first directory of the path, to have at least an idea how each plugin performs:
+      normalizePath,
+      promClient: { collectDefaultMetrics: {} },
+    });
+  }
+  ```
+
+  Last, extend your router configuration with the `metricsHandler`:
+
+  ```diff
+  +import { metricsHandler } from './metrics';
+
+  ...
+
+    const service = createServiceBuilder(module)
+      .loadConfig(config)
+      .addRouter('', await healthcheck(healthcheckEnv))
+  +   .addRouter('', metricsHandler())
+      .addRouter('/api', apiRouter);
+  ```
+
+  Your Prometheus metrics will be available at the `/metrics` endpoint.
+
+### Patch Changes
+
+- 38e24db00: Move the core url and auth logic to integration for the four major providers
+- Updated dependencies [38e24db00]
+- Updated dependencies [b8ecf6f48]
+- Updated dependencies [e3bd9fc2f]
+- Updated dependencies [e3bd9fc2f]
+  - @backstage/integration@0.1.3
+  - @backstage/config@0.1.2
+
 ## 0.3.3

 ### Patch Changes
@@ -1,7 +1,7 @@
 {
  "name": "@backstage/backend-common",
  "description": "Common functionality library for Backstage backends",
-  "version": "0.3.3",
+  "version": "0.4.1",
  "main": "src/index.ts",
  "types": "src/index.ts",
  "private": false,
@@ -30,9 +30,9 @@
  },
  "dependencies": {
    "@backstage/cli-common": "^0.1.1",
-    "@backstage/config": "^0.1.1",
+    "@backstage/config": "^0.1.2",
    "@backstage/config-loader": "^0.4.0",
-    "@backstage/integration": "^0.1.2",
+    "@backstage/integration": "^0.1.4",
    "@types/cors": "^2.8.6",
    "@types/express": "^4.17.6",
    "archiver": "^5.0.2",
@@ -43,7 +43,7 @@
    "express": "^4.17.1",
    "express-promise-router": "^3.0.3",
    "fs-extra": "^9.0.1",
-    "git-url-parse": "^11.4.0",
+    "git-url-parse": "^11.4.3",
    "helmet": "^4.0.0",
    "isomorphic-git": "^1.8.0",
    "knex": "^0.21.6",
@@ -66,8 +66,8 @@
    }
  },
  "devDependencies": {
-    "@backstage/cli": "^0.4.0",
-    "@backstage/test-utils": "^0.1.4",
+    "@backstage/cli": "^0.4.2",
+    "@backstage/test-utils": "^0.1.5",
    "@types/archiver": "^3.1.1",
    "@types/compression": "^1.7.0",
    "@types/concat-stream": "^1.6.0",
@@ -21,14 +21,6 @@ import { SingleConnectionDatabaseManager } from './SingleConnection';
 jest.mock('./connection');

 describe('SingleConnectionDatabaseManager', () => {
-  const createConfig = (data: any) =>
-    ConfigReader.fromConfigs([
-      {
-        context: '',
-        data,
-      },
-    ]);
-
  const defaultConfigOptions = {
    backend: {
      database: {
@@ -42,7 +34,7 @@ describe('SingleConnectionDatabaseManager', () => {
      },
    },
  };
-  const defaultConfig = () => createConfig(defaultConfigOptions);
+  const defaultConfig = () => new ConfigReader(defaultConfigOptions);

  // This is similar to the ts-jest `mocked` helper.
  const mocked = (f: Function) => f as jest.Mock;
@@ -18,19 +18,11 @@ import { ConfigReader } from '@backstage/config';
 import { createDatabaseClient } from './connection';

 describe('database connection', () => {
-  const createConfig = (data: any) =>
-    ConfigReader.fromConfigs([
-      {
-        context: '',
-        data,
-      },
-    ]);
-
  describe('createDatabaseClient', () => {
    it('returns a postgres connection', () => {
      expect(
        createDatabaseClient(
-          createConfig({
+          new ConfigReader({
            client: 'pg',
            connection: {
              host: 'acme',
@@ -46,7 +38,7 @@ describe('database connection', () => {
    it('returns an sqlite connection', () => {
      expect(
        createDatabaseClient(
-          createConfig({
+          new ConfigReader({
            client: 'sqlite3',
            connection: ':memory:',
          }),
@@ -57,7 +49,7 @@ describe('database connection', () => {
    it('tries to create a mysql connection as a passthrough', () => {
      expect(() =>
        createDatabaseClient(
-          createConfig({
+          new ConfigReader({
            client: 'mysql',
            connection: {
              host: '127.0.0.1',
@@ -73,7 +65,7 @@ describe('database connection', () => {
    it('accepts overrides', () => {
      expect(
        createDatabaseClient(
-          createConfig({
+          new ConfigReader({
            client: 'pg',
            connection: {
              host: 'acme',
@@ -94,7 +86,7 @@ describe('database connection', () => {
    it('throws an error without a client', () => {
      expect(() =>
        createDatabaseClient(
-          createConfig({
+          new ConfigReader({
            connection: '',
          }),
        ),
@@ -104,7 +96,7 @@ describe('database connection', () => {
    it('throws an error without a connection', () => {
      expect(() =>
        createDatabaseClient(
-          createConfig({
+          new ConfigReader({
            client: 'pg',
          }),
        ),
@@ -34,15 +34,7 @@ describe('postgres', () => {
    'postgresql://foo:bar@acme:5432/foodb';

  const createConfig = (connection: any): Config =>
-    ConfigReader.fromConfigs([
-      {
-        context: '',
-        data: {
-          client: 'pg',
-          connection,
-        },
-      },
-    ]);
+    new ConfigReader({ client: 'pg', connection });

  describe('buildPgDatabaseConfig', () => {
    it('builds a postgres config', () => {
@@ -22,15 +22,7 @@ import {

 describe('sqlite3', () => {
  const createConfig = (connection: any) =>
-    ConfigReader.fromConfigs([
-      {
-        context: '',
-        data: {
-          client: 'sqlite3',
-          connection,
-        },
-      },
-    ]);
+    new ConfigReader({ client: 'sqlite3', connection });

  describe('buildSqliteDatabaseConfig', () => {
    it('buidls a string connection', () => {
@@ -158,9 +158,7 @@ describe('AzureUrlReader', () => {

    it('returns the wanted files from an archive', async () => {
      const processor = new AzureUrlReader(
-        {
-          host: 'dev.azure.com',
-        },
+        { host: 'dev.azure.com' },
        { treeResponseFactory },
      );

@@ -14,15 +14,26 @@
 * limitations under the License.
 */

+import { ConfigReader } from '@backstage/config';
+import { msw } from '@backstage/test-utils';
+import fs from 'fs';
+import { rest } from 'msw';
+import { setupServer } from 'msw/node';
+import path from 'path';
 import { BitbucketUrlReader } from './BitbucketUrlReader';
+import { ReadTreeResponseFactory } from './tree';
+
+const treeResponseFactory = ReadTreeResponseFactory.create({
+  config: new ConfigReader({}),
+});

 describe('BitbucketUrlReader', () => {
  describe('implementation', () => {
    it('rejects unknown targets', async () => {
-      const processor = new BitbucketUrlReader({
-        host: 'bitbucket.org',
-        apiBaseUrl: 'https://api.bitbucket.org/2.0',
-      });
+      const processor = new BitbucketUrlReader(
+        { host: 'bitbucket.org', apiBaseUrl: 'https://api.bitbucket.org/2.0' },
+        { treeResponseFactory },
+      );
      await expect(
        processor.read('https://not.bitbucket.com/apa'),
      ).rejects.toThrow(
@@ -30,4 +41,141 @@ describe('BitbucketUrlReader', () => {
      );
    });
  });
+
+  describe('readTree', () => {
+    const worker = setupServer();
+    msw.setupDefaultHandlers(worker);
+
+    const repoBuffer = fs.readFileSync(
+      path.resolve(
+        'src',
+        'reading',
+        '__fixtures__',
+        'bitbucket-repo-with-commit-hash.zip',
+      ),
+    );
+
+    it('returns the wanted files from an archive', async () => {
+      worker.use(
+        rest.get(
+          'https://bitbucket.org/backstage/mock/get/master.zip',
+          (_, res, ctx) =>
+            res(
+              ctx.status(200),
+              ctx.set('Content-Type', 'application/zip'),
+              ctx.body(repoBuffer),
+            ),
+        ),
+        rest.get(
+          'https://api.bitbucket.org/2.0/repositories/backstage/mock/commits/master',
+          (_, res, ctx) =>
+            res(
+              ctx.status(200),
+              ctx.json({
+                values: [{ hash: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }],
+              }),
+            ),
+        ),
+      );
+
+      const processor = new BitbucketUrlReader(
+        { host: 'bitbucket.org', apiBaseUrl: 'https://api.bitbucket.org/2.0' },
+        { treeResponseFactory },
+      );
+
+      const response = await processor.readTree(
+        'https://bitbucket.org/backstage/mock/src/master',
+      );
+
+      const files = await response.files();
+
+      expect(files.length).toBe(2);
+      const indexMarkdownFile = await files[0].content();
+      const mkDocsFile = await files[1].content();
+
+      expect(indexMarkdownFile.toString()).toBe('# Test\n');
+      expect(mkDocsFile.toString()).toBe('site_name: Test\n');
+    });
+
+    it('uses private bitbucket host', async () => {
+      const privateBitbucketRepoBuffer = fs.readFileSync(
+        path.resolve(
+          'src',
+          'reading',
+          '__fixtures__',
+          'bitbucket-server-repo.zip',
+        ),
+      );
+      worker.use(
+        rest.get(
+          'https://api.bitbucket.mycompany.net/rest/api/1.0/projects/backstage/repos/mock/archive?format=zip&prefix=mock&path=docs',
+          (_, res, ctx) =>
+            res(
+              ctx.status(200),
+              ctx.set('Content-Type', 'application/zip'),
+              ctx.body(privateBitbucketRepoBuffer),
+            ),
+        ),
+      );
+
+      const processor = new BitbucketUrlReader(
+        {
+          host: 'bitbucket.mycompany.net',
+          apiBaseUrl: 'https://api.bitbucket.mycompany.net/rest/api/1.0',
+        },
+        { treeResponseFactory },
+      );
+
+      const response = await processor.readTree(
+        'https://bitbucket.mycompany.net/projects/backstage/repos/mock/browse/docs?at=some-branch',
+      );
+
+      const files = await response.files();
+
+      expect(files.length).toBe(1);
+      const indexMarkdownFile = await files[0].content();
+
+      expect(indexMarkdownFile.toString()).toBe('# Test\n');
+    });
+
+    it('returns the wanted files from an archive with a subpath', async () => {
+      worker.use(
+        rest.get(
+          'https://bitbucket.org/backstage/mock/get/master.zip',
+          (_, res, ctx) =>
+            res(
+              ctx.status(200),
+              ctx.set('Content-Type', 'application/zip'),
+              ctx.body(repoBuffer),
+            ),
+        ),
+        rest.get(
+          'https://api.bitbucket.org/2.0/repositories/backstage/mock/commits/master',
+          (_, res, ctx) =>
+            res(
+              ctx.status(200),
+              ctx.json({
+                values: [{ hash: '12ab34cd56ef78gh90ij12kl34mn56op78qr90st' }],
+              }),
+            ),
+        ),
+      );
+
+      const processor = new BitbucketUrlReader(
+        { host: 'bitbucket.org', apiBaseUrl: 'https://api.bitbucket.org/2.0' },
+        { treeResponseFactory },
+      );
+
+      const response = await processor.readTree(
+        'https://bitbucket.org/backstage/mock/src/master/docs',
+      );
+
+      const files = await response.files();
+
+      expect(files.length).toBe(1);
+      const indexMarkdownFile = await files[0].content();
+
+      expect(indexMarkdownFile.toString()).toBe('# Test\n');
+    });
+  });
 });
@@ -16,13 +16,23 @@

 import {
  BitbucketIntegrationConfig,
+  getBitbucketDefaultBranch,
+  getBitbucketDownloadUrl,
  getBitbucketFileFetchUrl,
  getBitbucketRequestOptions,
  readBitbucketIntegrationConfigs,
 } from '@backstage/integration';
 import fetch from 'cross-fetch';
+import parseGitUri from 'git-url-parse';
+import { Readable } from 'stream';
 import { NotFoundError } from '../errors';
-import { ReaderFactory, ReadTreeResponse, UrlReader } from './types';
+import { ReadTreeResponseFactory } from './tree';
+import {
+  ReaderFactory,
+  ReadTreeOptions,
+  ReadTreeResponse,
+  UrlReader,
+} from './types';

 /**
 * A processor that adds the ability to read files from Bitbucket v1 and v2 APIs, such as
@@ -30,19 +40,23 @@ import { ReaderFactory, ReadTreeResponse, UrlReader } from './types';
 */
 export class BitbucketUrlReader implements UrlReader {
  private readonly config: BitbucketIntegrationConfig;
+  private readonly treeResponseFactory: ReadTreeResponseFactory;

-  static factory: ReaderFactory = ({ config }) => {
+  static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
    const configs = readBitbucketIntegrationConfigs(
      config.getOptionalConfigArray('integrations.bitbucket') ?? [],
    );
    return configs.map(provider => {
-      const reader = new BitbucketUrlReader(provider);
+      const reader = new BitbucketUrlReader(provider, { treeResponseFactory });
      const predicate = (url: URL) => url.host === provider.host;
      return { reader, predicate };
    });
  };

-  constructor(config: BitbucketIntegrationConfig) {
+  constructor(
+    config: BitbucketIntegrationConfig,
+    deps: { treeResponseFactory: ReadTreeResponseFactory },
+  ) {
    const { host, apiBaseUrl, token, username, appPassword } = config;

    if (!apiBaseUrl) {
@@ -58,6 +72,7 @@ export class BitbucketUrlReader implements UrlReader {
    }

    this.config = config;
+    this.treeResponseFactory = deps.treeResponseFactory;
  }

  async read(url: string): Promise<Buffer> {
@@ -82,8 +97,39 @@ export class BitbucketUrlReader implements UrlReader {
    throw new Error(message);
  }

-  readTree(): Promise<ReadTreeResponse> {
-    throw new Error('BitbucketUrlReader does not implement readTree');
+  async readTree(
+    url: string,
+    options?: ReadTreeOptions,
+  ): Promise<ReadTreeResponse> {
+    const gitUrl: parseGitUri.GitUrl = parseGitUri(url);
+    const { name: repoName, owner: project, resource, filepath } = gitUrl;
+
+    const isHosted = resource === 'bitbucket.org';
+
+    const downloadUrl = await getBitbucketDownloadUrl(url, this.config);
+    const response = await fetch(
+      downloadUrl,
+      getBitbucketRequestOptions(this.config),
+    );
+    if (!response.ok) {
+      const message = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`;
+      if (response.status === 404) {
+        throw new NotFoundError(message);
+      }
+      throw new Error(message);
+    }
+
+    let folderPath = `${project}-${repoName}`;
+    if (isHosted) {
+      const lastCommitShortHash = await this.getLastCommitShortHash(url);
+      folderPath = `${project}-${repoName}-${lastCommitShortHash}`;
+    }
+
+    return this.treeResponseFactory.fromZipArchive({
+      stream: (response.body as unknown) as Readable,
+      path: `${folderPath}/${filepath}`,
+      filter: options?.filter,
+    });
  }

  toString() {
@@ -94,4 +140,37 @@ export class BitbucketUrlReader implements UrlReader {
    }
    return `bitbucket{host=${host},authed=${authed}}`;
  }
+
+  private async getLastCommitShortHash(url: string): Promise<String> {
+    const { name: repoName, owner: project, ref } = parseGitUri(url);
+
+    let branch = ref;
+    if (!branch) {
+      branch = await getBitbucketDefaultBranch(url, this.config);
+    }
+    const commitsApiUrl = `${this.config.apiBaseUrl}/repositories/${project}/${repoName}/commits/${branch}`;
+
+    const commitsResponse = await fetch(
+      commitsApiUrl,
+      getBitbucketRequestOptions(this.config),
+    );
+    if (!commitsResponse.ok) {
+      const message = `Failed to retrieve commits from ${commitsApiUrl}, ${commitsResponse.status} ${commitsResponse.statusText}`;
+      if (commitsResponse.status === 404) {
+        throw new NotFoundError(message);
+      }
+      throw new Error(message);
+    }
+
+    const commits = await commitsResponse.json();
+    if (
+      commits &&
+      commits.values &&
+      commits.values.length > 0 &&
+      commits.values[0].hash
+    ) {
+      return commits.values[0].hash.substring(0, 12);
+    }
+    throw new Error(`Failed to read response from ${commitsApiUrl}`);
+  }
 }
@@ -14,12 +14,14 @@
 * limitations under the License.
 */

+import { ConfigReader } from '@backstage/config';
+import { msw } from '@backstage/test-utils';
+import fs from 'fs';
 import { rest } from 'msw';
 import { setupServer } from 'msw/node';
-import { ConfigReader } from '@backstage/config';
+import path from 'path';
 import { getVoidLogger } from '../logging';
 import { GitlabUrlReader } from './GitlabUrlReader';
-import { msw } from '@backstage/test-utils';
 import { ReadTreeResponseFactory } from './tree';

 const logger = getVoidLogger();
@@ -30,105 +32,207 @@ const treeResponseFactory = ReadTreeResponseFactory.create({

 describe('GitlabUrlReader', () => {
  const worker = setupServer();
-
  msw.setupDefaultHandlers(worker);

-  beforeEach(() => {
-    worker.use(
-      rest.get('*/api/v4/projects/:name', (_, res, ctx) =>
-        res(ctx.status(200), ctx.json({ id: 12345 })),
-      ),
-      rest.get('*', (req, res, ctx) =>
-        res(
-          ctx.status(200),
-          ctx.json({
-            url: req.url.toString(),
-            headers: req.headers.getAllHeaders(),
-          }),
+  describe('implementation', () => {
+    beforeEach(() => {
+      worker.use(
+        rest.get('*/api/v4/projects/:name', (_, res, ctx) =>
+          res(ctx.status(200), ctx.json({ id: 12345 })),
        ),
-      ),
-    );
-  });
-
-  const createConfig = (token?: string) =>
-    new ConfigReader(
-      {
-        integrations: { gitlab: [{ host: 'gitlab.com', token }] },
-      },
-      'test-config',
-    );
-
-  it.each([
-    // Project URLs
-    {
-      url:
-        'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml',
-      config: createConfig(),
-      response: expect.objectContaining({
-        url:
-          'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
-        headers: expect.objectContaining({
-          'private-token': '',
-        }),
-      }),
-    },
-    {
-      url:
-        'https://gitlab.example.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml',
-      config: createConfig('0123456789'),
-      response: expect.objectContaining({
-        url:
-          'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
-        headers: expect.objectContaining({
-          'private-token': '0123456789',
-        }),
-      }),
-    },
-    {
-      url:
-        'https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/to/file.yaml', // Repo not in subgroup
-      config: createConfig(),
-      response: expect.objectContaining({
-        url:
-          'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
-      }),
-    },
-
-    // Raw URLs
-    {
-      url: 'https://gitlab.example.com/a/b/blob/master/c.yaml',
-      config: createConfig(),
-      response: expect.objectContaining({
-        url: 'https://gitlab.example.com/a/b/raw/master/c.yaml',
-      }),
-    },
-  ])('should handle happy path %#', async ({ url, config, response }) => {
-    const [{ reader }] = GitlabUrlReader.factory({
-      config,
-      logger,
-      treeResponseFactory,
+        rest.get('*', (req, res, ctx) =>
+          res(
+            ctx.status(200),
+            ctx.json({
+              url: req.url.toString(),
+              headers: req.headers.getAllHeaders(),
+            }),
+          ),
+        ),
+      );
    });

-    const data = await reader.read(url);
-    const res = await JSON.parse(data.toString('utf-8'));
-    expect(res).toEqual(response);
-  });
+    const createConfig = (token?: string) =>
+      new ConfigReader(
+        {
+          integrations: { gitlab: [{ host: 'gitlab.com', token }] },
+        },
+        'test-config',
+      );

-  it.each([
-    {
-      url: '',
-      config: createConfig(''),
-      error:
-        "Invalid type in config for key 'integrations.gitlab[0].token' in 'test-config', got empty-string, wanted string",
-    },
-  ])('should handle error path %#', async ({ url, config, error }) => {
-    await expect(async () => {
+    it.each([
+      // Project URLs
+      {
+        url:
+          'https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml',
+        config: createConfig(),
+        response: expect.objectContaining({
+          url:
+            'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
+          headers: expect.objectContaining({
+            'private-token': '',
+          }),
+        }),
+      },
+      {
+        url:
+          'https://gitlab.example.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/my/path/to/file.yaml',
+        config: createConfig('0123456789'),
+        response: expect.objectContaining({
+          url:
+            'https://gitlab.example.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
+          headers: expect.objectContaining({
+            'private-token': '0123456789',
+          }),
+        }),
+      },
+      {
+        url:
+          'https://gitlab.com/groupA/teams/teamA/repoA/-/blob/branch/my/path/to/file.yaml', // Repo not in subgroup
+        config: createConfig(),
+        response: expect.objectContaining({
+          url:
+            'https://gitlab.com/api/v4/projects/12345/repository/files/my%2Fpath%2Fto%2Ffile.yaml/raw?ref=branch',
+        }),
+      },
+
+      // Raw URLs
+      {
+        url: 'https://gitlab.example.com/a/b/blob/master/c.yaml',
+        config: createConfig(),
+        response: expect.objectContaining({
+          url: 'https://gitlab.example.com/a/b/raw/master/c.yaml',
+        }),
+      },
+    ])('should handle happy path %#', async ({ url, config, response }) => {
      const [{ reader }] = GitlabUrlReader.factory({
        config,
        logger,
        treeResponseFactory,
      });
-      await reader.read(url);
-    }).rejects.toThrow(error);
+
+      const data = await reader.read(url);
+      const res = await JSON.parse(data.toString('utf-8'));
+      expect(res).toEqual(response);
+    });
+
+    it.each([
+      {
+        url: '',
+        config: createConfig(''),
+        error:
+          "Invalid type in config for key 'integrations.gitlab[0].token' in 'test-config', got empty-string, wanted string",
+      },
+    ])('should handle error path %#', async ({ url, config, error }) => {
+      await expect(async () => {
+        const [{ reader }] = GitlabUrlReader.factory({
+          config,
+          logger,
+          treeResponseFactory,
+        });
+        await reader.read(url);
+      }).rejects.toThrow(error);
+    });
+  });
+
+  describe('readTree', () => {
+    const repoBuffer = fs.readFileSync(
+      path.resolve('src', 'reading', '__fixtures__', 'repo.zip'),
+    );
+
+    beforeEach(() => {
+      worker.use(
+        rest.get(
+          'https://gitlab.com/backstage/mock/-/archive/repo/mock-repo.zip',
+          (_, res, ctx) =>
+            res(
+              ctx.status(200),
+              ctx.set('Content-Type', 'application/zip'),
+              ctx.body(repoBuffer),
+            ),
+        ),
+      );
+    });
+
+    it('returns the wanted files from an archive', async () => {
+      const processor = new GitlabUrlReader(
+        { host: 'gitlab.com' },
+        { treeResponseFactory },
+      );
+
+      const response = await processor.readTree(
+        'https://gitlab.com/backstage/mock/tree/repo',
+      );
+
+      const files = await response.files();
+      expect(files.length).toBe(2);
+
+      const indexMarkdownFile = await files[0].content();
+      const mkDocsFile = await files[1].content();
+
+      expect(mkDocsFile.toString()).toBe('site_name: Test\n');
+      expect(indexMarkdownFile.toString()).toBe('# Test\n');
+    });
+
+    it('returns the wanted files from hosted gitlab', async () => {
+      worker.use(
+        rest.get(
+          'https://git.mycompany.com/backstage/mock/-/archive/repo/mock-repo.zip',
+          (_, res, ctx) =>
+            res(
+              ctx.status(200),
+              ctx.set('Content-Type', 'application/zip'),
+              ctx.body(repoBuffer),
+            ),
+        ),
+      );
+
+      const processor = new GitlabUrlReader(
+        { host: 'git.mycompany.com' },
+        { treeResponseFactory },
+      );
+
+      const response = await processor.readTree(
+        'https://git.mycompany.com/backstage/mock/tree/repo/docs',
+      );
+
+      const files = await response.files();
+
+      expect(files.length).toBe(1);
+      const indexMarkdownFile = await files[0].content();
+
+      expect(indexMarkdownFile.toString()).toBe('# Test\n');
+    });
+
+    it('throws an error when branch is not specified', async () => {
+      const processor = new GitlabUrlReader(
+        { host: 'gitlab.com' },
+        { treeResponseFactory },
+      );
+
+      await expect(
+        processor.readTree('https://gitlab.com/backstage/mock'),
+      ).rejects.toThrow(
+        'GitLab URL must contain a branch to be able to fetch its tree',
+      );
+    });
+
+    it('returns the wanted files from an archive with a subpath', async () => {
+      const processor = new GitlabUrlReader(
+        { host: 'gitlab.com' },
+        { treeResponseFactory },
+      );
+
+      const response = await processor.readTree(
+        'https://gitlab.com/backstage/mock/tree/repo/docs',
+      );
+
+      const files = await response.files();
+
+      expect(files.length).toBe(1);
+      const indexMarkdownFile = await files[0].content();
+
+      expect(indexMarkdownFile.toString()).toBe('# Test\n');
+    });
  });
 });
@@ -21,22 +21,37 @@ import {
  readGitLabIntegrationConfigs,
 } from '@backstage/integration';
 import fetch from 'cross-fetch';
-import { NotFoundError } from '../errors';
-import { ReaderFactory, ReadTreeResponse, UrlReader } from './types';
+import { InputError, NotFoundError } from '../errors';
+import { ReadTreeResponseFactory } from './tree';
+import {
+  ReaderFactory,
+  ReadTreeOptions,
+  ReadTreeResponse,
+  UrlReader,
+} from './types';
+import parseGitUri from 'git-url-parse';
+import { Readable } from 'stream';

 export class GitlabUrlReader implements UrlReader {
-  static factory: ReaderFactory = ({ config }) => {
+  private readonly treeResponseFactory: ReadTreeResponseFactory;
+
+  static factory: ReaderFactory = ({ config, treeResponseFactory }) => {
    const configs = readGitLabIntegrationConfigs(
      config.getOptionalConfigArray('integrations.gitlab') ?? [],
    );
    return configs.map(options => {
-      const reader = new GitlabUrlReader(options);
+      const reader = new GitlabUrlReader(options, { treeResponseFactory });
      const predicate = (url: URL) => url.host === options.host;
      return { reader, predicate };
    });
  };

-  constructor(private readonly options: GitLabIntegrationConfig) {}
+  constructor(
+    private readonly options: GitLabIntegrationConfig,
+    deps: { treeResponseFactory: ReadTreeResponseFactory },
+  ) {
+    this.treeResponseFactory = deps.treeResponseFactory;
+  }

  async read(url: string): Promise<Buffer> {
    const builtUrl = await getGitLabFileFetchUrl(url, this.options);
@@ -59,8 +74,45 @@ export class GitlabUrlReader implements UrlReader {
    throw new Error(message);
  }

-  readTree(): Promise<ReadTreeResponse> {
-    throw new Error('GitlabUrlReader does not implement readTree');
+  async readTree(
+    url: string,
+    options?: ReadTreeOptions,
+  ): Promise<ReadTreeResponse> {
+    const {
+      name: repoName,
+      ref,
+      protocol,
+      resource,
+      full_name,
+      filepath,
+    } = parseGitUri(url);
+
+    if (!ref) {
+      throw new InputError(
+        'GitLab URL must contain a branch to be able to fetch its tree',
+      );
+    }
+
+    const archive = `${protocol}://${resource}/${full_name}/-/archive/${ref}/${repoName}-${ref}.zip`;
+    const response = await fetch(
+      archive,
+      getGitLabRequestOptions(this.options),
+    );
+    if (!response.ok) {
+      const msg = `Failed to read tree from ${url}, ${response.status} ${response.statusText}`;
+      if (response.status === 404) {
+        throw new NotFoundError(msg);
+      }
+      throw new Error(msg);
+    }
+
+    const path = filepath ? `${repoName}-${ref}/${filepath}/` : '';
+
+    return this.treeResponseFactory.fromZipArchive({
+      stream: (response.body as unknown) as Readable,
+      path,
+      filter: options?.filter,
+    });
  }

  toString() {
@@ -52,7 +52,7 @@ const DEFAULT_CSP = {
  'frame-ancestors': ["'self'"],
  'img-src': ["'self'", 'data:'],
  'object-src': ["'none'"],
-  'script-src': ["'self'"],
+  'script-src': ["'self'", "'unsafe-eval'"],
  'script-src-attr': ["'none'"],
  'style-src': ["'self'", 'https:', "'unsafe-inline'"],
  'upgrade-insecure-requests': [] as string[],
@@ -20,9 +20,7 @@ import { readCspOptions } from './config';
 describe('config', () => {
  describe('readCspOptions', () => {
    it('reads valid values', () => {
-      const config = ConfigReader.fromConfigs([
-        { context: '', data: { csp: { key: ['value'] } } },
-      ]);
+      const config = new ConfigReader({ csp: { key: ['value'] } });
      expect(readCspOptions(config)).toEqual(
        expect.objectContaining({
          key: ['value'],
@@ -31,9 +29,7 @@ describe('config', () => {
    });

    it('accepts false', () => {
-      const config = ConfigReader.fromConfigs([
-        { context: '', data: { csp: { key: false } } },
-      ]);
+      const config = new ConfigReader({ csp: { key: false } });
      expect(readCspOptions(config)).toEqual(
        expect.objectContaining({
          key: false,
@@ -42,9 +38,7 @@ describe('config', () => {
    });

    it('rejects invalid value types', () => {
-      const config = ConfigReader.fromConfigs([
-        { context: '', data: { csp: { key: [4] } } },
-      ]);
+      const config = new ConfigReader({ csp: { key: [4] } });
      expect(() => readCspOptions(config)).toThrow(/wanted string-array/);
    });
  });