Merge pull request #5494 from goober/feature/expose-bitbucket-parser

[Catalog] Expose BitbucketRepositoryParser

.changeset/khaki-rice-accept.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Expose `BitbucketRepositoryParser` introduced in [#5295](https://github.com/backstage/backstage/pull/5295)

plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.test.ts

@@ -17,21 +17,73 @@ import { getVoidLogger } from '@backstage/backend-common';
 import { BitbucketDiscoveryProcessor } from './BitbucketDiscoveryProcessor';
 import { ConfigReader } from '@backstage/config';
 import { LocationSpec } from '@backstage/catalog-model';
-import {
-  BitbucketClient,
-  BitbucketRepositoryParser,
-  PagedResponse,
-} from './bitbucket';
+import { BitbucketRepositoryParser, PagedResponse } from './bitbucket';
 import { results } from './index';
+import { RequestHandler, rest } from 'msw';
+import { setupServer } from 'msw/node';
 
-function pagedResponse(values: any): PagedResponse<any> {
-  return {
-    values: values,
-    isLastPage: true,
-  } as PagedResponse<any>;
+const server = setupServer();
+
+function setupStubs(projects: any[]) {
+  function pagedResponse(values: any): PagedResponse<any> {
+    return {
+      values: values,
+      isLastPage: true,
+    } as PagedResponse<any>;
+  }
+
+  function stubbedProject(
+    project: string,
+    repos: string[],
+  ): RequestHandler<any, any> {
+    return rest.get(
+      `https://bitbucket.mycompany.com/api/rest/1.0/projects/${project}/repos`,
+      (_, res, ctx) => {
+        const response = [];
+        for (const repo of repos) {
+          response.push({
+            slug: repo,
+            links: {
+              self: [
+                {
+                  href: `https://bitbucket.mycompany.com/projects/${project}/repos/${repo}/browse`,
+                },
+              ],
+            },
+          });
+        }
+        return res(ctx.json(pagedResponse(response)));
+      },
+    );
+  }
+
+  server.use(
+    rest.get(
+      `https://bitbucket.mycompany.com/api/rest/1.0/projects`,
+      (_, res, ctx) => {
+        return res(
+          ctx.json(
+            pagedResponse(
+              projects.map(p => {
+                return { key: p.key };
+              }),
+            ),
+          ),
+        );
+      },
+    ),
+  );
+
+  for (const project of projects) {
+    server.use(stubbedProject(project.key, project.repos));
+  }
 }
 
 describe('BitbucketDiscoveryProcessor', () => {
+  beforeAll(() => server.listen());
+  afterEach(() => server.resetHandlers());
+  afterAll(() => server.close());
+
   afterEach(() => jest.resetAllMocks());
 
   describe('reject unrelated entries', () => {
@@ -81,58 +133,29 @@ describe('BitbucketDiscoveryProcessor', () => {
     const processor = BitbucketDiscoveryProcessor.fromConfig(
       new ConfigReader({
         integrations: {
-          bitbucket: [{ host: 'bitbucket.mycompany.com', token: 'blob' }],
+          bitbucket: [
+            {
+              host: 'bitbucket.mycompany.com',
+              token: 'blob',
+              apiBaseUrl: 'https://bitbucket.mycompany.com/api/rest/1.0',
+            },
+          ],
         },
       }),
       { logger: getVoidLogger() },
     );
 
     it('output all repositories', async () => {
+      setupStubs([
+        { key: 'backstage', repos: ['backstage'] },
+        { key: 'demo', repos: ['demo'] },
+      ]);
       const location: LocationSpec = {
         type: 'bitbucket-discovery',
         target:
           'https://bitbucket.mycompany.com/projects/*/repos/*/catalog.yaml',
       };
 
-      jest
-        .spyOn(BitbucketClient.prototype, 'listProjects')
-        .mockResolvedValue(
-          pagedResponse([{ key: 'backstage' }, { key: 'demo' }]),
-        );
-      jest
-        .spyOn(BitbucketClient.prototype, 'listRepositories')
-        .mockResolvedValueOnce(
-          pagedResponse([
-            {
-              slug: 'backstage',
-              links: {
-                self: [
-                  {
-                    href:
-                      'https://bitbucket.mycompany.com/projects/backstage/repos/backstage/browse',
-                  },
-                ],
-              },
-            },
-          ]),
-        );
-      jest
-        .spyOn(BitbucketClient.prototype, 'listRepositories')
-        .mockResolvedValueOnce(
-          pagedResponse([
-            {
-              slug: 'demo',
-              links: {
-                self: [
-                  {
-                    href:
-                      'https://bitbucket.mycompany.com/projects/demo/repos/demo/browse',
-                  },
-                ],
-              },
-            },
-          ]),
-        );
       const emitter = jest.fn();
 
       await processor.readLocation(location, false, emitter);
@@ -158,44 +181,16 @@ describe('BitbucketDiscoveryProcessor', () => {
     });
 
     it('output repositories with wildcards', async () => {
+      setupStubs([
+        { key: 'backstage', repos: ['backstage', 'techdocs-cli'] },
+        { key: 'demo', repos: ['demo'] },
+      ]);
       const location: LocationSpec = {
         type: 'bitbucket-discovery',
         target:
           'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-*/catalog.yaml',
       };
 
-      jest
-        .spyOn(BitbucketClient.prototype, 'listProjects')
-        .mockResolvedValue(pagedResponse([{ key: 'backstage' }]));
-      jest
-        .spyOn(BitbucketClient.prototype, 'listRepositories')
-        .mockResolvedValueOnce(
-          pagedResponse([
-            { slug: 'backstage' },
-            {
-              slug: 'techdocs-cli',
-              links: {
-                self: [
-                  {
-                    href:
-                      'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-cli/browse',
-                  },
-                ],
-              },
-            },
-            {
-              slug: 'techdocs-container',
-              links: {
-                self: [
-                  {
-                    href:
-                      'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse',
-                  },
-                ],
-              },
-            },
-          ]),
-        );
       const emitter = jest.fn();
       await processor.readLocation(location, false, emitter);
 
@@ -208,46 +203,15 @@ describe('BitbucketDiscoveryProcessor', () => {
         },
         optional: true,
       });
-      expect(emitter).toHaveBeenCalledWith({
-        type: 'location',
-        location: {
-          type: 'url',
-          target:
-            'https://bitbucket.mycompany.com/projects/backstage/repos/techdocs-container/browse/catalog.yaml',
-        },
-        optional: true,
-      });
     });
     it('filter unrelated repositories', async () => {
+      setupStubs([{ key: 'backstage', repos: ['test', 'abctest', 'testxyz'] }]);
       const location: LocationSpec = {
         type: 'bitbucket-discovery',
         target:
           'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml',
       };
 
-      jest
-        .spyOn(BitbucketClient.prototype, 'listProjects')
-        .mockResolvedValue(pagedResponse([{ key: 'backstage' }]));
-      jest
-        .spyOn(BitbucketClient.prototype, 'listRepositories')
-        .mockResolvedValue(
-          pagedResponse([
-            { slug: 'abstest' },
-            { slug: 'testxyz' },
-            {
-              slug: 'test',
-              links: {
-                self: [
-                  {
-                    href:
-                      'https://bitbucket.mycompany.com/projects/backstage/repos/test',
-                  },
-                ],
-              },
-            },
-          ]),
-        );
-
       const emitter = jest.fn();
       await processor.readLocation(location, false, emitter);
 
@@ -256,7 +220,7 @@ describe('BitbucketDiscoveryProcessor', () => {
         location: {
           type: 'url',
           target:
-            'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml',
+            'https://bitbucket.mycompany.com/projects/backstage/repos/test/browse/catalog.yaml',
         },
         optional: true,
       });
@@ -277,26 +241,27 @@ describe('BitbucketDiscoveryProcessor', () => {
     const processor = BitbucketDiscoveryProcessor.fromConfig(
       new ConfigReader({
         integrations: {
-          bitbucket: [{ host: 'bitbucket.mycompany.com', token: 'blob' }],
+          bitbucket: [
+            {
+              host: 'bitbucket.mycompany.com',
+              token: 'blob',
+              apiBaseUrl: 'https://bitbucket.mycompany.com/api/rest/1.0',
+            },
+          ],
         },
       }),
       { parser: customRepositoryParser, logger: getVoidLogger() },
     );
 
     it('use custom repository parser', async () => {
+      setupStubs([{ key: 'backstage', repos: ['test'] }]);
+
       const location: LocationSpec = {
         type: 'bitbucket-discovery',
         target:
           'https://bitbucket.mycompany.com/projects/backstage/repos/test/catalog.yaml',
       };
 
-      jest
-        .spyOn(BitbucketClient.prototype, 'listProjects')
-        .mockResolvedValue(pagedResponse([{ key: 'backstage' }]));
-      jest
-        .spyOn(BitbucketClient.prototype, 'listRepositories')
-        .mockResolvedValue(pagedResponse([{ slug: 'test' }]));
-
       const emitter = jest.fn();
       await processor.readLocation(location, false, emitter);
 

plugins/catalog-backend/src/ingestion/processors/BitbucketDiscoveryProcessor.ts

@@ -22,11 +22,11 @@ import {
 } from '@backstage/integration';
 import { LocationSpec } from '@backstage/catalog-model';
 import {
-  Repository,
   BitbucketRepositoryParser,
   BitbucketClient,
   defaultRepositoryParser,
   paginated,
+  BitbucketRepository,
 } from './bitbucket';
 import { CatalogProcessor, CatalogProcessorEmit } from './types';
 
@@ -66,20 +66,19 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
       return false;
     }
 
-    const bitbucketConfig = this.integrations.bitbucket.byUrl(location.target)
-      ?.config;
-    if (!bitbucketConfig) {
+    const integration = this.integrations.bitbucket.byUrl(location.target);
+    if (!integration) {
       throw new Error(
         `There is no Bitbucket integration that matches ${location.target}. Please add a configuration entry for it under integrations.bitbucket`,
       );
-    } else if (bitbucketConfig.host === 'bitbucket.org') {
+    } else if (integration.config.host === 'bitbucket.org') {
       throw new Error(
         `Component discovery for Bitbucket Cloud is not yet supported`,
       );
     }
 
     const client = new BitbucketClient({
-      config: bitbucketConfig,
+      config: integration.config,
     });
     const startTimestamp = Date.now();
     this.logger.info(`Reading Bitbucket repositories from ${location.target}`);
@@ -90,9 +89,9 @@ export class BitbucketDiscoveryProcessor implements CatalogProcessor {
 
     for (const repository of result.matches) {
       for await (const entity of this.parser({
-        client: client,
-        repository: repository,
-        path: catalogPath,
+        integration: integration,
+        target: `${repository.links.self[0].href}${catalogPath}`,
+        logger: this.logger,
       })) {
         emit(entity);
       }
@@ -159,5 +158,5 @@ function escapeRegExp(str: string): RegExp {
 
 type Result = {
   scanned: number;
-  matches: Repository[];
+  matches: BitbucketRepository[];
 };

plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.test.ts

@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 import { defaultRepositoryParser } from './BitbucketRepositoryParser';
-import { Project, Repository } from './types';
-import { BitbucketClient } from './client';
 import { results } from '../index';
+import { getVoidLogger } from '@backstage/backend-common';
+import { BitbucketIntegration } from '@backstage/integration';
 
 describe('BitbucketRepositoryParser', () => {
   describe('defaultRepositoryParser', () => {
@@ -34,15 +34,9 @@ describe('BitbucketRepositoryParser', () => {
         ),
       ];
       const actual = await defaultRepositoryParser({
-        client: {} as BitbucketClient,
-        repository: {
-          project: {} as Project,
-          slug: 'repo-slug',
-          links: {
-            self: [{ href: browseUrl }],
-          },
-        } as Repository,
-        path: path,
+        integration: {} as BitbucketIntegration,
+        target: `${browseUrl}${path}`,
+        logger: getVoidLogger(),
       });
 
       let i = 0;

plugins/catalog-backend/src/ingestion/processors/bitbucket/BitbucketRepositoryParser.ts

@@ -13,25 +13,24 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { Repository } from './types';
 import { CatalogProcessorResult } from '../types';
 import { results } from '../index';
-import { BitbucketClient } from './client';
+import { Logger } from 'winston';
+import { BitbucketIntegration } from '@backstage/integration';
 
 export type BitbucketRepositoryParser = (options: {
-  client: BitbucketClient;
-  repository: Repository;
-  path: string;
+  integration: BitbucketIntegration;
+  target: string;
+  logger: Logger;
 }) => AsyncIterable<CatalogProcessorResult>;
 
 export const defaultRepositoryParser: BitbucketRepositoryParser = async function* defaultRepositoryParser({
-  repository,
-  path,
+  target,
 }) {
   yield results.location(
     {
       type: 'url',
-      target: `${repository.links.self[0].href}${path}`,
+      target: target,
     },
     // Not all locations may actually exist, since the user defined them as a wildcard pattern.
     // Thus, we emit them as optional and let the downstream processor find them while not outputting

plugins/catalog-backend/src/ingestion/processors/bitbucket/client.ts

@@ -41,15 +41,6 @@ export class BitbucketClient {
     );
   }
 
-  async getRaw(
-    projectKey: string,
-    repo: string,
-    path: string,
-  ): Promise<Response> {
-    const request = `${this.config.apiBaseUrl}/projects/${projectKey}/repos/${repo}/raw/${path}`;
-    return fetch(request, getBitbucketRequestOptions(this.config));
-  }
-
   private async pagedRequest(
     endpoint: string,
     options?: ListOptions,

plugins/catalog-backend/src/ingestion/processors/bitbucket/index.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 export { BitbucketClient, paginated } from './client';
-export type { PagedResponse } from './client';
-export * from './types';
-export type { BitbucketRepositoryParser } from './BitbucketRepositoryParser';
 export { defaultRepositoryParser } from './BitbucketRepositoryParser';
+export type { PagedResponse } from './client';
+export type { BitbucketRepository } from './types';
+export type { BitbucketRepositoryParser } from './BitbucketRepositoryParser';

plugins/catalog-backend/src/ingestion/processors/bitbucket/types.ts

@@ -13,16 +13,15 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-export type Project = {
-  key: string;
-};
-
-export type Repository = {
-  project: Project;
+export type BitbucketRepository = {
+  project: {
+    key: string;
+  };
   slug: string;
-  links: Record<string, Link[]>;
-};
-
-export type Link = {
-  href: string;
+  links: Record<
+    string,
+    {
+      href: string;
+    }[]
+  >;
 };

plugins/catalog-backend/src/ingestion/processors/index.ts

@@ -35,3 +35,5 @@ export * from './types';
 export { UrlReaderProcessor } from './UrlReaderProcessor';
 export { parseEntityYaml } from './util/parse';
 export { results };
+
+export type { BitbucketRepositoryParser } from './bitbucket';