Merge pull request #8172 from goenning/go/azure-devops-discovery

adding AzureDevOps Discovery Processor

.changeset/strong-seahorses-scream.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend': patch
+---
+
+Added Azure DevOps discovery processor

docs/integrations/azure/discovery.md

@@ -0,0 +1,49 @@
+---
+id: discovery
+title: Azure DevOps Discovery
+sidebar_label: Discovery
+# prettier-ignore
+description: Automatically discovering catalog entities from repositories in an Azure DevOps organization
+---
+
+The Azure DevOps integration has a special discovery processor for discovering
+catalog entities within an Azure DevOps. The processor will crawl the Azure
+DevOps organization and register entities matching the configured path. This can
+be useful as an alternative to static locations or manually adding things to the
+catalog.
+
+To use the discovery processor, you'll need a GitHub integration
+[set up](locations.md) with a `AZURE_TOKEN`. Then you can add a location target
+to the catalog configuration:
+
+```yaml
+catalog:
+  locations:
+    # Scan all repositories for a catalog-info.yaml in the root of the default branch
+    - type: azure-discovery
+      target: https://dev.azure.com/myorg/myproject
+    # Or use a custom pattern for a subset of all repositories with default repository
+    - type: azure-discovery
+      target: https://dev.azure.com/myorg/myproject/_git/service-*
+    # Or use a custom file format and location
+    - type: azure-discovery
+      target: https://dev.azure.com/myorg/myproject/_git/*?path=/src/*/catalog-info.yaml
+```
+
+Note the `azure-discovery` type, as this is not a regular `url` processor.
+
+When using a custom pattern, the target is composed of five parts:
+
+- The base instance URL, `https://dev.azure.com` in this case
+- The organization name which is required, `myorg` in this case
+- The project name which is required, `myproject` in this case
+- The repository blob to scan, which accepts \* wildcard tokens and must be
+  added after `_git/`. This can simply be `*` to scan all repositories in the
+  project.
+- The path within each repository to find the catalog YAML file. This will
+  usually be `/catalog-info.yaml`, `/src/*/catalog-info.yaml` or a similar
+  variation for catalog files stored in the root directory of each repository.
+
+_Note:_ the path parameter follows the same rules as the search on Azure DevOps
+web interface. For more details visit the
+[official search documentation](https://docs.microsoft.com/en-us/azure/devops/project/search/get-started-search?view=azure-devops)

microsite/sidebars.json

@@ -125,7 +125,11 @@
       {
         "type": "subcategory",
         "label": "Azure",
-        "ids": ["integrations/azure/locations", "integrations/azure/org"]
+        "ids": [
+          "integrations/azure/locations",
+          "integrations/azure/discovery",
+          "integrations/azure/org"
+        ]
       },
       {
         "type": "subcategory",

mkdocs.yml

@@ -88,6 +88,7 @@ nav:
           - Discovery: 'integrations/aws-s3/discovery.md'
       - Azure:
           - Locations: 'integrations/azure/locations.md'
+          - Discovery: 'integrations/azure/discovery.md'
           - Org Data: 'integrations/azure/org.md'
       - Bitbucket:
           - Locations: 'integrations/bitbucket/locations.md'

plugins/catalog-backend/api-report.md

@@ -174,6 +174,26 @@ export class AwsS3DiscoveryProcessor implements CatalogProcessor {
   ): Promise<boolean>;
 }
 
+// Warning: (ae-missing-release-tag) "AzureDevOpsDiscoveryProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+//
+// @public
+export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor {
+  constructor(options: { integrations: ScmIntegrations; logger: Logger_2 });
+  // (undocumented)
+  static fromConfig(
+    config: Config,
+    options: {
+      logger: Logger_2;
+    },
+  ): AzureDevOpsDiscoveryProcessor;
+  // (undocumented)
+  readLocation(
+    location: LocationSpec,
+    _optional: boolean,
+    emit: CatalogProcessorEmit,
+  ): Promise<boolean>;
+}
+
 // Warning: (ae-missing-release-tag) "BitbucketDiscoveryProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 //
 // @public (undocumented)

plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.test.ts

@@ -0,0 +1,277 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { getVoidLogger } from '@backstage/backend-common';
+import { LocationSpec } from '@backstage/catalog-model';
+import { ConfigReader } from '@backstage/config';
+import { codeSearch } from './azure';
+import {
+  AzureDevOpsDiscoveryProcessor,
+  parseUrl,
+} from './AzureDevOpsDiscoveryProcessor';
+
+jest.mock('./azure');
+const mockCodeSearch = codeSearch as jest.MockedFunction<typeof codeSearch>;
+
+describe('AzureDevOpsDiscoveryProcessor', () => {
+  describe('parseUrl', () => {
+    it('parses well formed URLs', () => {
+      expect(parseUrl('https://dev.azure.com/my-org/my-proj')).toEqual({
+        baseUrl: 'https://dev.azure.com',
+        org: 'my-org',
+        project: 'my-proj',
+        repo: '',
+        catalogPath: '/catalog-info.yaml',
+      });
+
+      expect(
+        parseUrl(
+          'https://dev.azure.com/spotify/engineering/_git/backstage?path=/catalog.yaml',
+        ),
+      ).toEqual({
+        baseUrl: 'https://dev.azure.com',
+        org: 'spotify',
+        project: 'engineering',
+        repo: 'backstage',
+        catalogPath: '/catalog.yaml',
+      });
+
+      expect(
+        parseUrl(
+          'https://azuredevops.mycompany.com/spotify/engineering/_git/backstage?path=/src/*/catalog.yaml',
+        ),
+      ).toEqual({
+        baseUrl: 'https://azuredevops.mycompany.com',
+        org: 'spotify',
+        project: 'engineering',
+        repo: 'backstage',
+        catalogPath: '/src/*/catalog.yaml',
+      });
+    });
+
+    it('throws on incorrectly formed URLs', () => {
+      expect(() => parseUrl('https://dev.azure.com')).toThrow();
+      expect(() => parseUrl('https://dev.azure.com//')).toThrow();
+      expect(() => parseUrl('https://dev.azure.com//foo')).toThrow();
+    });
+  });
+
+  describe('reject unrelated entries', () => {
+    it('rejects unknown types', async () => {
+      const processor = AzureDevOpsDiscoveryProcessor.fromConfig(
+        new ConfigReader({
+          integrations: {
+            azure: [{ host: 'dev.azure.com', token: 'blob' }],
+          },
+        }),
+        { logger: getVoidLogger() },
+      );
+      const location: LocationSpec = {
+        type: 'not-azure-discovery',
+        target: 'https://dev.azure.com',
+      };
+      await expect(
+        processor.readLocation(location, false, () => {}),
+      ).resolves.toBeFalsy();
+    });
+  });
+
+  it('rejects unknown targets', async () => {
+    const processor = AzureDevOpsDiscoveryProcessor.fromConfig(
+      new ConfigReader({
+        integrations: {
+          github: [
+            { host: 'dev.azure.com', token: 'blob' },
+            { host: 'azure.myorg.com', token: 'blob' },
+          ],
+        },
+      }),
+      { logger: getVoidLogger() },
+    );
+    const location: LocationSpec = {
+      type: 'azure-discovery',
+      target: 'https://not.azure.com/org/project',
+    };
+    await expect(
+      processor.readLocation(location, false, () => {}),
+    ).rejects.toThrow(
+      /There is no Azure integration that matches https:\/\/not.azure.com\/org\/project. Please add a configuration entry for it under integrations.azure/,
+    );
+  });
+
+  describe('handles repositories', () => {
+    const processor = AzureDevOpsDiscoveryProcessor.fromConfig(
+      new ConfigReader({
+        integrations: {
+          github: [{ host: 'dev.azure.com', token: 'blob' }],
+        },
+      }),
+      { logger: getVoidLogger() },
+    );
+
+    beforeEach(() => {
+      mockCodeSearch.mockClear();
+    });
+
+    it('output all locations found on from code search', async () => {
+      const location: LocationSpec = {
+        type: 'azure-discovery',
+        target: 'https://dev.azure.com/shopify/engineering',
+      };
+      mockCodeSearch.mockResolvedValueOnce([
+        {
+          fileName: 'catalog-info.yaml',
+          path: '/catalog-info.yaml',
+          repository: {
+            name: 'backstage',
+          },
+        },
+        {
+          fileName: 'catalog-info.yaml',
+          path: '/src/catalog-info.yaml',
+          repository: {
+            name: 'ios-app',
+          },
+        },
+      ]);
+      const emitter = jest.fn();
+
+      await processor.readLocation(location, false, emitter);
+
+      expect(mockCodeSearch).toHaveBeenCalledWith(
+        { host: 'dev.azure.com' },
+        'shopify',
+        'engineering',
+        '',
+        '/catalog-info.yaml',
+      );
+      expect(emitter).toHaveBeenCalledTimes(2);
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://dev.azure.com/shopify/engineering/_git/backstage?path=/catalog-info.yaml',
+        },
+        optional: true,
+      });
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://dev.azure.com/shopify/engineering/_git/ios-app?path=/src/catalog-info.yaml',
+        },
+        optional: true,
+      });
+    });
+
+    it('output single locations from code search', async () => {
+      const location: LocationSpec = {
+        type: 'azure-discovery',
+        target: 'https://dev.azure.com/shopify/engineering/_git/backstage',
+      };
+      mockCodeSearch.mockResolvedValueOnce([
+        {
+          fileName: 'catalog-info.yaml',
+          path: '/catalog-info.yaml',
+          repository: {
+            name: 'backstage',
+          },
+        },
+      ]);
+      const emitter = jest.fn();
+
+      await processor.readLocation(location, false, emitter);
+
+      expect(mockCodeSearch).toHaveBeenCalledWith(
+        { host: 'dev.azure.com' },
+        'shopify',
+        'engineering',
+        'backstage',
+        '/catalog-info.yaml',
+      );
+      expect(emitter).toHaveBeenCalledTimes(1);
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://dev.azure.com/shopify/engineering/_git/backstage?path=/catalog-info.yaml',
+        },
+        optional: true,
+      });
+    });
+
+    it('output single locations with different file name from code search', async () => {
+      const location: LocationSpec = {
+        type: 'azure-discovery',
+        target:
+          'https://dev.azure.com/shopify/engineering?path=/src/*/catalog.yaml',
+      };
+      mockCodeSearch.mockResolvedValueOnce([
+        {
+          fileName: 'catalog.yaml',
+          path: '/src/main/catalog.yaml',
+          repository: {
+            name: 'backstage',
+          },
+        },
+      ]);
+      const emitter = jest.fn();
+
+      await processor.readLocation(location, false, emitter);
+
+      expect(mockCodeSearch).toHaveBeenCalledWith(
+        { host: 'dev.azure.com' },
+        'shopify',
+        'engineering',
+        '',
+        '/src/*/catalog.yaml',
+      );
+      expect(emitter).toHaveBeenCalledTimes(1);
+      expect(emitter).toHaveBeenCalledWith({
+        type: 'location',
+        location: {
+          type: 'url',
+          target:
+            'https://dev.azure.com/shopify/engineering/_git/backstage?path=/src/main/catalog.yaml',
+        },
+        optional: true,
+      });
+    });
+
+    it('output nothing when code search does not find anything', async () => {
+      const location: LocationSpec = {
+        type: 'azure-discovery',
+        target: 'https://dev.azure.com/shopify/engineering/_git/backstage',
+      };
+      mockCodeSearch.mockResolvedValueOnce([]);
+      const emitter = jest.fn();
+
+      await processor.readLocation(location, false, emitter);
+
+      expect(mockCodeSearch).toHaveBeenCalledWith(
+        { host: 'dev.azure.com' },
+        'shopify',
+        'engineering',
+        'backstage',
+        '/catalog-info.yaml',
+      );
+      expect(emitter).not.toHaveBeenCalled();
+    });
+  });
+});

plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts

@@ -0,0 +1,151 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { LocationSpec } from '@backstage/catalog-model';
+import { Config } from '@backstage/config';
+import { ScmIntegrations } from '@backstage/integration';
+import { Logger } from 'winston';
+import * as results from './results';
+import { CatalogProcessor, CatalogProcessorEmit } from './types';
+import { codeSearch } from './azure';
+
+/**
+ * Extracts repositories out of an Azure DevOps org.
+ *
+ * The following will create locations for all projects which have a catalog-info.yaml
+ * on the default branch. The first is shorthand for the second.
+ *
+ *    target: "https://dev.azure.com/org/project"
+ *    or
+ *    target: https://dev.azure.com/org/project?path=/catalog-info.yaml
+ *
+ * You may also explicitly specify a single repo:
+ *
+ *    target: https://dev.azure.com/org/project/_git/repo
+ **/
+export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor {
+  private readonly integrations: ScmIntegrations;
+  private readonly logger: Logger;
+
+  static fromConfig(config: Config, options: { logger: Logger }) {
+    const integrations = ScmIntegrations.fromConfig(config);
+
+    return new AzureDevOpsDiscoveryProcessor({
+      ...options,
+      integrations,
+    });
+  }
+
+  constructor(options: { integrations: ScmIntegrations; logger: Logger }) {
+    this.integrations = options.integrations;
+    this.logger = options.logger;
+  }
+
+  async readLocation(
+    location: LocationSpec,
+    _optional: boolean,
+    emit: CatalogProcessorEmit,
+  ): Promise<boolean> {
+    if (location.type !== 'azure-discovery') {
+      return false;
+    }
+
+    const azureConfig = this.integrations.azure.byUrl(location.target)?.config;
+    if (!azureConfig) {
+      throw new Error(
+        `There is no Azure integration that matches ${location.target}. Please add a configuration entry for it under integrations.azure`,
+      );
+    }
+
+    const { baseUrl, org, project, repo, catalogPath } = parseUrl(
+      location.target,
+    );
+    this.logger.info(
+      `Reading Azure DevOps repositories from ${location.target}`,
+    );
+
+    const files = await codeSearch(
+      azureConfig,
+      org,
+      project,
+      repo,
+      catalogPath,
+    );
+
+    this.logger.debug(
+      `Found ${files.length} files in Azure DevOps from ${location.target}.`,
+    );
+
+    for (const file of files) {
+      emit(
+        results.location(
+          {
+            type: 'url',
+            target: `${baseUrl}/${org}/${project}/_git/${file.repository.name}?path=${file.path}`,
+          },
+          // Not all locations may actually exist, since the user defined them as a wildcard pattern.
+          // Thus, we emit them as optional and let the downstream processor find them while not outputting
+          // an error if it couldn't.
+          true,
+        ),
+      );
+    }
+
+    return true;
+  }
+}
+
+/**
+ * parseUrl extracts segments from the Azure DevOps URL.
+ **/
+export function parseUrl(urlString: string): {
+  baseUrl: string;
+  org: string;
+  project: string;
+  repo: string;
+  catalogPath: string;
+} {
+  const url = new URL(urlString);
+  const path = url.pathname.substr(1).split('/');
+
+  const catalogPath = url.searchParams.get('path') || '/catalog-info.yaml';
+
+  if (path.length === 2 && path[0].length && path[1].length) {
+    return {
+      baseUrl: url.origin,
+      org: decodeURIComponent(path[0]),
+      project: decodeURIComponent(path[1]),
+      repo: '',
+      catalogPath,
+    };
+  } else if (
+    path.length === 4 &&
+    path[0].length &&
+    path[1].length &&
+    path[2].length &&
+    path[3].length
+  ) {
+    return {
+      baseUrl: url.origin,
+      org: decodeURIComponent(path[0]),
+      project: decodeURIComponent(path[1]),
+      repo: decodeURIComponent(path[3]),
+      catalogPath,
+    };
+  }
+
+  throw new Error(`Failed to parse ${urlString}`);
+}

plugins/catalog-backend/src/ingestion/processors/azure/azure.test.ts

@@ -0,0 +1,232 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { setupServer } from 'msw/node';
+import { setupRequestMockHandlers } from '@backstage/test-utils';
+import { rest } from 'msw';
+import { codeSearch, CodeSearchResponse } from './azure';
+
+describe('azure', () => {
+  const server = setupServer();
+  setupRequestMockHandlers(server);
+
+  describe('codeSearch', () => {
+    it('returns empty when nothing is found', async () => {
+      const response: CodeSearchResponse = { count: 0, results: [] };
+
+      server.use(
+        rest.post(
+          `https://almsearch.dev.azure.com/shopify/engineering/_apis/search/codesearchresults`,
+          (req, res, ctx) => {
+            expect(req.headers.get('Authorization')).toBe('Basic OkFCQw==');
+            expect(req.body).toEqual({
+              searchText: 'path:/catalog-info.yaml repo:*',
+              $skip: 0,
+              $top: 1000,
+            });
+            return res(ctx.json(response));
+          },
+        ),
+      );
+
+      await expect(
+        codeSearch(
+          { host: 'dev.azure.com', token: 'ABC' },
+          'shopify',
+          'engineering',
+          '',
+          '/catalog-info.yaml',
+        ),
+      ).resolves.toEqual([]);
+    });
+  });
+
+  it('returns entries when request matches some files', async () => {
+    const response: CodeSearchResponse = {
+      count: 2,
+      results: [
+        {
+          fileName: 'catalog-info.yaml',
+          path: '/catalog-info.yaml',
+          repository: {
+            name: 'backstage',
+          },
+        },
+        {
+          fileName: 'catalog-info.yaml',
+          path: '/catalog-info.yaml',
+          repository: {
+            name: 'ios-app',
+          },
+        },
+      ],
+    };
+
+    server.use(
+      rest.post(
+        `https://almsearch.dev.azure.com/shopify/engineering/_apis/search/codesearchresults`,
+        (req, res, ctx) => {
+          expect(req.headers.get('Authorization')).toBe('Basic OkFCQw==');
+          expect(req.body).toEqual({
+            searchText: 'path:/catalog-info.yaml repo:*',
+            $skip: 0,
+            $top: 1000,
+          });
+          return res(ctx.json(response));
+        },
+      ),
+    );
+
+    await expect(
+      codeSearch(
+        { host: 'dev.azure.com', token: 'ABC' },
+        'shopify',
+        'engineering',
+        '',
+        '/catalog-info.yaml',
+      ),
+    ).resolves.toEqual(response.results);
+  });
+
+  it('searches in specific repo if parameter is set', async () => {
+    const response: CodeSearchResponse = {
+      count: 1,
+      results: [
+        {
+          fileName: 'catalog-info.yaml',
+          path: '/catalog-info.yaml',
+          repository: {
+            name: 'backstage',
+          },
+        },
+      ],
+    };
+
+    server.use(
+      rest.post(
+        `https://almsearch.dev.azure.com/shopify/engineering/_apis/search/codesearchresults`,
+        (req, res, ctx) => {
+          expect(req.headers.get('Authorization')).toBe('Basic OkFCQw==');
+          expect(req.body).toEqual({
+            searchText: 'path:/catalog-info.yaml repo:backstage',
+            $skip: 0,
+            $top: 1000,
+          });
+          return res(ctx.json(response));
+        },
+      ),
+    );
+
+    await expect(
+      codeSearch(
+        { host: 'dev.azure.com', token: 'ABC' },
+        'shopify',
+        'engineering',
+        'backstage',
+        '/catalog-info.yaml',
+      ),
+    ).resolves.toEqual(response.results);
+  });
+
+  it('can search using onpremise api', async () => {
+    const response: CodeSearchResponse = {
+      count: 1,
+      results: [
+        {
+          fileName: 'catalog-info.yaml',
+          path: '/catalog-info.yaml',
+          repository: {
+            name: 'backstage',
+          },
+        },
+      ],
+    };
+
+    server.use(
+      rest.post(
+        `https://azuredevops.mycompany.com/shopify/engineering/_apis/search/codesearchresults`,
+        (req, res, ctx) => {
+          expect(req.headers.get('Authorization')).toBe('Basic OkFCQw==');
+          expect(req.body).toEqual({
+            searchText: 'path:/catalog-info.yaml repo:*',
+            $skip: 0,
+            $top: 1000,
+          });
+          return res(ctx.json(response));
+        },
+      ),
+    );
+
+    await expect(
+      codeSearch(
+        { host: 'azuredevops.mycompany.com', token: 'ABC' },
+        'shopify',
+        'engineering',
+        '',
+        '/catalog-info.yaml',
+      ),
+    ).resolves.toEqual(response.results);
+  });
+
+  it('searches multiple pages if response contains many items', async () => {
+    const totalCount = 2401;
+    const generateItems = (count: number) => {
+      return Array.from(Array(count).keys()).map(_ => ({
+        fileName: 'catalog-info.yaml',
+        path: '/catalog-info.yaml',
+        repository: {
+          name: 'backstage',
+        },
+      }));
+    };
+
+    server.use(
+      rest.post(
+        `https://almsearch.dev.azure.com/shopify/engineering/_apis/search/codesearchresults`,
+        (req, res, ctx) => {
+          expect(req.headers.get('Authorization')).toBe('Basic OkFCQw==');
+          expect(req.body).toMatchObject({
+            searchText: 'path:/catalog-info.yaml repo:backstage',
+            $top: 1000,
+          });
+
+          const body = req.body as { $skip: number; $top: number };
+          const countItemsToReturn =
+            body.$top + body.$skip > totalCount
+              ? totalCount - body.$skip
+              : body.$top;
+
+          return res(
+            ctx.json({
+              count: totalCount,
+              results: generateItems(countItemsToReturn),
+            }),
+          );
+        },
+      ),
+    );
+
+    await expect(
+      codeSearch(
+        { host: 'dev.azure.com', token: 'ABC' },
+        'shopify',
+        'engineering',
+        'backstage',
+        '/catalog-info.yaml',
+      ),
+    ).resolves.toHaveLength(totalCount);
+  });
+});

plugins/catalog-backend/src/ingestion/processors/azure/azure.ts

@@ -0,0 +1,80 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fetch from 'cross-fetch';
+import {
+  AzureIntegrationConfig,
+  getAzureRequestOptions,
+} from '@backstage/integration';
+
+export interface CodeSearchResponse {
+  count: number;
+  results: CodeSearchResultItem[];
+}
+
+export interface CodeSearchResultItem {
+  fileName: string;
+  path: string;
+  repository: {
+    name: string;
+  };
+}
+
+const isCloud = (host: string) => host === 'dev.azure.com';
+const PAGE_SIZE = 1000;
+
+// codeSearch returns all files that matches the given search path.
+export async function codeSearch(
+  azureConfig: AzureIntegrationConfig,
+  org: string,
+  project: string,
+  repo: string,
+  path: string,
+): Promise<CodeSearchResultItem[]> {
+  const searchBaseUrl = isCloud(azureConfig.host)
+    ? 'https://almsearch.dev.azure.com'
+    : `https://${azureConfig.host}`;
+  const searchUrl = `${searchBaseUrl}/${org}/${project}/_apis/search/codesearchresults?api-version=6.0-preview.1`;
+
+  let items: CodeSearchResultItem[] = [];
+  let hasMorePages = true;
+
+  do {
+    const response = await fetch(searchUrl, {
+      ...getAzureRequestOptions(azureConfig, {
+        'Content-Type': 'application/json',
+      }),
+      method: 'POST',
+      body: JSON.stringify({
+        searchText: `path:${path} repo:${repo || '*'}`,
+        $skip: items.length,
+        $top: PAGE_SIZE,
+      }),
+    });
+
+    if (response.status !== 200) {
+      throw new Error(
+        `Azure DevOps search failed with response status ${response.status}`,
+      );
+    }
+
+    const body: CodeSearchResponse = await response.json();
+    items = [...items, ...body.results];
+    hasMorePages = body.count > items.length;
+  } while (hasMorePages);
+
+  return items;
+}

plugins/catalog-backend/src/ingestion/processors/azure/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2021 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export * from './azure';

plugins/catalog-backend/src/ingestion/processors/index.ts

@@ -26,6 +26,7 @@ export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor';
 export { CodeOwnersProcessor } from './CodeOwnersProcessor';
 export { FileReaderProcessor } from './FileReaderProcessor';
 export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor';
+export { AzureDevOpsDiscoveryProcessor } from './AzureDevOpsDiscoveryProcessor';
 export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor';
 export { GithubMultiOrgReaderProcessor } from './GithubMultiOrgReaderProcessor';
 export { GitLabDiscoveryProcessor } from './GitLabDiscoveryProcessor';

plugins/catalog-backend/src/legacy/service/CatalogBuilder.ts

@@ -42,6 +42,7 @@ import {
   CodeOwnersProcessor,
   FileReaderProcessor,
   GithubDiscoveryProcessor,
+  AzureDevOpsDiscoveryProcessor,
   GithubOrgReaderProcessor,
   GitLabDiscoveryProcessor,
   LocationEntityProcessor,
@@ -317,6 +318,7 @@ export class CatalogBuilder {
         new FileReaderProcessor(),
         BitbucketDiscoveryProcessor.fromConfig(config, { logger }),
         GithubDiscoveryProcessor.fromConfig(config, { logger }),
+        AzureDevOpsDiscoveryProcessor.fromConfig(config, { logger }),
         GithubOrgReaderProcessor.fromConfig(config, { logger }),
         GitLabDiscoveryProcessor.fromConfig(config, { logger }),
         new UrlReaderProcessor({ reader, logger }),

plugins/catalog-backend/src/service/NextCatalogBuilder.ts

@@ -44,6 +44,7 @@ import {
   CatalogProcessorParser,
   CodeOwnersProcessor,
   FileReaderProcessor,
+  AzureDevOpsDiscoveryProcessor,
   GithubDiscoveryProcessor,
   GithubOrgReaderProcessor,
   GitLabDiscoveryProcessor,
@@ -292,6 +293,7 @@ export class NextCatalogBuilder {
     return [
       new FileReaderProcessor(),
       BitbucketDiscoveryProcessor.fromConfig(config, { logger }),
+      AzureDevOpsDiscoveryProcessor.fromConfig(config, { logger }),
       GithubDiscoveryProcessor.fromConfig(config, { logger }),
       GithubOrgReaderProcessor.fromConfig(config, { logger }),
       GitLabDiscoveryProcessor.fromConfig(config, { logger }),