fix comments, support onpremise and paging

Signed-off-by: goenning <me@goenning.net>

.changeset/strong-seahorses-scream.md

@@ -1,5 +1,5 @@
 ---
-'@backstage/plugin-catalog-backend': minor
+'@backstage/plugin-catalog-backend': patch
 ---
 
 Added Azure DevOps discovery processor

microsite/sidebars.json

@@ -124,7 +124,11 @@
       {
         "type": "subcategory",
         "label": "Azure",
-        "ids": ["integrations/azure/locations", "integrations/azure/org"]
+        "ids": [
+          "integrations/azure/locations",
+          "integrations/azure/discovery",
+          "integrations/azure/org"
+        ]
       },
       {
         "type": "subcategory",

mkdocs.yml

@@ -88,6 +88,7 @@ nav:
           - Discovery: 'integrations/aws-s3/discovery.md'
       - Azure:
           - Locations: 'integrations/azure/locations.md'
+          - Discovery: 'integrations/azure/discovery.md'
           - Org Data: 'integrations/azure/org.md'
       - Bitbucket:
           - Locations: 'integrations/bitbucket/locations.md'

plugins/catalog-backend/src/ingestion/processors/AzureDevOpsDiscoveryProcessor.ts

@@ -85,7 +85,9 @@ export class AzureDevOpsDiscoveryProcessor implements CatalogProcessor {
       catalogPath,
     );
 
-    this.logger.info(`Found ${files.length} files in Azure DevOps.`);
+    this.logger.debug(
+      `Found ${files.length} files in Azure DevOps from ${location.target}.`,
+    );
 
     for (const file of files) {
       emit(

plugins/catalog-backend/src/ingestion/processors/azure/azure.test.ts

@@ -34,6 +34,7 @@ describe('azure', () => {
             expect(req.headers.get('Authorization')).toBe('Basic OkFCQw==');
             expect(req.body).toEqual({
               searchText: 'path:/catalog-info.yaml repo:*',
+              $skip: 0,
               $top: 1000,
             });
             return res(ctx.json(response));
@@ -81,6 +82,7 @@ describe('azure', () => {
           expect(req.headers.get('Authorization')).toBe('Basic OkFCQw==');
           expect(req.body).toEqual({
             searchText: 'path:/catalog-info.yaml repo:*',
+            $skip: 0,
             $top: 1000,
           });
           return res(ctx.json(response));
@@ -120,6 +122,7 @@ describe('azure', () => {
           expect(req.headers.get('Authorization')).toBe('Basic OkFCQw==');
           expect(req.body).toEqual({
             searchText: 'path:/catalog-info.yaml repo:backstage',
+            $skip: 0,
             $top: 1000,
           });
           return res(ctx.json(response));
@@ -137,4 +140,93 @@ describe('azure', () => {
       ),
     ).resolves.toEqual(response.results);
   });
+
+  it('can search using onpremise api', async () => {
+    const response: CodeSearchResponse = {
+      count: 1,
+      results: [
+        {
+          fileName: 'catalog-info.yaml',
+          path: '/catalog-info.yaml',
+          repository: {
+            name: 'backstage',
+          },
+        },
+      ],
+    };
+
+    server.use(
+      rest.post(
+        `https://azuredevops.mycompany.com/shopify/engineering/_apis/search/codesearchresults`,
+        (req, res, ctx) => {
+          expect(req.headers.get('Authorization')).toBe('Basic OkFCQw==');
+          expect(req.body).toEqual({
+            searchText: 'path:/catalog-info.yaml repo:*',
+            $skip: 0,
+            $top: 1000,
+          });
+          return res(ctx.json(response));
+        },
+      ),
+    );
+
+    await expect(
+      codeSearch(
+        { host: 'azuredevops.mycompany.com', token: 'ABC' },
+        'shopify',
+        'engineering',
+        '',
+        '/catalog-info.yaml',
+      ),
+    ).resolves.toEqual(response.results);
+  });
+
+  it('searches multiple pages if response contains many items', async () => {
+    const totalCount = 2401;
+    const generateItems = (count: number) => {
+      return Array.from(Array(count).keys()).map(_ => ({
+        fileName: 'catalog-info.yaml',
+        path: '/catalog-info.yaml',
+        repository: {
+          name: 'backstage',
+        },
+      }));
+    };
+
+    server.use(
+      rest.post(
+        `https://almsearch.dev.azure.com/shopify/engineering/_apis/search/codesearchresults`,
+        (req, res, ctx) => {
+          expect(req.headers.get('Authorization')).toBe('Basic OkFCQw==');
+          expect(req.body).toMatchObject({
+            searchText: 'path:/catalog-info.yaml repo:backstage',
+            $top: 1000,
+          });
+
+          const body = req.body as { $skip: number; $top: number };
+          const countItemsToReturn =
+            body.$top + body.$skip > totalCount
+              ? totalCount - body.$skip
+              : body.$top;
+
+          return res(
+            ctx.json({
+              count: totalCount,
+              results: generateItems(countItemsToReturn),
+            }),
+          );
+        },
+      ),
+    );
+
+    await expect(
+      codeSearch(
+        { host: 'dev.azure.com', token: 'ABC' },
+        'shopify',
+        'engineering',
+        'backstage',
+        '/catalog-info.yaml',
+      ),
+    ).resolves.toHaveLength(totalCount);
+  });
 });

plugins/catalog-backend/src/ingestion/processors/azure/azure.ts

@@ -33,6 +33,9 @@ export interface CodeSearchResultItem {
   };
 }
 
+const isCloud = (host: string) => host === 'dev.azure.com';
+const PAGE_SIZE = 1000;
+
 // codeSearch returns all files that matches the given search path.
 export async function codeSearch(
   azureConfig: AzureIntegrationConfig,
@@ -41,27 +44,37 @@ export async function codeSearch(
   repo: string,
   path: string,
 ): Promise<CodeSearchResultItem[]> {
-  // TODO:  What's the search URL for onpremises DevOps?
-  const searchUrl = `https://almsearch.dev.azure.com/${org}/${project}/_apis/search/codesearchresults?api-version=6.0-preview.1`;
-  const opts = getAzureRequestOptions(azureConfig);
-  const response = await fetch(searchUrl, {
-    method: 'POST',
-    headers: {
-      ...opts.headers,
-      'Content-Type': 'application/json',
-    },
-    body: JSON.stringify({
-      searchText: `path:${path} repo:${repo || '*'}`,
-      $top: 1000,
-    }),
-  });
+  const searchBaseUrl = isCloud(azureConfig.host)
+    ? 'https://almsearch.dev.azure.com'
+    : `https://${azureConfig.host}`;
+  const searchUrl = `${searchBaseUrl}/${org}/${project}/_apis/search/codesearchresults?api-version=6.0-preview.1`;
 
-  if (response.status !== 200) {
-    throw new Error(
-      `Azure DevOps search failed with response status ${response.status}`,
-    );
-  }
+  let items: CodeSearchResultItem[] = [];
+  let hasMorePages = true;
 
-  const responseBody: CodeSearchResponse = await response.json();
-  return responseBody.results;
+  do {
+    const response = await fetch(searchUrl, {
+      ...getAzureRequestOptions(azureConfig, {
+        'Content-Type': 'application/json',
+      }),
+      method: 'POST',
+      body: JSON.stringify({
+        searchText: `path:${path} repo:${repo || '*'}`,
+        $skip: items.length,
+        $top: PAGE_SIZE,
+      }),
+    });
+
+    if (response.status !== 200) {
+      throw new Error(
+        `Azure DevOps search failed with response status ${response.status}`,
+      );
+    }
+
+    const body: CodeSearchResponse = await response.json();
+    items = [...items, ...body.results];
+    hasMorePages = body.count > items.length;
+  } while (hasMorePages);
+
+  return items;
 }