backend-common: UrlReader/Azure implement SHA based caching

packages/backend-common/src/reading/AzureUrlReader.test.ts

@@ -23,6 +23,7 @@ import { getVoidLogger } from '../logging';
 import { AzureUrlReader } from './AzureUrlReader';
 import { msw } from '@backstage/test-utils';
 import { ReadTreeResponseFactory } from './tree';
+import { NotModifiedError } from '../errors';
 
 const logger = getVoidLogger();
 
@@ -142,6 +143,11 @@ describe('AzureUrlReader', () => {
       path.resolve('src', 'reading', '__fixtures__', 'mock-main.zip'),
     );
 
+    const processor = new AzureUrlReader(
+      { host: 'dev.azure.com' },
+      { treeResponseFactory },
+    );
+
     beforeEach(() => {
       worker.use(
         rest.get(
@@ -153,19 +159,65 @@ describe('AzureUrlReader', () => {
               ctx.body(repoBuffer),
             ),
         ),
+        rest.get(
+          // https://docs.microsoft.com/en-us/rest/api/azure/devops/git/commits/get%20commits?view=azure-devops-rest-6.0#on-a-branch
+          'https://dev.azure.com/organization/project/_apis/git/repositories/repository/commits',
+          (_, res, ctx) =>
+            res(
+              ctx.status(200),
+              ctx.json({
+                count: 2,
+                value: [
+                  {
+                    commitId: '123abc2',
+                    comment: 'second commit',
+                  },
+                  {
+                    commitId: '123abc1',
+                    comment: 'first commit',
+                  },
+                ],
+              }),
+            ),
+        ),
       );
     });
 
     it('returns the wanted files from an archive', async () => {
-      const processor = new AzureUrlReader(
-        { host: 'dev.azure.com' },
-        { treeResponseFactory },
-      );
-
       const response = await processor.readTree(
         'https://dev.azure.com/organization/project/_git/repository',
       );
 
+      expect(response.sha).toBe('123abc2');
+
+      const files = await response.files();
+
+      expect(files.length).toBe(2);
+      const mkDocsFile = await files[0].content();
+      const indexMarkdownFile = await files[1].content();
+
+      expect(mkDocsFile.toString()).toBe('site_name: Test\n');
+      expect(indexMarkdownFile.toString()).toBe('# Test\n');
+    });
+
+    it('throws a NotModifiedError when given a sha in options', async () => {
+      const fnAzure = async () => {
+        await processor.readTree(
+          'https://dev.azure.com/organization/project/_git/repository',
+          { sha: '123abc2' },
+        );
+      };
+
+      await expect(fnAzure).rejects.toThrow(NotModifiedError);
+    });
+
+    it('should not throw a NotModifiedError when given an outdated sha in options', async () => {
+      const response = await processor.readTree(
+        'https://dev.azure.com/organization/project/_git/repository',
+        { sha: 'outdated123abc' },
+      );
+
+      expect(response.sha).toBe('123abc2');
       const files = await response.files();
 
       expect(files.length).toBe(2);

packages/backend-common/src/reading/AzureUrlReader.ts

@@ -20,10 +20,11 @@ import {
   getAzureFileFetchUrl,
   getAzureDownloadUrl,
   getAzureRequestOptions,
+  getAzureCommitsUrl,
 } from '@backstage/integration';
 import fetch from 'cross-fetch';
 import { Readable } from 'stream';
-import { NotFoundError } from '../errors';
+import { NotFoundError, NotModifiedError } from '../errors';
 import {
   ReaderFactory,
   ReadTreeOptions,
@@ -75,6 +76,25 @@ export class AzureUrlReader implements UrlReader {
     url: string,
     options?: ReadTreeOptions,
   ): Promise<ReadTreeResponse> {
+    // Get latest commit SHA
+
+    const commitsAzureResponse = await fetch(
+      getAzureCommitsUrl(url),
+      getAzureRequestOptions(this.options),
+    );
+    if (!commitsAzureResponse.ok) {
+      const message = `Failed to read tree from ${url}, ${commitsAzureResponse.status} ${commitsAzureResponse.statusText}`;
+      if (commitsAzureResponse.status === 404) {
+        throw new NotFoundError(message);
+      }
+      throw new Error(message);
+    }
+
+    const commitSha = (await commitsAzureResponse.json()).value[0].commitId;
+    if (options?.sha && options.sha === commitSha) {
+      throw new NotModifiedError();
+    }
+
     const archiveAzureResponse = await fetch(
       getAzureDownloadUrl(url),
       getAzureRequestOptions(this.options, { Accept: 'application/zip' }),
@@ -93,8 +113,7 @@ export class AzureUrlReader implements UrlReader {
     });
 
     const response = archiveResponse as ReadTreeResponse;
-    // TODO: Just a placeholder for now.
-    response.sha = '';
+    response.sha = commitSha;
     return response;
   }
 

packages/integration/src/azure/core.ts

@@ -108,6 +108,62 @@ export function getAzureDownloadUrl(url: string): string {
   return `${protocol}://${resource}/${organization}/${project}/_apis/git/repositories/${repoName}/items?recursionLevel=full&download=true&api-version=6.0${scopePath}`;
 }
 
+/**
+ * Given a URL, return the API URL to fetch commits on the branch.
+ *
+ * @param url A URL pointing to a repository or a sub-path
+ */
+export function getAzureCommitsUrl(url: string): string {
+  try {
+    const parsedUrl = new URL(url);
+
+    const [
+      empty,
+      userOrOrg,
+      project,
+      srcKeyword,
+      repoName,
+    ] = parsedUrl.pathname.split('/');
+
+    // Remove the "GB" from "GBmain" for example.
+    const ref = parsedUrl.searchParams.get('version')?.substr(2);
+
+    if (
+      empty !== '' ||
+      userOrOrg === '' ||
+      project === '' ||
+      srcKeyword !== '_git' ||
+      repoName === ''
+    ) {
+      throw new Error('Wrong Azure Devops URL');
+    }
+
+    // transform to commits api
+    parsedUrl.pathname = [
+      empty,
+      userOrOrg,
+      project,
+      '_apis',
+      'git',
+      'repositories',
+      repoName,
+      'commits',
+    ].join('/');
+
+    const queryParams = [];
+    if (ref) {
+      queryParams.push(`searchCriteria.itemVersion.version=${ref}`);
+    }
+    parsedUrl.search = queryParams.join('&');
+
+    parsedUrl.protocol = 'https';
+
+    return parsedUrl.toString();
+  } catch (e) {
+    throw new Error(`Incorrect URL: ${url}, ${e}`);
+  }
+}
+
 /**
  * Gets the request options necessary to make requests to a given provider.
  *

packages/integration/src/azure/index.ts

@@ -23,4 +23,5 @@ export {
   getAzureDownloadUrl,
   getAzureFileFetchUrl,
   getAzureRequestOptions,
+  getAzureCommitsUrl,
 } from './core';