Merge branch 'master' of github.com:spotify/backstage into migrate-to-msw

* 'master' of github.com:spotify/backstage: (110 commits)
  chore(catalog-backend): removing redudant classes and some functions
  chore(deps-dev): bump @types/webpack from 4.41.21 to 4.41.22 (#2765)
  move codecov.yml to .github
  feat(catalog-backend): add batch concurrency
  create-app: remove build step
  cli: simplify jest transform ignore regex
  feat(catalog-backend): introduce batching, speed up reading and writing of large datasets
  Techdocs: add Azure DevOps prepare support (#2748)
  feat(techdocs-header): Show breadcrumbs on docs page (#2786)
  changesets: add entry for create-app template location fix
  create-app: revert to github location type for example templates
  fix: make catalog filter work again
  Use new url scheme for techdocs
  feat: remove LocationProcessor.processEntity
  Add Dockerfile for helm chart
  feat: use the new UrlReader in the CodeOwnersProcessor
  feat: use new UrlReader in PlaceholderProcessor
  feat: remove the backstage.io/definition-at-location annotation
  Update loud-lamps-visit.md
  feat(proxy-backend): limit the forwarded http headers to a safe set
  ...

plugins/techdocs-backend/src/default-branch.ts

@@ -61,6 +61,16 @@ function getGitlabApiUrl(url: string): URL {
   );
 }
 
+function getAzureApiUrl(url: string): URL {
+  const { protocol, resource, organization, owner, name } = parseGitUrl(url);
+  const apiRepoPath = '_apis/git/repositories';
+  const apiVersion = 'api-version=6.0';
+
+  return new URL(
+    `${protocol}://${resource}/${organization}/${owner}/${apiRepoPath}/${name}?${apiVersion}`,
+  );
+}
+
 function getGithubRequestOptions(config: Config): RequestInit {
   const headers: HeadersInit = {
     Accept: 'application/vnd.github.v3.raw',
@@ -69,7 +79,7 @@ function getGithubRequestOptions(config: Config): RequestInit {
   const token =
     config.getOptionalString('catalog.processors.github.privateToken') ??
     config.getOptionalString('catalog.processors.githubApi.privateToken') ??
-    process.env.GITHUB_PRIVATE_TOKEN;
+    process.env.GITHUB_TOKEN;
 
   if (token) {
     headers.Authorization = `token ${token}`;
@@ -88,7 +98,7 @@ function getGitlabRequestOptions(config: Config): RequestInit {
   const token =
     config.getOptionalString('catalog.processors.gitlab.privateToken') ??
     config.getOptionalString('catalog.processors.gitlabApi.privateToken') ??
-    process.env.GITLAB_ACCESS_TOKEN;
+    process.env.GITLAB_TOKEN;
 
   if (token) {
     headers['PRIVATE-TOKEN'] = token;
@@ -99,6 +109,26 @@ function getGitlabRequestOptions(config: Config): RequestInit {
   };
 }
 
+function getAzureRequestOptions(config: Config): RequestInit {
+  const headers: HeadersInit = {};
+
+  const token =
+    config.getOptionalString('catalog.processors.azureApi.privateToken') ??
+    process.env.AZURE_TOKEN;
+
+  if (token !== '') {
+    headers.Authorization = `Basic ${Buffer.from(`:${token}`, 'utf8').toString(
+      'base64',
+    )}`;
+  }
+
+  const requestOptions: RequestInit = {
+    headers,
+  };
+
+  return requestOptions;
+}
+
 async function getGithubDefaultBranch(
   repositoryUrl: string,
   config: Config,
@@ -159,6 +189,42 @@ async function getGitlabDefaultBranch(
   }
 }
 
+async function getAzureDefaultBranch(
+  repositoryUrl: string,
+  config: Config,
+): Promise<string> {
+  const path = getAzureApiUrl(repositoryUrl).toString();
+
+  const options = getAzureRequestOptions(config);
+
+  try {
+    const urlResponse = await fetch(path, options);
+    if (!urlResponse.ok) {
+      throw new Error(
+        `Failed to load url: ${urlResponse.status} ${urlResponse.statusText}. Make sure you have permission to repository: ${repositoryUrl}`,
+      );
+    }
+    const urlResult = await urlResponse.json();
+
+    const idResponse = await fetch(urlResult.url, options);
+    if (!idResponse.ok) {
+      throw new Error(
+        `Failed to load url: ${idResponse.status} ${idResponse.statusText}. Make sure you have permission to repository: ${urlResult.repository.url}`,
+      );
+    }
+    const idResult = await idResponse.json();
+    const name = idResult.defaultBranch;
+
+    if (!name) {
+      throw new Error('Not found Azure DevOps default branch');
+    }
+
+    return name;
+  } catch (error) {
+    throw new Error(`Failed to get Azure DevOps default branch: ${error}`);
+  }
+}
+
 export const getDefaultBranch = async (
   repositoryUrl: string,
 ): Promise<string> => {
@@ -166,6 +232,7 @@ export const getDefaultBranch = async (
   const typeMapping = [
     { url: /github*/g, type: 'github' },
     { url: /gitlab*/g, type: 'gitlab' },
+    { url: /azure*/g, type: 'azure/api' },
   ];
 
   const type = typeMapping.filter(item => item.url.test(repositoryUrl))[0]
@@ -177,6 +244,8 @@ export const getDefaultBranch = async (
         return await getGithubDefaultBranch(repositoryUrl, config);
       case 'gitlab':
         return await getGitlabDefaultBranch(repositoryUrl, config);
+      case 'azure/api':
+        return await getAzureDefaultBranch(repositoryUrl, config);
 
       default:
         throw new Error('Failed to get repository type');

plugins/techdocs-backend/src/helpers.ts

@@ -76,6 +76,7 @@ export const getLocationForEntity = (
   switch (type) {
     case 'github':
     case 'gitlab':
+    case 'azure/api':
       return { type, target };
     case 'dir':
       if (path.isAbsolute(target)) return { type, target };
@@ -124,10 +125,12 @@ export const checkoutGitRepository = async (
   const user =
     process.env.GITHUB_PRIVATE_TOKEN_USER ||
     process.env.GITLAB_PRIVATE_TOKEN_USER ||
+    process.env.AZURE_PRIVATE_TOKEN_USER ||
     '';
   const token =
-    process.env.GITHUB_PRIVATE_TOKEN ||
+    process.env.GITHUB_TOKEN ||
     process.env.GITLAB_PRIVATE_TOKEN_USER ||
+    process.env.AZURE_TOKEN ||
     '';
 
   if (fs.existsSync(repositoryTmpPath)) {

plugins/techdocs-backend/src/service/router.ts

@@ -78,7 +78,7 @@ export async function createRouter({
     }
   });
 
-  router.get('/metadata/entity/:kind/:namespace/:name', async (req, res) => {
+  router.get('/metadata/entity/:namespace/:kind/:name', async (req, res) => {
     const baseUrl = config.getString('backend.baseUrl');
     const { kind, namespace, name } = req.params;
 
@@ -101,7 +101,7 @@ export async function createRouter({
     }
   });
 
-  router.get('/docs/:kind/:namespace/:name/*', async (req, res) => {
+  router.get('/docs/:namespace/:kind/:name/*', async (req, res) => {
     const storageUrl = config.getString('techdocs.storageUrl');
 
     const { kind, namespace, name } = req.params;

plugins/techdocs-backend/src/techdocs/stages/prepare/azure.test.ts

@@ -0,0 +1,63 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { getVoidLogger } from '@backstage/backend-common';
+import { AzurePreparer } from './azure';
+import { checkoutGitRepository } from '../../../helpers';
+
+function normalizePath(path: string) {
+  return path
+    .replace(/^[a-z]:/i, '')
+    .split('\\')
+    .join('/');
+}
+
+jest.mock('../../../helpers', () => ({
+  ...jest.requireActual<{}>('../../../helpers'),
+  checkoutGitRepository: jest.fn(() => '/tmp/backstage-repo/org/name/branch'),
+}));
+
+const createMockEntity = (annotations = {}) => {
+  return {
+    apiVersion: 'version',
+    kind: 'TestKind',
+    metadata: {
+      name: 'test-component-name',
+      annotations: {
+        ...annotations,
+      },
+    },
+  };
+};
+
+const logger = getVoidLogger();
+
+describe('Azure DevOps preparer', () => {
+  it('should prepare temp docs path from Azure DevOps repo', async () => {
+    const preparer = new AzurePreparer(logger);
+
+    const mockEntity = createMockEntity({
+      'backstage.io/techdocs-ref':
+        'azure/api:https://dev.azure.com/backstage-org/backstage-project/_git/template-repo?path=%2Ftemplate.yaml',
+    });
+
+    const tempDocsPath = await preparer.prepare(mockEntity);
+    expect(checkoutGitRepository).toHaveBeenCalledTimes(1);
+    expect(normalizePath(tempDocsPath)).toEqual(
+      '/tmp/backstage-repo/org/name/branch/template.yaml',
+    );
+  });
+});

plugins/techdocs-backend/src/techdocs/stages/prepare/azure.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import path from 'path';
+import { Entity } from '@backstage/catalog-model';
+import { InputError } from '@backstage/backend-common';
+import { PreparerBase } from './types';
+import parseGitUrl from 'git-url-parse';
+import {
+  parseReferenceAnnotation,
+  checkoutGitRepository,
+} from '../../../helpers';
+
+import { Logger } from 'winston';
+
+export class AzurePreparer implements PreparerBase {
+  private readonly logger: Logger;
+
+  constructor(logger: Logger) {
+    this.logger = logger;
+  }
+
+  async prepare(entity: Entity): Promise<string> {
+    const { type, target } = parseReferenceAnnotation(
+      'backstage.io/techdocs-ref',
+      entity,
+    );
+
+    if (type !== 'azure/api') {
+      throw new InputError(`Wrong target type: ${type}, should be 'azure/api'`);
+    }
+
+    try {
+      const repoPath = await checkoutGitRepository(target, this.logger);
+      const parsedGitLocation = parseGitUrl(target);
+
+      return path.join(repoPath, parsedGitLocation.filepath);
+    } catch (error) {
+      this.logger.debug(`Repo checkout failed with error ${error.message}`);
+      throw error;
+    }
+  }
+}

plugins/techdocs-backend/src/techdocs/stages/prepare/dir.ts

@@ -42,7 +42,8 @@ export class DirectoryPreparer implements PreparerBase {
     );
     switch (type) {
       case 'github':
-      case 'gitlab': {
+      case 'gitlab':
+      case 'azure/api': {
         const parsedGitLocation = parseGitUrl(target);
         const repoLocation = await checkoutGitRepository(target, this.logger);
 

plugins/techdocs-backend/src/techdocs/stages/prepare/index.ts

@@ -16,5 +16,6 @@
 export { DirectoryPreparer } from './dir';
 export { GithubPreparer } from './github';
 export { GitlabPreparer } from './gitlab';
+export { AzurePreparer } from './azure';
 export { Preparers } from './preparers';
 export type { PreparerBuilder, PreparerBase } from './types';

plugins/techdocs-backend/src/techdocs/stages/prepare/types.ts

@@ -30,4 +30,4 @@ export type PreparerBuilder = {
   get(entity: Entity): PreparerBase;
 };
 
-export type RemoteProtocol = 'dir' | 'github' | 'gitlab' | 'file';
+export type RemoteProtocol = 'dir' | 'github' | 'gitlab' | 'file' | 'azure/api';

plugins/techdocs-backend/src/techdocs/stages/publish/local.test.ts

@@ -53,7 +53,7 @@ describe('local publisher', () => {
 
     const resultDir = path.resolve(
       __dirname,
-      `../../../../static/docs/${mockEntity.kind}/default/${mockEntity.metadata.name}`,
+      `../../../../static/docs/default/${mockEntity.kind}/${mockEntity.metadata.name}`,
     );
 
     expect(fs.existsSync(resultDir)).toBeTruthy();

plugins/techdocs-backend/src/techdocs/stages/publish/local.ts

@@ -42,8 +42,8 @@ export class LocalPublish implements PublisherBase {
     const publishDir = resolvePackagePath(
       '@backstage/plugin-techdocs-backend',
       'static/docs',
-      entity.kind,
       entityNamespace,
+      entity.kind,
       entity.metadata.name,
     );