Merge pull request #4059 from RoadieHQ/gh-3811-repo-url-file-registration

Modifying component registration from Github to search for existing metadata file from repo URL

.changeset/eleven-tables-tease.md

@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-import': patch
+---
+
+Modifying import functionality to register existing catalog-info.yaml if one exists in given GitHub repository

plugins/catalog-import/package.json

@@ -32,8 +32,8 @@
   "dependencies": {
     "@backstage/catalog-model": "^0.6.1",
     "@backstage/core": "^0.4.4",
-    "@backstage/plugin-catalog": "^0.2.11",
     "@backstage/integration": "^0.2.0",
+    "@backstage/plugin-catalog": "^0.2.11",
     "@backstage/theme": "^0.2.2",
     "@material-ui/core": "^4.11.0",
     "@material-ui/icons": "^4.9.1",

plugins/catalog-import/src/api/CatalogImportApi.ts

@@ -30,6 +30,11 @@ export interface CatalogImportApi {
     fileContent: string;
     githubIntegrationConfig: GitHubIntegrationConfig;
   }): Promise<{ link: string; location: string }>;
+  checkForExistingCatalogInfo(options: {
+    owner: string;
+    repo: string;
+    githubIntegrationConfig: GitHubIntegrationConfig;
+  }): Promise<{ exists: boolean; url?: string }>;
   createRepositoryLocation(options: { location: string }): Promise<void>;
   generateEntityDefinitions(options: {
     repo: string;

plugins/catalog-import/src/api/CatalogImportClient.test.ts

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { CatalogImportClient } from './CatalogImportClient';
+
+jest.mock('@octokit/rest', () => ({
+  Octokit: jest.fn().mockImplementation(() => {
+    return {
+      repos: {
+        get: () =>
+          Promise.resolve({
+            data: {
+              default_branch: 'main',
+            },
+          }),
+      },
+      search: {
+        code: () =>
+          Promise.resolve({
+            data: {
+              total_count: 2,
+              items: [
+                { path: 'simple/path/catalog-info.yaml' },
+                { path: 'co/mple/x/path/catalog-info.yaml' },
+                { path: 'catalog-info.yaml' },
+              ],
+            },
+          }),
+      },
+    };
+  }),
+}));
+
+describe('CatalogImportClient', () => {
+  describe('checkForExistingCatalogInfo', () => {
+    const cic = new CatalogImportClient({
+      discoveryApi: { getBaseUrl: () => Promise.resolve('base') },
+      githubAuthApi: { getAccessToken: (_, __) => Promise.resolve('token') },
+      configApi: {} as any,
+    });
+    it('should return the closest-to-root catalog-info from multiple responses', async () => {
+      const respo = await cic.checkForExistingCatalogInfo({
+        owner: 'test-user',
+        repo: 'rest-repo',
+        githubIntegrationConfig: { host: 'https://github.com' },
+      });
+      expect(respo.exists).toBe(true);
+      expect(respo.url).toBe('blob/main/catalog-info.yaml');
+    });
+  });
+});

plugins/catalog-import/src/api/CatalogImportClient.ts

@@ -90,6 +90,52 @@ export class CatalogImportClient implements CatalogImportApi {
     }
   }
 
+  async checkForExistingCatalogInfo({
+    owner,
+    repo,
+    githubIntegrationConfig,
+  }: {
+    owner: string;
+    repo: string;
+    githubIntegrationConfig: GitHubIntegrationConfig;
+  }): Promise<{ exists: boolean; url?: string }> {
+    const token = await this.githubAuthApi.getAccessToken(['repo']);
+    const octo = new Octokit({
+      auth: token,
+      baseUrl: githubIntegrationConfig.apiBaseUrl,
+    });
+    const catalogFileName = 'catalog-info.yaml';
+    const query = `repo:${owner}/${repo}+filename:${catalogFileName}`;
+
+    const searchResult = await octo.search.code({ q: query }).catch(e => {
+      throw new Error(
+        formatHttpErrorMessage(
+          "Couldn't search repository for metadata file.",
+          e,
+        ),
+      );
+    });
+    const exists = searchResult.data.total_count > 0;
+    if (exists) {
+      const repoInformation = await octo.repos.get({ owner, repo }).catch(e => {
+        throw new Error(formatHttpErrorMessage("Couldn't fetch repo data", e));
+      });
+      const defaultBranch = repoInformation.data.default_branch;
+
+      // Github search sorts returned values with 'best match' using 'multiple factors to boost the most relevant item',
+      // aka magic.
+      // Sorting to use the shortest item, closest to the repository root.
+      const catalogInfoItem = searchResult.data.items
+        .map(it => it.path)
+        .sort((a, b) => a.length - b.length)[0];
+      return {
+        url: `blob/${defaultBranch}/${catalogInfoItem}`,
+        exists,
+      };
+    }
+    return { exists };
+  }
+
   async submitPrToRepo({
     owner,
     repo,

plugins/catalog-import/src/components/ImportComponentForm.test.tsx

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2021 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import React from 'react';
+import { renderInTestApp } from '@backstage/test-utils';
+import { RegisterComponentForm } from './ImportComponentForm';
+import {
+  ApiProvider,
+  ApiRegistry,
+  DiscoveryApi,
+  errorApiRef,
+} from '@backstage/core';
+import { catalogApiRef, CatalogClient } from '@backstage/plugin-catalog';
+import { catalogImportApiRef, CatalogImportClient } from '../api';
+import { fireEvent, waitFor, screen } from '@testing-library/react';
+
+describe('<RegisterComponentForm />', () => {
+  let apis: ApiRegistry;
+
+  const mockErrorApi: jest.Mocked<typeof errorApiRef.T> = {
+    post: jest.fn(),
+    error$: jest.fn(),
+  };
+
+  beforeEach(() => {
+    apis = ApiRegistry.from([
+      [catalogApiRef, new CatalogClient({ discoveryApi: {} as DiscoveryApi })],
+      [
+        catalogImportApiRef,
+        new CatalogImportClient({
+          discoveryApi: { getBaseUrl: () => Promise.resolve('base') },
+          githubAuthApi: {
+            getAccessToken: (_, __) => Promise.resolve('token'),
+          },
+          configApi: {} as any,
+        }),
+      ],
+      [errorApiRef, mockErrorApi],
+    ]);
+  });
+
+  async function renderSUT(
+    nextStep: () => void = () => {},
+    saveConfig: () => void = () => {},
+  ) {
+    return await renderInTestApp(
+      <ApiProvider apis={apis}>
+        <RegisterComponentForm
+          nextStep={nextStep}
+          saveConfig={saveConfig}
+          repository="GitHub"
+        />
+      </ApiProvider>,
+    );
+  }
+
+  it('Renders without exploding', async () => {
+    await renderSUT();
+    expect(
+      screen.getByPlaceholderText('https://github.com/backstage/backstage'),
+    ).toBeInTheDocument();
+  });
+
+  it('Should have basic URL validation for input', async () => {
+    await renderSUT();
+    await waitFor(() => {
+      fireEvent.input(
+        screen.getByPlaceholderText('https://github.com/backstage/backstage'),
+        { target: { value: 'not a url' } },
+      );
+    });
+    await waitFor(() => {
+      fireEvent.click(screen.getByText('Next'));
+    });
+    expect(screen.getByText('Must start with https://.')).toBeInTheDocument();
+  });
+});

plugins/catalog-import/src/components/ImportComponentForm.tsx

@@ -64,27 +64,45 @@ export const RegisterComponentForm = ({
 
   const isMounted = useMountedState();
   const errorApi = useApi(errorApiRef);
-  const { generateEntityDefinitions } = useGithubRepos();
+  const {
+    generateEntityDefinitions,
+    checkForExistingCatalogInfo,
+  } = useGithubRepos();
 
   const onSubmit = async (formData: Record<string, string>) => {
     const { componentLocation: target } = formData;
-    try {
-      if (!isMounted()) return;
-      const type = urlType(target);
+    async function saveCatalogFileConfig(target: string) {
+      const data = await catalogApi.addLocation({ target });
+      saveConfig({
+        type: 'file',
+        location: data.location.target,
+        config: data.entities,
+      });
+    }
 
-      if (type === 'tree') {
+    async function trySaveRepositoryConfig(target: string) {
+      const existingCatalog = await checkForExistingCatalogInfo(target);
+      if (existingCatalog.exists) {
+        const targetUrl = target.endsWith('/')
+          ? `${target}${existingCatalog.url}`
+          : `${target}/${existingCatalog.url}`;
+        await saveCatalogFileConfig(targetUrl);
+      } else {
         saveConfig({
-          type,
+          type: 'tree',
           location: target,
           config: await generateEntityDefinitions(target),
         });
+      }
+    }
+
+    try {
+      if (!isMounted()) return;
+      const type = urlType(target);
+      if (type === 'tree') {
+        await trySaveRepositoryConfig(target);
       } else {
-        const data = await catalogApi.addLocation({ target });
-        saveConfig({
-          type,
-          location: data.location.target,
-          config: data.entities,
-        });
+        await saveCatalogFileConfig(target);
       }
       nextStep();
     } catch (e) {

plugins/catalog-import/src/components/ImportComponentPage.test.tsx

@@ -0,0 +1,267 @@
+/*
+ * Copyright 2021 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import React from 'react';
+import { msw, renderInTestApp } from '@backstage/test-utils';
+import { ImportComponentPage } from './ImportComponentPage';
+import {
+  ApiProvider,
+  ApiRegistry,
+  configApiRef,
+  errorApiRef,
+} from '@backstage/core';
+import { catalogApiRef, CatalogClient } from '@backstage/plugin-catalog';
+import { catalogImportApiRef, CatalogImportClient } from '../api';
+
+import { fireEvent, screen, waitFor } from '@testing-library/react';
+
+import { rest } from 'msw';
+import { setupServer } from 'msw/node';
+
+let codeSearchMockResponse: () => Promise<{
+  data: {
+    total_count: number;
+    items: Array<{ path: string }>;
+  };
+}>;
+
+let findGithubConfigMockResponse = () => ({
+  host: 'test.localhost',
+  owner: 'someuser',
+});
+
+jest.mock('@backstage/integration', () => ({
+  readGitHubIntegrationConfigs: () => ({
+    find: findGithubConfigMockResponse,
+  }),
+}));
+
+jest.mock('@octokit/rest', () => ({
+  Octokit: jest.fn().mockImplementation(() => {
+    return {
+      repos: {
+        get: () =>
+          Promise.resolve({
+            data: {
+              default_branch: 'main',
+            },
+          }),
+      },
+      search: {
+        code: codeSearchMockResponse,
+      },
+    };
+  }),
+}));
+
+const OUR_GITHUB_TEST_REPO = 'https://github.com/someuser/somerepo';
+describe('<ImportComponentPage />', () => {
+  const server = setupServer();
+  msw.setupDefaultHandlers(server);
+
+  beforeEach(() => {
+    server.use(
+      rest.post('https://backend.localhost/locations', (_, res, ctx) => {
+        return res(
+          ctx.status(201),
+          ctx.json(require('../mocks/locations-POST-response.json')),
+        );
+      }),
+      rest.post('https://backend.localhost/analyze-location', (_, res, ctx) => {
+        return res(
+          ctx.json(require('../mocks/analyze-location-POST-response.json')),
+        );
+      }),
+    );
+  });
+  beforeAll(() => server.listen());
+  afterEach(() => server.resetHandlers());
+  afterAll(() => server.close());
+
+  let apis: ApiRegistry;
+
+  const mockErrorApi: jest.Mocked<typeof errorApiRef.T> = {
+    post: jest.fn(),
+    error$: jest.fn(),
+  };
+
+  beforeEach(() => {
+    const getBaseUrl = () => Promise.resolve('https://backend.localhost');
+    apis = ApiRegistry.from([
+      [
+        catalogApiRef,
+        new CatalogClient({
+          discoveryApi: { getBaseUrl },
+        }),
+      ],
+      [
+        catalogImportApiRef,
+        new CatalogImportClient({
+          discoveryApi: { getBaseUrl },
+          githubAuthApi: {
+            getAccessToken: (_, __) => Promise.resolve('token'),
+          },
+          configApi: {} as any,
+        }),
+      ],
+      [
+        configApiRef,
+        {
+          getOptional: () => 'Title',
+          getOptionalConfigArray: () => [],
+          has: () => true,
+          getConfig: () => ({
+            has: () => true,
+          }),
+        },
+      ],
+      [errorApiRef, mockErrorApi],
+    ]);
+  });
+
+  async function renderSUT() {
+    return await renderInTestApp(
+      <ApiProvider apis={apis}>
+        <ImportComponentPage catalogRouteRef={{ path: 'path', title: 'ttl' }} />
+      </ApiProvider>,
+    );
+  }
+
+  it('Should not explode on non-Github URLs', async () => {
+    findGithubConfigMockResponse = () => undefined!!;
+    await renderSUT();
+    await waitFor(() => {
+      fireEvent.input(
+        screen.getByPlaceholderText('https://github.com/backstage/backstage'),
+        {
+          target: {
+            value: 'https://test-git-provider.localhost/someuser/somerepo',
+          },
+        },
+      );
+    });
+
+    fireEvent.click(screen.getByText('Next'));
+    await waitFor(() => {
+      const firstStepInput = screen.queryByPlaceholderText(
+        'https://github.com/backstage/backstage',
+      );
+      expect(firstStepInput).not.toBeInTheDocument();
+    });
+  });
+
+  it('Should offer direct file import from non-Github URLs', async () => {
+    findGithubConfigMockResponse = () => undefined!!;
+    await renderSUT();
+    await waitFor(() => {
+      fireEvent.input(
+        screen.getByPlaceholderText('https://github.com/backstage/backstage'),
+        {
+          target: {
+            value:
+              'https://test-git-provider.localhost/someuser/somerepo/catalog-info.yaml',
+          },
+        },
+      );
+    });
+
+    fireEvent.click(screen.getByText('Next'));
+    await waitFor(() => {
+      const firstStepInput = screen.queryByPlaceholderText(
+        'https://github.com/backstage/backstage',
+      );
+      expect(firstStepInput).not.toBeInTheDocument();
+    });
+    expect(
+      screen.getByText(
+        'https://test-git-provider.localhost/someuser/somerepo/catalog-info.yaml',
+      ),
+    ).toBeInTheDocument();
+  });
+
+  it('Should use found yaml file directly and not create a pull request if GitHub api returns one', async () => {
+    findGithubConfigMockResponse = () => ({
+      host: 'test.localhost',
+      owner: 'someuser',
+    });
+    codeSearchMockResponse = () =>
+      Promise.resolve({
+        data: {
+          total_count: 3,
+          items: [
+            { path: 'simple/path/catalog-info.yaml' },
+            { path: 'co/mple/x/path/catalog-info.yaml' },
+            { path: 'catalog-info.yaml' },
+          ],
+        },
+      });
+    await renderSUT();
+    await waitFor(() => {
+      fireEvent.input(
+        screen.getByPlaceholderText('https://github.com/backstage/backstage'),
+        { target: { value: OUR_GITHUB_TEST_REPO } },
+      );
+    });
+
+    fireEvent.click(screen.getByText('Next'));
+    await waitFor(() => {
+      expect(
+        screen.getByText(
+          'https://github.com/someusername/somerepo/blob/master/src/catalog-info.yaml',
+        ),
+      ).toBeInTheDocument();
+
+      const pullReqText = screen.queryByText('pull request');
+      expect(pullReqText).not.toBeInTheDocument();
+    });
+  });
+
+  it('Should indicate a pull request creation when no yaml file found in the repo', async () => {
+    findGithubConfigMockResponse = () => ({
+      host: 'test.localhost',
+      owner: 'someuser',
+    });
+    codeSearchMockResponse = () =>
+      Promise.resolve({
+        data: {
+          total_count: 0,
+          items: [],
+        },
+      });
+    const { container } = await renderSUT();
+    await waitFor(() => {
+      fireEvent.input(
+        screen.getByPlaceholderText('https://github.com/backstage/backstage'),
+        { target: { value: OUR_GITHUB_TEST_REPO } },
+      );
+    });
+
+    fireEvent.click(screen.getByText('Next'));
+    await waitFor(() => {
+      expect(screen.getByText(OUR_GITHUB_TEST_REPO)).toBeInTheDocument();
+    });
+    const textNode = container
+      .querySelector('a[href="https://github.com/someuser/somerepo"]')
+      ?.closest('p');
+    expect(textNode?.innerHTML).toContain(
+      'Following config object will be submitted in a pull request to the repository',
+    );
+    expect(
+      screen.queryByText(
+        'https://github.com/someusername/somerepo/blob/master/src/catalog-info.yaml',
+      ),
+    ).not.toBeInTheDocument();
+  });
+});

plugins/catalog-import/src/components/ImportComponentPage.tsx

@@ -109,10 +109,11 @@ export const ImportComponentPage = ({
                 <React.Fragment>
                   <Typography variant="h6">GitHub Repo</Typography>
                   <Typography variant="body2" paragraph>
-                    If you already have code in a GitHub repository, enter the
-                    full URL to your repo and a new pull request with a sample
-                    Backstage metadata Entity File (
-                    <code>catalog-info.yaml</code>) will be opened for you.
+                    If you already have code in a GitHub repository without
+                    Backstage metadata file set up for it, enter the full URL to
+                    your repo and a new pull request with a sample Backstage
+                    metadata Entity File (<code>catalog-info.yaml</code>) will
+                    be opened for you.
                   </Typography>
                 </React.Fragment>
               )}

plugins/catalog-import/src/mocks/analyze-location-POST-response.json

@@ -0,0 +1,22 @@
+{
+  "existingEntityFiles": [],
+  "generateEntities": [
+    {
+      "entity": {
+        "apiVersion": "backstage.io/v1alpha1",
+        "kind": "Component",
+        "metadata": {
+          "name": "somerepo",
+          "annotations": {
+            "github.com/project-slug": "someuser/somerepo"
+          }
+        },
+        "spec": {
+          "type": "other",
+          "lifecycle": "unknown"
+        }
+      },
+      "fields": []
+    }
+  ]
+}

plugins/catalog-import/src/mocks/locations-POST-response.json

@@ -0,0 +1,39 @@
+{
+  "location": {
+    "id": "d4a64359-a709-4c91-a9de-0905a033bf22",
+    "type": "url",
+    "target": "https://github.com/someusername/somerepo/blob/master/src/catalog-info.yaml"
+  },
+  "entities": [
+    {
+      "metadata": {
+        "namespace": "default",
+        "annotations": {
+          "backstage.io/managed-by-location": "url:https://github.com/someusername/somerepo/blob/master/src/catalog-info.yaml",
+          "github.com/project-slug": "someusername/somerepo"
+        },
+        "name": "somerepo",
+        "uid": "e992d5ee-7c70-4316-90cf-325f1a0a5146",
+        "etag": "YWE2M2Q5MzgtNjdkNi00N2QwLWJkZjYtNDM0MTMzMDI4Y2I0",
+        "generation": 1
+      },
+      "apiVersion": "backstage.io/v1alpha1",
+      "kind": "Component",
+      "spec": {
+        "type": "other",
+        "lifecycle": "unknown",
+        "owner": "unknown"
+      },
+      "relations": [
+        {
+          "target": {
+            "kind": "group",
+            "namespace": "default",
+            "name": "unknown"
+          },
+          "type": "ownedBy"
+        }
+      ]
+    }
+  ]
+}

plugins/catalog-import/src/setupTests.ts

@@ -15,3 +15,4 @@
  */
 
 import '@testing-library/jest-dom';
+import 'cross-fetch/polyfill';

plugins/catalog-import/src/util/useGithubRepos.ts

@@ -22,18 +22,21 @@ import parseGitUrl from 'git-url-parse';
 
 // TODO: (O5ten) Refactor into a core API instead of direct usage like this
 // https://github.com/backstage/backstage/pull/3613#issuecomment-7408929430
-import { readGitHubIntegrationConfigs } from '@backstage/integration';
+import {
+  GitHubIntegrationConfig,
+  readGitHubIntegrationConfigs,
+} from '@backstage/integration';
 
 export function useGithubRepos() {
   const api = useApi(catalogImportApiRef);
   const config = useApi(configApiRef);
 
-  const submitPrToRepo = async (selectedRepo: ConfigSpec) => {
+  const getGithubIntegrationConfig = (location: string) => {
     const {
       name: repoName,
       owner: ownerName,
       resource: hostname,
-    } = parseGitUrl(selectedRepo.location);
+    } = parseGitUrl(location);
 
     const configs = readGitHubIntegrationConfigs(
       config.getOptionalConfigArray('integrations.github') ?? [],
@@ -41,9 +44,22 @@ export function useGithubRepos() {
     const githubIntegrationConfig = configs.find(v => v.host === hostname);
     if (!githubIntegrationConfig) {
       throw new Error(
-        `Unable to locate github-integration for repo-location: ${selectedRepo.location}`,
+        `Unable to locate github-integration for repo-location, ${location}`,
       );
     }
+    return {
+      repoName,
+      ownerName,
+      githubIntegrationConfig,
+    };
+  };
+
+  const submitPrToRepo = async (selectedRepo: ConfigSpec) => {
+    const {
+      repoName,
+      ownerName,
+      githubIntegrationConfig,
+    } = getGithubIntegrationConfig(selectedRepo.location);
     const submitPRResponse = await api
       .submitPrToRepo({
         owner: ownerName,
@@ -54,7 +70,7 @@ export function useGithubRepos() {
         githubIntegrationConfig,
       })
       .catch(e => {
-        throw new Error(`Failed to submit PR to repo:\n${e.message}`);
+        throw new Error(`Failed to submit PR to repo, ${e.message}`);
       });
 
     await api
@@ -62,14 +78,41 @@ export function useGithubRepos() {
         location: submitPRResponse.location,
       })
       .catch(e => {
-        throw new Error(`Failed to create repository location:\n${e.message}`);
+        throw new Error(`Failed to create repository location, ${e.message}`);
       });
 
     return submitPRResponse;
   };
 
+  const checkForExistingCatalogInfo = async (
+    location: string,
+  ): Promise<{ exists: boolean; url?: string }> => {
+    let githubConfig: {
+      repoName: string;
+      ownerName: string;
+      githubIntegrationConfig: GitHubIntegrationConfig;
+    };
+    try {
+      githubConfig = getGithubIntegrationConfig(location);
+    } catch (e) {
+      return { exists: false };
+    }
+    return await api
+      .checkForExistingCatalogInfo({
+        owner: githubConfig.ownerName,
+        repo: githubConfig.repoName,
+        githubIntegrationConfig: githubConfig.githubIntegrationConfig,
+      })
+      .catch(e => {
+        throw new Error(
+          `Failed to inspect repository for existing catalog-info.yaml, ${e.message}`,
+        );
+      });
+  };
+
   return {
     submitPrToRepo,
+    checkForExistingCatalogInfo,
     generateEntityDefinitions: (repo: string) =>
       api.generateEntityDefinitions({ repo }),
     addLocation: (location: string) =>