diff --git a/.changeset/eleven-tables-tease.md b/.changeset/eleven-tables-tease.md new file mode 100644 index 0000000000..3f95fbb9d2 --- /dev/null +++ b/.changeset/eleven-tables-tease.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-import': patch +--- + +Modifying import functionality to register existing catalog-info.yaml if one exists in given GitHub repository diff --git a/plugins/catalog-import/package.json b/plugins/catalog-import/package.json index 5e601ad896..c4e8d918d1 100644 --- a/plugins/catalog-import/package.json +++ b/plugins/catalog-import/package.json @@ -32,8 +32,8 @@ "dependencies": { "@backstage/catalog-model": "^0.6.1", "@backstage/core": "^0.4.4", - "@backstage/plugin-catalog": "^0.2.11", "@backstage/integration": "^0.2.0", + "@backstage/plugin-catalog": "^0.2.11", "@backstage/theme": "^0.2.2", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", diff --git a/plugins/catalog-import/src/api/CatalogImportApi.ts b/plugins/catalog-import/src/api/CatalogImportApi.ts index 99526a3eaa..5abb0e3e53 100644 --- a/plugins/catalog-import/src/api/CatalogImportApi.ts +++ b/plugins/catalog-import/src/api/CatalogImportApi.ts @@ -30,6 +30,11 @@ export interface CatalogImportApi { fileContent: string; githubIntegrationConfig: GitHubIntegrationConfig; }): Promise<{ link: string; location: string }>; + checkForExistingCatalogInfo(options: { + owner: string; + repo: string; + githubIntegrationConfig: GitHubIntegrationConfig; + }): Promise<{ exists: boolean; url?: string }>; createRepositoryLocation(options: { location: string }): Promise; generateEntityDefinitions(options: { repo: string; diff --git a/plugins/catalog-import/src/api/CatalogImportClient.test.ts b/plugins/catalog-import/src/api/CatalogImportClient.test.ts new file mode 100644 index 0000000000..0e0cf4b323 --- /dev/null +++ b/plugins/catalog-import/src/api/CatalogImportClient.test.ts @@ -0,0 +1,64 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { CatalogImportClient } from './CatalogImportClient'; + +jest.mock('@octokit/rest', () => ({ + Octokit: jest.fn().mockImplementation(() => { + return { + repos: { + get: () => + Promise.resolve({ + data: { + default_branch: 'main', + }, + }), + }, + search: { + code: () => + Promise.resolve({ + data: { + total_count: 2, + items: [ + { path: 'simple/path/catalog-info.yaml' }, + { path: 'co/mple/x/path/catalog-info.yaml' }, + { path: 'catalog-info.yaml' }, + ], + }, + }), + }, + }; + }), +})); + +describe('CatalogImportClient', () => { + describe('checkForExistingCatalogInfo', () => { + const cic = new CatalogImportClient({ + discoveryApi: { getBaseUrl: () => Promise.resolve('base') }, + githubAuthApi: { getAccessToken: (_, __) => Promise.resolve('token') }, + configApi: {} as any, + }); + it('should return the closest-to-root catalog-info from multiple responses', async () => { + const respo = await cic.checkForExistingCatalogInfo({ + owner: 'test-user', + repo: 'rest-repo', + githubIntegrationConfig: { host: 'https://github.com' }, + }); + expect(respo.exists).toBe(true); + expect(respo.url).toBe('blob/main/catalog-info.yaml'); + }); + }); +}); diff --git a/plugins/catalog-import/src/api/CatalogImportClient.ts b/plugins/catalog-import/src/api/CatalogImportClient.ts index a578f37a96..d1509c2105 100644 --- a/plugins/catalog-import/src/api/CatalogImportClient.ts +++ b/plugins/catalog-import/src/api/CatalogImportClient.ts @@ -90,6 +90,52 @@ export class CatalogImportClient implements CatalogImportApi { } } + async checkForExistingCatalogInfo({ + owner, + repo, + githubIntegrationConfig, + }: { + owner: string; + repo: string; + githubIntegrationConfig: GitHubIntegrationConfig; + }): Promise<{ exists: boolean; url?: string }> { + const token = await this.githubAuthApi.getAccessToken(['repo']); + const octo = new Octokit({ + auth: token, + baseUrl: githubIntegrationConfig.apiBaseUrl, + }); + const catalogFileName = 'catalog-info.yaml'; + const query = `repo:${owner}/${repo}+filename:${catalogFileName}`; + + const searchResult = await octo.search.code({ q: query }).catch(e => { + throw new Error( + formatHttpErrorMessage( + "Couldn't search repository for metadata file.", + e, + ), + ); + }); + const exists = searchResult.data.total_count > 0; + if (exists) { + const repoInformation = await octo.repos.get({ owner, repo }).catch(e => { + throw new Error(formatHttpErrorMessage("Couldn't fetch repo data", e)); + }); + const defaultBranch = repoInformation.data.default_branch; + + // Github search sorts returned values with 'best match' using 'multiple factors to boost the most relevant item', + // aka magic. + // Sorting to use the shortest item, closest to the repository root. + const catalogInfoItem = searchResult.data.items + .map(it => it.path) + .sort((a, b) => a.length - b.length)[0]; + return { + url: `blob/${defaultBranch}/${catalogInfoItem}`, + exists, + }; + } + return { exists }; + } + async submitPrToRepo({ owner, repo, diff --git a/plugins/catalog-import/src/components/ImportComponentForm.test.tsx b/plugins/catalog-import/src/components/ImportComponentForm.test.tsx new file mode 100644 index 0000000000..91a5208bee --- /dev/null +++ b/plugins/catalog-import/src/components/ImportComponentForm.test.tsx @@ -0,0 +1,89 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import React from 'react'; +import { renderInTestApp } from '@backstage/test-utils'; +import { RegisterComponentForm } from './ImportComponentForm'; +import { + ApiProvider, + ApiRegistry, + DiscoveryApi, + errorApiRef, +} from '@backstage/core'; +import { catalogApiRef, CatalogClient } from '@backstage/plugin-catalog'; +import { catalogImportApiRef, CatalogImportClient } from '../api'; +import { fireEvent, waitFor, screen } from '@testing-library/react'; + +describe('', () => { + let apis: ApiRegistry; + + const mockErrorApi: jest.Mocked = { + post: jest.fn(), + error$: jest.fn(), + }; + + beforeEach(() => { + apis = ApiRegistry.from([ + [catalogApiRef, new CatalogClient({ discoveryApi: {} as DiscoveryApi })], + [ + catalogImportApiRef, + new CatalogImportClient({ + discoveryApi: { getBaseUrl: () => Promise.resolve('base') }, + githubAuthApi: { + getAccessToken: (_, __) => Promise.resolve('token'), + }, + configApi: {} as any, + }), + ], + [errorApiRef, mockErrorApi], + ]); + }); + + async function renderSUT( + nextStep: () => void = () => {}, + saveConfig: () => void = () => {}, + ) { + return await renderInTestApp( + + + , + ); + } + + it('Renders without exploding', async () => { + await renderSUT(); + expect( + screen.getByPlaceholderText('https://github.com/backstage/backstage'), + ).toBeInTheDocument(); + }); + + it('Should have basic URL validation for input', async () => { + await renderSUT(); + await waitFor(() => { + fireEvent.input( + screen.getByPlaceholderText('https://github.com/backstage/backstage'), + { target: { value: 'not a url' } }, + ); + }); + await waitFor(() => { + fireEvent.click(screen.getByText('Next')); + }); + expect(screen.getByText('Must start with https://.')).toBeInTheDocument(); + }); +}); diff --git a/plugins/catalog-import/src/components/ImportComponentForm.tsx b/plugins/catalog-import/src/components/ImportComponentForm.tsx index 7434f07b10..c233be5a8c 100644 --- a/plugins/catalog-import/src/components/ImportComponentForm.tsx +++ b/plugins/catalog-import/src/components/ImportComponentForm.tsx @@ -64,27 +64,45 @@ export const RegisterComponentForm = ({ const isMounted = useMountedState(); const errorApi = useApi(errorApiRef); - const { generateEntityDefinitions } = useGithubRepos(); + const { + generateEntityDefinitions, + checkForExistingCatalogInfo, + } = useGithubRepos(); const onSubmit = async (formData: Record) => { const { componentLocation: target } = formData; - try { - if (!isMounted()) return; - const type = urlType(target); + async function saveCatalogFileConfig(target: string) { + const data = await catalogApi.addLocation({ target }); + saveConfig({ + type: 'file', + location: data.location.target, + config: data.entities, + }); + } - if (type === 'tree') { + async function trySaveRepositoryConfig(target: string) { + const existingCatalog = await checkForExistingCatalogInfo(target); + if (existingCatalog.exists) { + const targetUrl = target.endsWith('/') + ? `${target}${existingCatalog.url}` + : `${target}/${existingCatalog.url}`; + await saveCatalogFileConfig(targetUrl); + } else { saveConfig({ - type, + type: 'tree', location: target, config: await generateEntityDefinitions(target), }); + } + } + + try { + if (!isMounted()) return; + const type = urlType(target); + if (type === 'tree') { + await trySaveRepositoryConfig(target); } else { - const data = await catalogApi.addLocation({ target }); - saveConfig({ - type, - location: data.location.target, - config: data.entities, - }); + await saveCatalogFileConfig(target); } nextStep(); } catch (e) { diff --git a/plugins/catalog-import/src/components/ImportComponentPage.test.tsx b/plugins/catalog-import/src/components/ImportComponentPage.test.tsx new file mode 100644 index 0000000000..932f9aafa0 --- /dev/null +++ b/plugins/catalog-import/src/components/ImportComponentPage.test.tsx @@ -0,0 +1,267 @@ +/* + * Copyright 2021 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import React from 'react'; +import { msw, renderInTestApp } from '@backstage/test-utils'; +import { ImportComponentPage } from './ImportComponentPage'; +import { + ApiProvider, + ApiRegistry, + configApiRef, + errorApiRef, +} from '@backstage/core'; +import { catalogApiRef, CatalogClient } from '@backstage/plugin-catalog'; +import { catalogImportApiRef, CatalogImportClient } from '../api'; + +import { fireEvent, screen, waitFor } from '@testing-library/react'; + +import { rest } from 'msw'; +import { setupServer } from 'msw/node'; + +let codeSearchMockResponse: () => Promise<{ + data: { + total_count: number; + items: Array<{ path: string }>; + }; +}>; + +let findGithubConfigMockResponse = () => ({ + host: 'test.localhost', + owner: 'someuser', +}); + +jest.mock('@backstage/integration', () => ({ + readGitHubIntegrationConfigs: () => ({ + find: findGithubConfigMockResponse, + }), +})); + +jest.mock('@octokit/rest', () => ({ + Octokit: jest.fn().mockImplementation(() => { + return { + repos: { + get: () => + Promise.resolve({ + data: { + default_branch: 'main', + }, + }), + }, + search: { + code: codeSearchMockResponse, + }, + }; + }), +})); + +const OUR_GITHUB_TEST_REPO = 'https://github.com/someuser/somerepo'; +describe('', () => { + const server = setupServer(); + msw.setupDefaultHandlers(server); + + beforeEach(() => { + server.use( + rest.post('https://backend.localhost/locations', (_, res, ctx) => { + return res( + ctx.status(201), + ctx.json(require('../mocks/locations-POST-response.json')), + ); + }), + rest.post('https://backend.localhost/analyze-location', (_, res, ctx) => { + return res( + ctx.json(require('../mocks/analyze-location-POST-response.json')), + ); + }), + ); + }); + beforeAll(() => server.listen()); + afterEach(() => server.resetHandlers()); + afterAll(() => server.close()); + + let apis: ApiRegistry; + + const mockErrorApi: jest.Mocked = { + post: jest.fn(), + error$: jest.fn(), + }; + + beforeEach(() => { + const getBaseUrl = () => Promise.resolve('https://backend.localhost'); + apis = ApiRegistry.from([ + [ + catalogApiRef, + new CatalogClient({ + discoveryApi: { getBaseUrl }, + }), + ], + [ + catalogImportApiRef, + new CatalogImportClient({ + discoveryApi: { getBaseUrl }, + githubAuthApi: { + getAccessToken: (_, __) => Promise.resolve('token'), + }, + configApi: {} as any, + }), + ], + [ + configApiRef, + { + getOptional: () => 'Title', + getOptionalConfigArray: () => [], + has: () => true, + getConfig: () => ({ + has: () => true, + }), + }, + ], + [errorApiRef, mockErrorApi], + ]); + }); + + async function renderSUT() { + return await renderInTestApp( + + + , + ); + } + + it('Should not explode on non-Github URLs', async () => { + findGithubConfigMockResponse = () => undefined!!; + await renderSUT(); + await waitFor(() => { + fireEvent.input( + screen.getByPlaceholderText('https://github.com/backstage/backstage'), + { + target: { + value: 'https://test-git-provider.localhost/someuser/somerepo', + }, + }, + ); + }); + + fireEvent.click(screen.getByText('Next')); + await waitFor(() => { + const firstStepInput = screen.queryByPlaceholderText( + 'https://github.com/backstage/backstage', + ); + expect(firstStepInput).not.toBeInTheDocument(); + }); + }); + + it('Should offer direct file import from non-Github URLs', async () => { + findGithubConfigMockResponse = () => undefined!!; + await renderSUT(); + await waitFor(() => { + fireEvent.input( + screen.getByPlaceholderText('https://github.com/backstage/backstage'), + { + target: { + value: + 'https://test-git-provider.localhost/someuser/somerepo/catalog-info.yaml', + }, + }, + ); + }); + + fireEvent.click(screen.getByText('Next')); + await waitFor(() => { + const firstStepInput = screen.queryByPlaceholderText( + 'https://github.com/backstage/backstage', + ); + expect(firstStepInput).not.toBeInTheDocument(); + }); + expect( + screen.getByText( + 'https://test-git-provider.localhost/someuser/somerepo/catalog-info.yaml', + ), + ).toBeInTheDocument(); + }); + + it('Should use found yaml file directly and not create a pull request if GitHub api returns one', async () => { + findGithubConfigMockResponse = () => ({ + host: 'test.localhost', + owner: 'someuser', + }); + codeSearchMockResponse = () => + Promise.resolve({ + data: { + total_count: 3, + items: [ + { path: 'simple/path/catalog-info.yaml' }, + { path: 'co/mple/x/path/catalog-info.yaml' }, + { path: 'catalog-info.yaml' }, + ], + }, + }); + await renderSUT(); + await waitFor(() => { + fireEvent.input( + screen.getByPlaceholderText('https://github.com/backstage/backstage'), + { target: { value: OUR_GITHUB_TEST_REPO } }, + ); + }); + + fireEvent.click(screen.getByText('Next')); + await waitFor(() => { + expect( + screen.getByText( + 'https://github.com/someusername/somerepo/blob/master/src/catalog-info.yaml', + ), + ).toBeInTheDocument(); + + const pullReqText = screen.queryByText('pull request'); + expect(pullReqText).not.toBeInTheDocument(); + }); + }); + + it('Should indicate a pull request creation when no yaml file found in the repo', async () => { + findGithubConfigMockResponse = () => ({ + host: 'test.localhost', + owner: 'someuser', + }); + codeSearchMockResponse = () => + Promise.resolve({ + data: { + total_count: 0, + items: [], + }, + }); + const { container } = await renderSUT(); + await waitFor(() => { + fireEvent.input( + screen.getByPlaceholderText('https://github.com/backstage/backstage'), + { target: { value: OUR_GITHUB_TEST_REPO } }, + ); + }); + + fireEvent.click(screen.getByText('Next')); + await waitFor(() => { + expect(screen.getByText(OUR_GITHUB_TEST_REPO)).toBeInTheDocument(); + }); + const textNode = container + .querySelector('a[href="https://github.com/someuser/somerepo"]') + ?.closest('p'); + expect(textNode?.innerHTML).toContain( + 'Following config object will be submitted in a pull request to the repository', + ); + expect( + screen.queryByText( + 'https://github.com/someusername/somerepo/blob/master/src/catalog-info.yaml', + ), + ).not.toBeInTheDocument(); + }); +}); diff --git a/plugins/catalog-import/src/components/ImportComponentPage.tsx b/plugins/catalog-import/src/components/ImportComponentPage.tsx index 7e0d0ff92e..e284e95456 100644 --- a/plugins/catalog-import/src/components/ImportComponentPage.tsx +++ b/plugins/catalog-import/src/components/ImportComponentPage.tsx @@ -109,10 +109,11 @@ export const ImportComponentPage = ({ GitHub Repo - If you already have code in a GitHub repository, enter the - full URL to your repo and a new pull request with a sample - Backstage metadata Entity File ( - catalog-info.yaml) will be opened for you. + If you already have code in a GitHub repository without + Backstage metadata file set up for it, enter the full URL to + your repo and a new pull request with a sample Backstage + metadata Entity File (catalog-info.yaml) will + be opened for you. )} diff --git a/plugins/catalog-import/src/mocks/analyze-location-POST-response.json b/plugins/catalog-import/src/mocks/analyze-location-POST-response.json new file mode 100644 index 0000000000..eaadf1054a --- /dev/null +++ b/plugins/catalog-import/src/mocks/analyze-location-POST-response.json @@ -0,0 +1,22 @@ +{ + "existingEntityFiles": [], + "generateEntities": [ + { + "entity": { + "apiVersion": "backstage.io/v1alpha1", + "kind": "Component", + "metadata": { + "name": "somerepo", + "annotations": { + "github.com/project-slug": "someuser/somerepo" + } + }, + "spec": { + "type": "other", + "lifecycle": "unknown" + } + }, + "fields": [] + } + ] +} diff --git a/plugins/catalog-import/src/mocks/locations-POST-response.json b/plugins/catalog-import/src/mocks/locations-POST-response.json new file mode 100644 index 0000000000..c44a0f4d61 --- /dev/null +++ b/plugins/catalog-import/src/mocks/locations-POST-response.json @@ -0,0 +1,39 @@ +{ + "location": { + "id": "d4a64359-a709-4c91-a9de-0905a033bf22", + "type": "url", + "target": "https://github.com/someusername/somerepo/blob/master/src/catalog-info.yaml" + }, + "entities": [ + { + "metadata": { + "namespace": "default", + "annotations": { + "backstage.io/managed-by-location": "url:https://github.com/someusername/somerepo/blob/master/src/catalog-info.yaml", + "github.com/project-slug": "someusername/somerepo" + }, + "name": "somerepo", + "uid": "e992d5ee-7c70-4316-90cf-325f1a0a5146", + "etag": "YWE2M2Q5MzgtNjdkNi00N2QwLWJkZjYtNDM0MTMzMDI4Y2I0", + "generation": 1 + }, + "apiVersion": "backstage.io/v1alpha1", + "kind": "Component", + "spec": { + "type": "other", + "lifecycle": "unknown", + "owner": "unknown" + }, + "relations": [ + { + "target": { + "kind": "group", + "namespace": "default", + "name": "unknown" + }, + "type": "ownedBy" + } + ] + } + ] +} diff --git a/plugins/catalog-import/src/setupTests.ts b/plugins/catalog-import/src/setupTests.ts index 825bcd4115..aea2220869 100644 --- a/plugins/catalog-import/src/setupTests.ts +++ b/plugins/catalog-import/src/setupTests.ts @@ -15,3 +15,4 @@ */ import '@testing-library/jest-dom'; +import 'cross-fetch/polyfill'; diff --git a/plugins/catalog-import/src/util/useGithubRepos.ts b/plugins/catalog-import/src/util/useGithubRepos.ts index 89448a1be1..a2d299b9f4 100644 --- a/plugins/catalog-import/src/util/useGithubRepos.ts +++ b/plugins/catalog-import/src/util/useGithubRepos.ts @@ -22,18 +22,21 @@ import parseGitUrl from 'git-url-parse'; // TODO: (O5ten) Refactor into a core API instead of direct usage like this // https://github.com/backstage/backstage/pull/3613#issuecomment-7408929430 -import { readGitHubIntegrationConfigs } from '@backstage/integration'; +import { + GitHubIntegrationConfig, + readGitHubIntegrationConfigs, +} from '@backstage/integration'; export function useGithubRepos() { const api = useApi(catalogImportApiRef); const config = useApi(configApiRef); - const submitPrToRepo = async (selectedRepo: ConfigSpec) => { + const getGithubIntegrationConfig = (location: string) => { const { name: repoName, owner: ownerName, resource: hostname, - } = parseGitUrl(selectedRepo.location); + } = parseGitUrl(location); const configs = readGitHubIntegrationConfigs( config.getOptionalConfigArray('integrations.github') ?? [], @@ -41,9 +44,22 @@ export function useGithubRepos() { const githubIntegrationConfig = configs.find(v => v.host === hostname); if (!githubIntegrationConfig) { throw new Error( - `Unable to locate github-integration for repo-location: ${selectedRepo.location}`, + `Unable to locate github-integration for repo-location, ${location}`, ); } + return { + repoName, + ownerName, + githubIntegrationConfig, + }; + }; + + const submitPrToRepo = async (selectedRepo: ConfigSpec) => { + const { + repoName, + ownerName, + githubIntegrationConfig, + } = getGithubIntegrationConfig(selectedRepo.location); const submitPRResponse = await api .submitPrToRepo({ owner: ownerName, @@ -54,7 +70,7 @@ export function useGithubRepos() { githubIntegrationConfig, }) .catch(e => { - throw new Error(`Failed to submit PR to repo:\n${e.message}`); + throw new Error(`Failed to submit PR to repo, ${e.message}`); }); await api @@ -62,14 +78,41 @@ export function useGithubRepos() { location: submitPRResponse.location, }) .catch(e => { - throw new Error(`Failed to create repository location:\n${e.message}`); + throw new Error(`Failed to create repository location, ${e.message}`); }); return submitPRResponse; }; + const checkForExistingCatalogInfo = async ( + location: string, + ): Promise<{ exists: boolean; url?: string }> => { + let githubConfig: { + repoName: string; + ownerName: string; + githubIntegrationConfig: GitHubIntegrationConfig; + }; + try { + githubConfig = getGithubIntegrationConfig(location); + } catch (e) { + return { exists: false }; + } + return await api + .checkForExistingCatalogInfo({ + owner: githubConfig.ownerName, + repo: githubConfig.repoName, + githubIntegrationConfig: githubConfig.githubIntegrationConfig, + }) + .catch(e => { + throw new Error( + `Failed to inspect repository for existing catalog-info.yaml, ${e.message}`, + ); + }); + }; + return { submitPrToRepo, + checkForExistingCatalogInfo, generateEntityDefinitions: (repo: string) => api.generateEntityDefinitions({ repo }), addLocation: (location: string) =>