Merge pull request #13800 from RoadieHQ/kissmikijr/use-configured-creds-catalog-wizard

[catalog-import] Move github code search to the backend
2022-10-11 09:21:35 +01:00
parent 0cc6914ae3 9552527c37
commit 847caff9c8
33 changed files with 954 additions and 257 deletions
@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-node': minor
+---
+
+Deprecated the `LocationSpec` type. It got moved from this package to the `@backstage/plugin-catalog-common` so make sure imports are updated.
@@ -0,0 +1,13 @@
+---
+'@backstage/plugin-catalog-backend': minor
+---
+
+Added a new method `addLocationAnalyzers` to the `CatalogBuilder`. With this you can add location analyzers to your catalog. These analyzers will be used by the /analyze-location endpoint to decide if the provided URL contains any catalog-info.yaml files already or not.
+
+Moved the following types from this package to `@backstage/plugin-catalog-backend`.
+
+- AnalyzeLocationResponse
+- AnalyzeLocationRequest
+- AnalyzeLocationExistingEntity
+- AnalyzeLocationGenerateEntity
+- AnalyzeLocationEntityField
@@ -0,0 +1,11 @@
+---
+'@backstage/plugin-catalog-common': patch
+---
+
+Moved the following types from `@backstage/plugin-catalog-backend` to this package.
+
+- AnalyzeLocationResponse
+- AnalyzeLocationRequest
+- AnalyzeLocationExistingEntity
+- AnalyzeLocationGenerateEntity
+- AnalyzeLocationEntityField
@@ -0,0 +1,21 @@
+---
+'@backstage/plugin-catalog-import': minor
+---
+
+**Breaking**
+Moved the code search for the existing catalog-info.yaml files to the backend from the frontend. It means it will use the configured GitHub integration's credentials.
+
+Add the following to your `CatalogBuilder` to have the repo URL ingestion working again.
+
+```ts
+// catalog.ts
+import { GitHubLocationAnalyzer } from '@backstage/plugin-catalog-backend-module-github';
+...
+  builder.addLocationAnalyzers(
+    new GitHubLocationAnalyzer({
+      discovery: env.discovery,
+      config: env.config,
+    }),
+  );
+...
+```
@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-backend-module-github': patch
+---
+
+Added `GitHubLocationAnalyzer`. This can be used to add to the `CatalogBuilder`. When added this will be used by `RepoLocationAnalyzer` to figure out if the given URL that you are trying to import from the /catalog-import page already contains catalog-info.yaml files.
@@ -3,17 +3,21 @@
 > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).

 ```ts
+import { AnalyzeOptions } from '@backstage/plugin-catalog-backend';
 import { BackendFeature } from '@backstage/backend-plugin-api';
 import { CatalogProcessor } from '@backstage/plugin-catalog-backend';
 import { CatalogProcessorEmit } from '@backstage/plugin-catalog-backend';
 import { Config } from '@backstage/config';
+import { Entity } from '@backstage/catalog-model';
 import { EntityProvider } from '@backstage/plugin-catalog-backend';
 import { EntityProviderConnection } from '@backstage/plugin-catalog-backend';
 import { GithubCredentialsProvider } from '@backstage/integration';
 import { GitHubIntegrationConfig } from '@backstage/integration';
 import { LocationSpec } from '@backstage/plugin-catalog-backend';
 import { Logger } from 'winston';
+import { PluginEndpointDiscovery } from '@backstage/backend-common';
 import { ScmIntegrationRegistry } from '@backstage/integration';
+import { ScmLocationAnalyzer } from '@backstage/plugin-catalog-backend';
 import { TaskRunner } from '@backstage/backend-tasks';
 import { TaskScheduleDefinition } from '@backstage/backend-tasks';

@@ -70,6 +74,30 @@ export type GithubEntityProviderCatalogModuleOptions = {
  schedule?: TaskScheduleDefinition;
 };

+// @public (undocumented)
+export class GitHubLocationAnalyzer implements ScmLocationAnalyzer {
+  constructor(options: GitHubLocationAnalyzerOptions);
+  // (undocumented)
+  analyze({ url, catalogFilename }: AnalyzeOptions): Promise<{
+    existing: {
+      location: {
+        type: string;
+        target: string;
+      };
+      isRegistered: boolean;
+      entity: Entity;
+    }[];
+  }>;
+  // (undocumented)
+  supports(url: string): boolean;
+}
+
+// @public (undocumented)
+export type GitHubLocationAnalyzerOptions = {
+  config: Config;
+  discovery: PluginEndpointDiscovery;
+};
+
 // @public
 export type GithubMultiOrgConfig = Array<{
  name: string;
@@ -36,6 +36,7 @@
    "@backstage/backend-common": "workspace:^",
    "@backstage/backend-plugin-api": "workspace:^",
    "@backstage/backend-tasks": "workspace:^",
+    "@backstage/catalog-client": "workspace:^",
    "@backstage/catalog-model": "workspace:^",
    "@backstage/config": "workspace:^",
    "@backstage/errors": "workspace:^",
@@ -44,6 +45,8 @@
    "@backstage/plugin-catalog-node": "workspace:^",
    "@backstage/types": "workspace:^",
    "@octokit/graphql": "^5.0.0",
+    "@octokit/rest": "^19.0.3",
+    "git-url-parse": "^13.0.0",
    "lodash": "^4.17.21",
    "msw": "^0.47.0",
    "node-fetch": "^2.6.7",
@@ -0,0 +1,156 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const octokit = {
+  search: {
+    code: jest.fn(),
+  },
+  repos: {
+    get: jest.fn(),
+  },
+};
+
+jest.mock('@octokit/rest', () => {
+  class Octokit {
+    constructor() {
+      return octokit;
+    }
+  }
+  return { Octokit };
+});
+
+import { PluginEndpointDiscovery } from '@backstage/backend-common';
+import { GitHubLocationAnalyzer } from './GitHubLocationAnalyzer';
+import { setupRequestMockHandlers } from '@backstage/backend-test-utils';
+import { setupServer } from 'msw/node';
+import { rest } from 'msw';
+import { ConfigReader } from '@backstage/config';
+
+const server = setupServer();
+
+describe('GitHubLocationAnalyzer', () => {
+  const mockDiscoveryApi: jest.Mocked<PluginEndpointDiscovery> = {
+    getBaseUrl: jest.fn().mockResolvedValue('http://localhost:7007'),
+    getExternalBaseUrl: jest.fn(),
+  };
+  const config = new ConfigReader({
+    integrations: {
+      github: [
+        {
+          host: 'h.com',
+          token: 't',
+        },
+      ],
+    },
+  });
+
+  setupRequestMockHandlers(server);
+
+  beforeEach(() => {
+    server.use(
+      rest.post('http://localhost:7007/locations', async (_, res, ctx) => {
+        return res(
+          ctx.status(201),
+          ctx.json({
+            location: 'test',
+            exists: false,
+            entities: [
+              {
+                apiVersion: 'backstage.io/v1alpha1',
+                kind: 'Location',
+                metadata: {
+                  name: 'test-entity',
+                },
+                spec: {
+                  type: 'url',
+                  target: 'whatever',
+                },
+              },
+              {
+                apiVersion: 'backstage.io/v1alpha1',
+                kind: 'Component',
+                metadata: {
+                  title: 'Test Entity',
+                  name: 'test-entity-2',
+                  description: 'The expected description 2',
+                },
+                spec: {
+                  type: 'some-type',
+                  lifecycle: 'experimental',
+                  owner: 'someone',
+                },
+              },
+            ],
+          }),
+        );
+      }),
+    );
+
+    octokit.repos.get.mockResolvedValue({
+      data: { default_branch: 'my_default_branch' },
+    });
+  });
+
+  it('should analyze', async () => {
+    octokit.search.code.mockImplementation((opts: { q: string }) => {
+      if (opts.q === 'filename:catalog-info.yaml repo:foo/bar') {
+        return Promise.resolve({
+          data: { items: [{ path: 'catalog-info.yaml' }], total_count: 1 },
+        });
+      }
+      return Promise.reject();
+    });
+
+    const analyzer = new GitHubLocationAnalyzer({
+      discovery: mockDiscoveryApi,
+      config,
+    });
+    const result = await analyzer.analyze({
+      url: 'https://github.com/foo/bar',
+    });
+
+    expect(result.existing[0].isRegistered).toBeFalsy();
+    expect(result.existing[0].location).toEqual({
+      type: 'url',
+      target:
+        'https://github.com/foo/bar/blob/my_default_branch/catalog-info.yaml',
+    });
+  });
+  it('should use the provided entity filename for search', async () => {
+    octokit.search.code.mockImplementation((opts: { q: string }) => {
+      if (opts.q === 'filename:anvil.yaml repo:foo/bar') {
+        return Promise.resolve({
+          data: { items: [{ path: 'anvil.yaml' }], total_count: 1 },
+        });
+      }
+      return Promise.reject();
+    });
+
+    const analyzer = new GitHubLocationAnalyzer({
+      discovery: mockDiscoveryApi,
+      config,
+    });
+    const result = await analyzer.analyze({
+      url: 'https://github.com/foo/bar',
+      catalogFilename: 'anvil.yaml',
+    });
+
+    expect(result.existing[0].location).toEqual({
+      type: 'url',
+      target: 'https://github.com/foo/bar/blob/my_default_branch/anvil.yaml',
+    });
+  });
+});
@@ -0,0 +1,103 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { CatalogApi, CatalogClient } from '@backstage/catalog-client';
+import { ScmIntegrations } from '@backstage/integration';
+import { Octokit } from '@octokit/rest';
+import { trimEnd } from 'lodash';
+import parseGitUrl from 'git-url-parse';
+import {
+  AnalyzeOptions,
+  ScmLocationAnalyzer,
+} from '@backstage/plugin-catalog-backend';
+import { PluginEndpointDiscovery } from '@backstage/backend-common';
+import { Config } from '@backstage/config';
+
+/** @public */
+export type GitHubLocationAnalyzerOptions = {
+  config: Config;
+  discovery: PluginEndpointDiscovery;
+};
+/** @public */
+export class GitHubLocationAnalyzer implements ScmLocationAnalyzer {
+  private readonly catalogClient: CatalogApi;
+  private readonly config: Config;
+
+  constructor(options: GitHubLocationAnalyzerOptions) {
+    this.config = options.config;
+    this.catalogClient = new CatalogClient({ discoveryApi: options.discovery });
+  }
+  supports(url: string) {
+    const integrations = ScmIntegrations.fromConfig(this.config);
+    const integration = integrations.byUrl(url);
+    return integration?.type === 'github';
+  }
+  async analyze({ url, catalogFilename }: AnalyzeOptions) {
+    const { owner, name: repo } = parseGitUrl(url);
+
+    const catalogFile = catalogFilename || 'catalog-info.yaml';
+
+    const query = `filename:${catalogFile} repo:${owner}/${repo}`;
+
+    const integration = ScmIntegrations.fromConfig(this.config).github.byUrl(
+      url,
+    );
+    if (!integration) {
+      throw new Error('Make sure you have a GitHub integration configured');
+    }
+
+    const octokitClient = new Octokit({
+      auth: integration.config.token,
+      baseUrl: integration.config.apiBaseUrl,
+    });
+
+    const searchResult = await octokitClient.search
+      .code({ q: query })
+      .catch(e => {
+        throw new Error(`Couldn't search repository for metadata file, ${e}`);
+      });
+
+    const exists = searchResult.data.total_count > 0;
+    if (exists) {
+      const repoInformation = await octokitClient.repos
+        .get({ owner, repo })
+        .catch(e => {
+          throw new Error(`Couldn't fetch repo data, ${e}`);
+        });
+      const defaultBranch = repoInformation.data.default_branch;
+
+      const result = await Promise.all(
+        searchResult.data.items
+          .map(i => `${trimEnd(url, '/')}/blob/${defaultBranch}/${i.path}`)
+          .map(async target => {
+            const addLocationResult = await this.catalogClient.addLocation({
+              type: 'url',
+              target,
+              dryRun: true,
+            });
+            return addLocationResult.entities.map(e => ({
+              location: { type: 'url', target },
+              isRegistered: !!addLocationResult.exists,
+              entity: e,
+            }));
+          }),
+      );
+
+      return { existing: result.flat() };
+    }
+    return { existing: [] };
+  }
+}
@@ -29,3 +29,5 @@ export type { GitHubOrgEntityProviderOptions } from './providers/GitHubOrgEntity
 export type { GithubMultiOrgConfig } from './lib';
 export { githubEntityProviderCatalogModule } from './module';
 export type { GithubEntityProviderCatalogModuleOptions } from './module';
+export { GitHubLocationAnalyzer } from './analyzers/GitHubLocationAnalyzer';
+export type { GitHubLocationAnalyzerOptions } from './analyzers/GitHubLocationAnalyzer';
@@ -5,6 +5,11 @@
 ```ts
 /// <reference types="node" />

+import { AnalyzeLocationEntityField as AnalyzeLocationEntityField_2 } from '@backstage/plugin-catalog-common';
+import { AnalyzeLocationExistingEntity as AnalyzeLocationExistingEntity_2 } from '@backstage/plugin-catalog-common';
+import { AnalyzeLocationGenerateEntity as AnalyzeLocationGenerateEntity_2 } from '@backstage/plugin-catalog-common';
+import { AnalyzeLocationRequest as AnalyzeLocationRequest_2 } from '@backstage/plugin-catalog-common';
+import { AnalyzeLocationResponse as AnalyzeLocationResponse_2 } from '@backstage/plugin-catalog-common';
 import { BackendFeature } from '@backstage/backend-plugin-api';
 import { CatalogApi } from '@backstage/catalog-client';
 import { CatalogEntityDocument } from '@backstage/plugin-catalog-common';
@@ -51,39 +56,25 @@ import { TokenManager } from '@backstage/backend-common';
 import { UrlReader } from '@backstage/backend-common';
 import { Validators } from '@backstage/catalog-model';

-// @public (undocumented)
-export type AnalyzeLocationEntityField = {
-  field: string;
-  state:
-    | 'analysisSuggestedValue'
-    | 'analysisSuggestedNoValue'
-    | 'needsUserInput';
-  value: string | null;
-  description: string;
-};
+// @public @deprecated
+export type AnalyzeLocationEntityField = AnalyzeLocationEntityField_2;

-// @public
-export type AnalyzeLocationExistingEntity = {
-  location: LocationSpec;
-  isRegistered: boolean;
-  entity: Entity;
-};
+// @public @deprecated
+export type AnalyzeLocationExistingEntity = AnalyzeLocationExistingEntity_2;

-// @public
-export type AnalyzeLocationGenerateEntity = {
-  entity: RecursivePartial<Entity>;
-  fields: AnalyzeLocationEntityField[];
-};
+// @public @deprecated
+export type AnalyzeLocationGenerateEntity = AnalyzeLocationGenerateEntity_2;
+
+// @public @deprecated (undocumented)
+export type AnalyzeLocationRequest = AnalyzeLocationRequest_2;
+
+// @public @deprecated (undocumented)
+export type AnalyzeLocationResponse = AnalyzeLocationResponse_2;

 // @public (undocumented)
-export type AnalyzeLocationRequest = {
-  location: LocationSpec;
-};
-
-// @public (undocumented)
-export type AnalyzeLocationResponse = {
-  existingEntityFiles: AnalyzeLocationExistingEntity[];
-  generateEntities: AnalyzeLocationGenerateEntity[];
+export type AnalyzeOptions = {
+  url: string;
+  catalogFilename?: string;
 };

 // @public (undocumented)
@@ -133,6 +124,9 @@ export class CatalogBuilder {
  addEntityProvider(
    ...providers: Array<EntityProvider | Array<EntityProvider>>
  ): CatalogBuilder;
+  addLocationAnalyzers(
+    ...analyzers: Array<ScmLocationAnalyzer | Array<ScmLocationAnalyzer>>
+  ): CatalogBuilder;
  // @alpha
  addPermissionRules(
    ...permissionRules: Array<
@@ -531,6 +525,14 @@ export type ProcessingIntervalFunction = () => number;

 export { processingResult };

+// @public (undocumented)
+export type ScmLocationAnalyzer = {
+  supports(url: string): boolean;
+  analyze(options: AnalyzeOptions): Promise<{
+    existing: AnalyzeLocationExistingEntity[];
+  }>;
+};
+
 // @public (undocumented)
 export class UrlReaderProcessor implements CatalogProcessor {
  constructor(options: { reader: UrlReader; logger: Logger });
@@ -18,34 +18,32 @@ import { Logger } from 'winston';
 import parseGitUrl from 'git-url-parse';
 import { Entity } from '@backstage/catalog-model';
 import { ScmIntegrationRegistry } from '@backstage/integration';
+import { LocationAnalyzer, ScmLocationAnalyzer } from './types';
 import {
  AnalyzeLocationRequest,
  AnalyzeLocationResponse,
-  LocationAnalyzer,
-} from './types';
+} from '@backstage/plugin-catalog-common';

 export class RepoLocationAnalyzer implements LocationAnalyzer {
  private readonly logger: Logger;
  private readonly scmIntegrations: ScmIntegrationRegistry;
+  private readonly analyzers: ScmLocationAnalyzer[];

-  constructor(logger: Logger, scmIntegrations: ScmIntegrationRegistry) {
+  constructor(
+    logger: Logger,
+    scmIntegrations: ScmIntegrationRegistry,
+    analyzers: ScmLocationAnalyzer[],
+  ) {
    this.logger = logger;
    this.scmIntegrations = scmIntegrations;
+    this.analyzers = analyzers;
  }
  async analyzeLocation(
    request: AnalyzeLocationRequest,
  ): Promise<AnalyzeLocationResponse> {
-    const { owner, name } = parseGitUrl(request.location.target);
-    const entity: Entity = {
-      apiVersion: 'backstage.io/v1alpha1',
-      kind: 'Component',
-      metadata: {
-        name: name,
-      },
-      spec: { type: 'other', lifecycle: 'unknown' },
-    };
-
    const integration = this.scmIntegrations.byUrl(request.location.target);
+    const { owner, name } = parseGitUrl(request.location.target);
+
    let annotationPrefix;
    switch (integration?.type) {
      case 'azure':
@@ -64,6 +62,33 @@ export class RepoLocationAnalyzer implements LocationAnalyzer {
        break;
    }

+    const analyzer = this.analyzers.find(a =>
+      a.supports(request.location.target),
+    );
+    if (analyzer) {
+      const analyzerResult = await analyzer.analyze({
+        url: request.location.target,
+      });
+      if (analyzerResult.existing.length > 0) {
+        this.logger.debug(
+          `entity for ${request.location.target} already exists.`,
+        );
+        return {
+          existingEntityFiles: analyzerResult.existing,
+          generateEntities: [],
+        };
+      }
+    }
+
+    const entity: Entity = {
+      apiVersion: 'backstage.io/v1alpha1',
+      kind: 'Component',
+      metadata: {
+        name: name,
+      },
+      spec: { type: 'other', lifecycle: 'unknown' },
+    };
+
    if (annotationPrefix) {
      entity.metadata.annotations = {
        [`${annotationPrefix}/project-slug`]: `${owner}/${name}`,
@@ -21,4 +21,6 @@ export type {
  AnalyzeLocationRequest,
  AnalyzeLocationResponse,
  LocationAnalyzer,
+  ScmLocationAnalyzer,
+  AnalyzeOptions,
 } from './types';
@@ -14,9 +14,57 @@
 * limitations under the License.
 */

-import { Entity } from '@backstage/catalog-model';
-import { RecursivePartial } from '../util/RecursivePartial';
-import { LocationSpec } from '@backstage/plugin-catalog-node';
+import {
+  AnalyzeLocationRequest as NonDeprecatedAnalyzeLocationRequest,
+  AnalyzeLocationResponse as NonDeprecatedAnalyzeLocationResponse,
+  AnalyzeLocationExistingEntity as NonDeprecatedAnalyzeLocationExistingEntity,
+  AnalyzeLocationGenerateEntity as NonDeprecatedAnalyzeLocationGenerateEntity,
+  AnalyzeLocationEntityField as NonDeprecatedAnalyzeLocationEntityField,
+} from '@backstage/plugin-catalog-common';
+
+/**
+ * @public
+ * @deprecated use the same type from `@backstage/plugin-catalog-common` instead
+ */
+export type AnalyzeLocationRequest = NonDeprecatedAnalyzeLocationRequest;
+/**
+ * @public
+ * @deprecated use the same type from `@backstage/plugin-catalog-common` instead
+ */
+export type AnalyzeLocationResponse = NonDeprecatedAnalyzeLocationResponse;
+
+/**
+ * If the folder pointed to already contained catalog info yaml files, they are
+ * read and emitted like this so that the frontend can inform the user that it
+ * located them and can make sure to register them as well if they weren't
+ * already
+ * @public
+ * @deprecated use the same type from `@backstage/plugin-catalog-common` instead
+ */
+export type AnalyzeLocationExistingEntity =
+  NonDeprecatedAnalyzeLocationExistingEntity;
+/**
+ * This is some form of representation of what the analyzer could deduce.
+ * We should probably have a chat about how this can best be conveyed to
+ * the frontend. It'll probably contain a (possibly incomplete) entity, plus
+ * enough info for the frontend to know what form data to show to the user
+ * for overriding/completing the info.
+ * @public
+ * @deprecated use the same type from `@backstage/plugin-catalog-common` instead
+ */
+export type AnalyzeLocationGenerateEntity =
+  NonDeprecatedAnalyzeLocationGenerateEntity;
+
+/**
+ *
+ * This is where I get really vague. Something like this perhaps? Or it could be
+ * something like a json-schema that contains enough info for the frontend to
+ * be able to present a form and explanations
+ * @public
+ * @deprecated use the same type from `@backstage/plugin-catalog-common` instead
+ */
+export type AnalyzeLocationEntityField =
+  NonDeprecatedAnalyzeLocationEntityField;

 /** @public */
 export type LocationAnalyzer = {
@@ -30,71 +78,19 @@ export type LocationAnalyzer = {
    location: AnalyzeLocationRequest,
  ): Promise<AnalyzeLocationResponse>;
 };
-
 /** @public */
-export type AnalyzeLocationRequest = {
-  location: LocationSpec;
+export type AnalyzeOptions = {
+  url: string;
+  catalogFilename?: string;
 };

 /** @public */
-export type AnalyzeLocationResponse = {
-  existingEntityFiles: AnalyzeLocationExistingEntity[];
-  generateEntities: AnalyzeLocationGenerateEntity[];
-};
-
-/**
- * If the folder pointed to already contained catalog info yaml files, they are
- * read and emitted like this so that the frontend can inform the user that it
- * located them and can make sure to register them as well if they weren't
- * already
- * @public
- */
-export type AnalyzeLocationExistingEntity = {
-  location: LocationSpec;
-  isRegistered: boolean;
-  entity: Entity;
-};
-
-/**
- * This is some form of representation of what the analyzer could deduce.
- * We should probably have a chat about how this can best be conveyed to
- * the frontend. It'll probably contain a (possibly incomplete) entity, plus
- * enough info for the frontend to know what form data to show to the user
- * for overriding/completing the info.
- * @public
- */
-export type AnalyzeLocationGenerateEntity = {
-  // Some form of partial representation of the entity
-  entity: RecursivePartial<Entity>;
-  // Lists the suggestions that the user may want to override
-  fields: AnalyzeLocationEntityField[];
-};
-
-// This is where I get really vague. Something like this perhaps? Or it could be
-// something like a json-schema that contains enough info for the frontend to
-// be able to present a form and explanations
-/** @public */
-export type AnalyzeLocationEntityField = {
-  /**
-   * e.g. "spec.owner"? The frontend needs to know how to "inject" the field into the
-   * entity again if the user wants to change it
-   */
-  field: string;
-
-  /** The outcome of the analysis for this particular field */
-  state:
-    | 'analysisSuggestedValue'
-    | 'analysisSuggestedNoValue'
-    | 'needsUserInput';
-
-  // If the analysis did suggest a value, this is where it would be. Not sure if we want
-  // to limit this to strings or if we want it to be any JsonValue
-  value: string | null;
-  /**
-   * A text to show to the user to inform about the choices made. Like, it could say
-   * "Found a CODEOWNERS file that covers this target, so we suggest leaving this
-   * field empty; which would currently make it owned by X" where X is taken from the
-   * codeowners file.
-   */
-  description: string;
+export type ScmLocationAnalyzer = {
+  /** The method that decides if this analyzer can work with the provided url */
+  supports(url: string): boolean;
+  /** This function can return an array of already existing entities */
+  analyze(options: AnalyzeOptions): Promise<{
+    /** Existing entities in the analyzed location */
+    existing: AnalyzeLocationExistingEntity[];
+  }>;
 };
@@ -56,7 +56,7 @@ import {
  yamlPlaceholderResolver,
 } from '../modules/core/PlaceholderProcessor';
 import { defaultEntityDataParser } from '../modules/util/parse';
-import { LocationAnalyzer } from '../ingestion/types';
+import { LocationAnalyzer, ScmLocationAnalyzer } from '../ingestion/types';
 import { CatalogProcessingEngine } from '../processing';
 import { DefaultProcessingDatabase } from '../database/DefaultProcessingDatabase';
 import { applyDatabaseMigrations } from '../database/migrations';
@@ -115,6 +115,10 @@ export type CatalogEnvironment = {
 *   after the processors' pre-processing steps. All policies are given the
 *   chance to inspect the entity, and all of them have to pass in order for
 *   the entity to be considered valid from an overall point of view.
+ * - Location analyzers can be added. These are responsible for analyzing
+ *   repositories when onboarding them into the catalog, by finding
+ *   catalog-info.yaml files and other artifacts that can help automatically
+ *   register or create catalog data on the user's behalf.
 * - Placeholder resolvers can be replaced or added. These run on the raw
 *   structured data between the parsing and pre-processing steps, to replace
 *   dollar-prefixed entries with their actual values (like $file).
@@ -135,6 +139,7 @@ export class CatalogBuilder {
  private fieldFormatValidators: Partial<Validators>;
  private entityProviders: EntityProvider[];
  private processors: CatalogProcessor[];
+  private locationAnalyzers: ScmLocationAnalyzer[];
  private processorsReplace: boolean;
  private parser: CatalogProcessorParser | undefined;
  private onProcessingError?: (event: {
@@ -165,6 +170,7 @@ export class CatalogBuilder {
    this.fieldFormatValidators = {};
    this.entityProviders = [];
    this.processors = [];
+    this.locationAnalyzers = [];
    this.processorsReplace = false;
    this.parser = undefined;
    this.permissionRules = Object.values(catalogPermissionRules);
@@ -335,6 +341,21 @@ export class CatalogBuilder {
    ];
  }

+  /**
+   * Adds Location Analyzers. These are responsible for analyzing
+   * repositories when onboarding them into the catalog, by finding
+   * catalog-info.yaml files and other artifacts that can help automatically
+   * register or create catalog data on the user's behalf.
+   *
+   * @param locationAnalyzers - One or more location analyzers
+   */
+  addLocationAnalyzers(
+    ...analyzers: Array<ScmLocationAnalyzer | Array<ScmLocationAnalyzer>>
+  ): CatalogBuilder {
+    this.locationAnalyzers.push(...analyzers.flat());
+    return this;
+  }
+
  /**
   * Sets up the catalog to use a custom parser for entity data.
   *
@@ -478,7 +499,8 @@ export class CatalogBuilder {
    );

    const locationAnalyzer =
-      this.locationAnalyzer ?? new RepoLocationAnalyzer(logger, integrations);
+      this.locationAnalyzer ??
+      new RepoLocationAnalyzer(logger, integrations, this.locationAnalyzers);
    const locationService = new AuthorizedLocationService(
      new DefaultLocationService(locationStore, orchestrator, {
        allowedLocationTypes: this.allowedLocationType,
@@ -229,9 +229,15 @@ export async function createRouter(
    router.post('/analyze-location', async (req, res) => {
      const body = await validateRequestBody(
        req,
-        z.object({ location: locationInput }),
+        z.object({
+          location: locationInput,
+          catalogFilename: z.string().optional(),
+        }),
      );
-      const schema = z.object({ location: locationInput });
+      const schema = z.object({
+        location: locationInput,
+        catalogFilename: z.string().optional(),
+      });
      const output = await locationAnalyzer.analyzeLocation(schema.parse(body));
      res.status(200).json(output);
    });
@@ -4,9 +4,46 @@

 ```ts
 import { BasicPermission } from '@backstage/plugin-permission-common';
+import { Entity } from '@backstage/catalog-model';
 import { IndexableDocument } from '@backstage/plugin-search-common';
 import { ResourcePermission } from '@backstage/plugin-permission-common';

+// @public (undocumented)
+export type AnalyzeLocationEntityField = {
+  field: string;
+  state:
+    | 'analysisSuggestedValue'
+    | 'analysisSuggestedNoValue'
+    | 'needsUserInput';
+  value: string | null;
+  description: string;
+};
+
+// @public
+export type AnalyzeLocationExistingEntity = {
+  location: LocationSpec;
+  isRegistered: boolean;
+  entity: Entity;
+};
+
+// @public
+export type AnalyzeLocationGenerateEntity = {
+  entity: RecursivePartial<Entity>;
+  fields: AnalyzeLocationEntityField[];
+};
+
+// @public (undocumented)
+export type AnalyzeLocationRequest = {
+  location: LocationSpec;
+  catalogFilename?: string;
+};
+
+// @public (undocumented)
+export type AnalyzeLocationResponse = {
+  existingEntityFiles: AnalyzeLocationExistingEntity[];
+  generateEntities: AnalyzeLocationGenerateEntity[];
+};
+
 // @alpha
 export const catalogEntityCreatePermission: BasicPermission;

@@ -55,6 +92,13 @@ export const catalogPermissions: (
  | ResourcePermission<'catalog-entity'>
 )[];

+// @public
+export type LocationSpec = {
+  type: string;
+  target: string;
+  presence?: 'optional' | 'required';
+};
+
 // @alpha
 export const RESOURCE_TYPE_CATALOG_ENTITY = 'catalog-entity';
 ```
@@ -33,6 +33,7 @@
    "clean": "backstage-cli package clean"
  },
  "dependencies": {
+    "@backstage/catalog-model": "workspace:^",
    "@backstage/plugin-permission-common": "workspace:^",
    "@backstage/plugin-search-common": "workspace:^"
  },
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Holds the entity location information.
+ *
+ * @remarks
+ *
+ *  `presence` flag: when using repo importer plugin, location is being created before the component yaml file is merged to the main branch.
+ *  This flag is then set to indicate that the file can be not present.
+ *  default value: 'required'.
+ *
+ * @public
+ */
+export type LocationSpec = {
+  type: string;
+  target: string;
+  presence?: 'optional' | 'required';
+};
@@ -35,3 +35,5 @@ export {
 export type { CatalogEntityPermission } from './permissions';

 export * from './search';
+export * from './ingestion';
+export type { LocationSpec } from './common';
@@ -0,0 +1,88 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { LocationSpec } from '../common';
+import { Entity } from '@backstage/catalog-model';
+import { RecursivePartial } from './RecursivePartial';
+
+/** @public */
+export type AnalyzeLocationRequest = {
+  location: LocationSpec;
+  catalogFilename?: string;
+};
+
+/** @public */
+export type AnalyzeLocationResponse = {
+  existingEntityFiles: AnalyzeLocationExistingEntity[];
+  generateEntities: AnalyzeLocationGenerateEntity[];
+};
+
+/**
+ * If the folder pointed to already contained catalog info yaml files, they are
+ * read and emitted like this so that the frontend can inform the user that it
+ * located them and can make sure to register them as well if they weren't
+ * already
+ * @public
+ */
+export type AnalyzeLocationExistingEntity = {
+  location: LocationSpec;
+  isRegistered: boolean;
+  entity: Entity;
+};
+
+/**
+ * This is some form of representation of what the analyzer could deduce.
+ * We should probably have a chat about how this can best be conveyed to
+ * the frontend. It'll probably contain a (possibly incomplete) entity, plus
+ * enough info for the frontend to know what form data to show to the user
+ * for overriding/completing the info.
+ * @public
+ */
+export type AnalyzeLocationGenerateEntity = {
+  // Some form of partial representation of the entity
+  entity: RecursivePartial<Entity>;
+  // Lists the suggestions that the user may want to override
+  fields: AnalyzeLocationEntityField[];
+};
+
+// This is where I get really vague. Something like this perhaps? Or it could be
+// something like a json-schema that contains enough info for the frontend to
+// be able to present a form and explanations
+/** @public */
+export type AnalyzeLocationEntityField = {
+  /**
+   * e.g. "spec.owner"? The frontend needs to know how to "inject" the field into the
+   * entity again if the user wants to change it
+   */
+  field: string;
+
+  /** The outcome of the analysis for this particular field */
+  state:
+    | 'analysisSuggestedValue'
+    | 'analysisSuggestedNoValue'
+    | 'needsUserInput';
+
+  // If the analysis did suggest a value, this is where it would be. Not sure if we want
+  // to limit this to strings or if we want it to be any JsonValue
+  value: string | null;
+  /**
+   * A text to show to the user to inform about the choices made. Like, it could say
+   * "Found a CODEOWNERS file that covers this target, so we suggest leaving this
+   * field empty; which would currently make it owned by X" where X is taken from the
+   * codeowners file.
+   */
+  description: string;
+};
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { RecursivePartial } from './RecursivePartial';
+
+describe('RecursivePartial', () => {
+  it('is recursive', () => {
+    type X = {
+      required: {
+        required: string;
+      };
+    };
+    const x: RecursivePartial<X> = {
+      required: {},
+    };
+    expect(x).toEqual({ required: {} });
+  });
+});
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2020 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Makes all keys of an entire hierarchy optional.
+ * @ignore
+ */
+export type RecursivePartial<T> = {
+  [P in keyof T]?: T[P] extends (infer U)[]
+    ? RecursivePartial<U>[]
+    : T[P] extends object
+    ? RecursivePartial<T[P]>
+    : T[P];
+};
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export type {
+  AnalyzeLocationResponse,
+  AnalyzeLocationRequest,
+  AnalyzeLocationExistingEntity,
+  AnalyzeLocationGenerateEntity,
+  AnalyzeLocationEntityField,
+} from './LocationAnalyzer';
@@ -40,6 +40,7 @@
    "@backstage/errors": "workspace:^",
    "@backstage/integration": "workspace:^",
    "@backstage/integration-react": "workspace:^",
+    "@backstage/plugin-catalog-common": "workspace:^",
    "@backstage/plugin-catalog-react": "workspace:^",
    "@material-ui/core": "^4.12.2",
    "@material-ui/icons": "^4.9.1",
@@ -297,7 +297,7 @@ describe('CatalogImportClient', () => {
    it('should find locations from github', async () => {
      (new Octokit().search.code as any as jest.Mock).mockResolvedValueOnce({
        data: {
-          total_count: 2,
+          total_count: 3,
          items: [
            { path: 'simple/path/catalog-info.yaml' },
            { path: 'co/mple/x/path/catalog-info.yaml' },
@@ -305,24 +305,72 @@ describe('CatalogImportClient', () => {
          ],
        },
      });
-
-      catalogApi.addLocation.mockImplementation(async ({ type, target }) => ({
-        location: {
-          id: 'id-0',
-          type: type ?? 'url',
-          target,
-        },
-        entities: [
-          {
-            apiVersion: '1',
-            kind: 'k',
-            metadata: {
-              name: 'e',
-              namespace: 'n',
+      server.use(
+        rest.post(`${mockBaseUrl}/analyze-location`, (req, res, ctx) => {
+          expect(req.body).toEqual({
+            location: {
+              target: 'https://github.com/backstage/backstage',
+              type: 'url',
            },
-          },
-        ],
-      }));
+          });
+
+          return res(
+            ctx.json({
+              generateEntities: [],
+              existingEntityFiles: [
+                {
+                  isRegistered: false,
+                  location: {
+                    type: 'url',
+                    target:
+                      'https://github.com/backstage/backstage/blob/main/simple/path/catalog-info.yaml',
+                  },
+                  entity: {
+                    apiVersion: '1',
+                    kind: 'k',
+                    metadata: {
+                      name: 'e',
+                      namespace: 'n',
+                    },
+                  },
+                },
+                {
+                  isRegistered: false,
+                  location: {
+                    type: 'url',
+                    target:
+                      'https://github.com/backstage/backstage/blob/main/co/mple/x/path/catalog-info.yaml',
+                  },
+                  entity: {
+                    apiVersion: '1',
+                    kind: 'k',
+                    metadata: {
+                      name: 'e',
+                      namespace: 'n',
+                    },
+                  },
+                },
+                {
+                  isRegistered: false,
+                  location: {
+                    type: 'url',
+                    target:
+                      'https://github.com/backstage/backstage/blob/main/catalog-info.yaml',
+                  },
+                  entity: {
+                    apiVersion: '1',
+                    kind: 'k',
+                    metadata: {
+                      name: 'e',
+                      namespace: 'n',
+                    },
+                  },
+                },
+              ],
+            }),
+          );
+        }),
+      );

      await expect(
        catalogImportClient.analyzeUrl(
@@ -332,16 +380,19 @@ describe('CatalogImportClient', () => {
        locations: [
          {
            entities: [{ kind: 'k', name: 'e', namespace: 'n' }],
+            exists: false,
            target:
              'https://github.com/backstage/backstage/blob/main/simple/path/catalog-info.yaml',
          },
          {
            entities: [{ kind: 'k', name: 'e', namespace: 'n' }],
+            exists: false,
            target:
              'https://github.com/backstage/backstage/blob/main/co/mple/x/path/catalog-info.yaml',
          },
          {
            entities: [{ kind: 'k', name: 'e', namespace: 'n' }],
+            exists: false,
            target:
              'https://github.com/backstage/backstage/blob/main/catalog-info.yaml',
          },
@@ -426,31 +477,57 @@ describe('CatalogImportClient', () => {
        }),
      );

-      catalogApi.addLocation.mockImplementation(async ({ type, target }) => ({
-        location: {
-          id: 'id-0',
-          type: type ?? 'url',
-          target,
-        },
-        entities: [
-          {
-            apiVersion: '1',
-            kind: 'Location',
-            metadata: {
-              name: 'my-entity',
-              namespace: 'my-namespace',
+      server.use(
+        rest.post(`${mockBaseUrl}/analyze-location`, (req, res, ctx) => {
+          expect(req.body).toEqual({
+            location: {
+              target: 'https://github.com/acme-corp/our-awesome-api',
+              type: 'url',
            },
-          },
-          {
-            apiVersion: '1',
-            kind: 'Component',
-            metadata: {
-              name: 'my-entity',
-              namespace: 'my-namespace',
-            },
-          },
-        ],
-      }));
+            catalogFilename: 'anvil.yaml',
+          });
+
+          return res(
+            ctx.json({
+              generateEntities: [],
+              existingEntityFiles: [
+                {
+                  isRegistered: false,
+                  location: {
+                    type: 'url',
+                    target:
+                      'https://github.com/acme-corp/our-awesome-api/blob/main/anvil.yaml',
+                  },
+                  entity: {
+                    apiVersion: '1',
+                    kind: 'Location',
+                    metadata: {
+                      name: 'my-entity',
+                      namespace: 'my-namespace',
+                    },
+                  },
+                },
+                {
+                  isRegistered: false,
+                  location: {
+                    type: 'url',
+                    target:
+                      'https://github.com/acme-corp/our-awesome-api/blob/main/anvil.yaml',
+                  },
+                  entity: {
+                    apiVersion: '1',
+                    kind: 'Component',
+                    metadata: {
+                      name: 'my-entity',
+                      namespace: 'my-namespace',
+                    },
+                  },
+                },
+              ],
+            }),
+          );
+        }),
+      );

      await expect(
        catalogImportClient.analyzeUrl(repositoryUrl),
@@ -470,6 +547,7 @@ describe('CatalogImportClient', () => {
              },
            ],
            target: `${repositoryUrl}/blob/main/${entityFilename}`,
+            exists: false,
          },
        ],
        type: 'locations',
@@ -15,7 +15,6 @@
 */

 import { CatalogApi } from '@backstage/catalog-client';
-import { CompoundEntityRef } from '@backstage/catalog-model';
 import {
  ConfigApi,
  DiscoveryApi,
@@ -28,11 +27,11 @@ import {
 import { ScmAuthApi } from '@backstage/integration-react';
 import { Octokit } from '@octokit/rest';
 import { Base64 } from 'js-base64';
-import { PartialEntity } from '../types';
 import { AnalyzeResult, CatalogImportApi } from './CatalogImportApi';
 import { getGithubIntegrationConfig } from './GitHub';
-import { trimEnd } from 'lodash';
 import { getBranchName, getCatalogFilename } from '../components/helpers';
+import { AnalyzeLocationResponse } from '@backstage/plugin-catalog-common';
+import { CompoundEntityRef } from '@backstage/catalog-model';

 /**
 * The default implementation of the {@link CatalogImportApi}.
@@ -105,16 +104,40 @@ export class CatalogImportClient implements CatalogImportApi {
      );
    }

-    // TODO: this could be part of the analyze-location endpoint
-    const locations = await this.checkGitHubForExistingCatalogInfo({
-      ...ghConfig,
-      url,
+    const analyzation = await this.analyzeLocation({
+      repo: url,
    });

-    if (locations.length > 0) {
+    if (analyzation.existingEntityFiles.length > 0) {
+      const locations = analyzation.existingEntityFiles.reduce<
+        Record<
+          string,
+          {
+            target: string;
+            exists?: boolean;
+            entities: CompoundEntityRef[];
+          }
+        >
+      >((state, curr) => {
+        state[curr.location.target] = {
+          target: curr.location.target,
+          exists: curr.isRegistered,
+          entities: [
+            ...(curr.location.target in state
+              ? state[curr.location.target].entities
+              : []),
+            {
+              name: curr.entity.metadata.name,
+              namespace: curr.entity.metadata.namespace ?? 'default',
+              kind: curr.entity.kind,
+            },
+          ],
+        };
+        return state;
+      }, {});
      return {
        type: 'locations',
-        locations,
+        locations: Object.values(locations),
      };
    }

@@ -122,9 +145,7 @@ export class CatalogImportClient implements CatalogImportApi {
      type: 'repository',
      integrationType: 'github',
      url: url,
-      generatedEntities: await this.generateEntityDefinitions({
-        repo: url,
-      }),
+      generatedEntities: analyzation.generateEntities.map(x => x.entity),
    };
  }

@@ -174,9 +195,9 @@ the component will become available.\n\nFor more information, read an \
  }

  // TODO: this could be part of the catalog api
-  private async generateEntityDefinitions(options: {
+  private async analyzeLocation(options: {
    repo: string;
-  }): Promise<PartialEntity[]> {
+  }): Promise<AnalyzeLocationResponse> {
    const { token } = await this.identityApi.getCredentials();
    const response = await fetch(
      `${await this.discoveryApi.getBaseUrl('catalog')}/analyze-location`,
@@ -188,6 +209,13 @@ the component will become available.\n\nFor more information, read an \
        method: 'POST',
        body: JSON.stringify({
          location: { type: 'url', target: options.repo },
+          ...(this.configApi.getOptionalString(
+            'catalog.import.entityFilename',
+          ) && {
+            catalogFilename: this.configApi.getOptionalString(
+              'catalog.import.entityFilename',
+            ),
+          }),
        }),
      },
    ).catch(e => {
@@ -200,69 +228,7 @@ the component will become available.\n\nFor more information, read an \
    }

    const payload = await response.json();
-    return payload.generateEntities.map((x: any) => x.entity);
-  }
-
-  // TODO: this response should better be part of the analyze-locations response and scm-independent / implemented per scm
-  private async checkGitHubForExistingCatalogInfo(options: {
-    url: string;
-    owner: string;
-    repo: string;
-    githubIntegrationConfig: GitHubIntegrationConfig;
-  }): Promise<
-    Array<{
-      target: string;
-      entities: CompoundEntityRef[];
-    }>
-  > {
-    const { url, owner, repo, githubIntegrationConfig } = options;
-
-    const { token } = await this.scmAuthApi.getCredentials({ url });
-    const octo = new Octokit({
-      auth: token,
-      baseUrl: githubIntegrationConfig.apiBaseUrl,
-    });
-    const catalogFilename = getCatalogFilename(this.configApi);
-    const query = `repo:${owner}/${repo}+filename:${catalogFilename}`;
-
-    const searchResult = await octo.search.code({ q: query }).catch(e => {
-      throw new Error(
-        formatHttpErrorMessage(
-          "Couldn't search repository for metadata file.",
-          e,
-        ),
-      );
-    });
-    const exists = searchResult.data.total_count > 0;
-    if (exists) {
-      const repoInformation = await octo.repos.get({ owner, repo }).catch(e => {
-        throw new Error(formatHttpErrorMessage("Couldn't fetch repo data", e));
-      });
-      const defaultBranch = repoInformation.data.default_branch;
-
-      return await Promise.all(
-        searchResult.data.items
-          .map(i => `${trimEnd(url, '/')}/blob/${defaultBranch}/${i.path}`)
-          .map(async target => {
-            const result = await this.catalogApi.addLocation({
-              type: 'url',
-              target,
-              dryRun: true,
-            });
-            return {
-              target,
-              exists: result.exists,
-              entities: result.entities.map(e => ({
-                kind: e.kind,
-                namespace: e.metadata.namespace ?? 'default',
-                name: e.metadata.name,
-              })),
-            };
-          }),
-      );
-    }
-
-    return [];
+    return payload;
  }

  // TODO: extract this function and implement for non-github
@@ -10,6 +10,7 @@ import { CompoundEntityRef } from '@backstage/catalog-model';
 import { Entity } from '@backstage/catalog-model';
 import { ExtensionPoint } from '@backstage/backend-plugin-api';
 import { JsonValue } from '@backstage/types';
+import { LocationSpec as LocationSpec_2 } from '@backstage/plugin-catalog-common';
 import { ServiceRef } from '@backstage/backend-plugin-api';

 // @alpha (undocumented)
@@ -31,7 +32,7 @@ export const catalogProcessingExtensionPoint: ExtensionPoint<CatalogProcessingEx
 export type CatalogProcessor = {
  getProcessorName(): string;
  readLocation?(
-    location: LocationSpec,
+    location: LocationSpec_2,
    optional: boolean,
    emit: CatalogProcessorEmit,
    parser: CatalogProcessorParser,
@@ -39,15 +40,15 @@ export type CatalogProcessor = {
  ): Promise<boolean>;
  preProcessEntity?(
    entity: Entity,
-    location: LocationSpec,
+    location: LocationSpec_2,
    emit: CatalogProcessorEmit,
-    originLocation: LocationSpec,
+    originLocation: LocationSpec_2,
    cache: CatalogProcessorCache,
  ): Promise<Entity>;
  validateEntityKind?(entity: Entity): Promise<boolean>;
  postProcessEntity?(
    entity: Entity,
-    location: LocationSpec,
+    location: LocationSpec_2,
    emit: CatalogProcessorEmit,
    cache: CatalogProcessorCache,
  ): Promise<Entity>;
@@ -66,26 +67,26 @@ export type CatalogProcessorEmit = (generated: CatalogProcessorResult) => void;
 export type CatalogProcessorEntityResult = {
  type: 'entity';
  entity: Entity;
-  location: LocationSpec;
+  location: LocationSpec_2;
 };

 // @public (undocumented)
 export type CatalogProcessorErrorResult = {
  type: 'error';
  error: Error;
-  location: LocationSpec;
+  location: LocationSpec_2;
 };

 // @public (undocumented)
 export type CatalogProcessorLocationResult = {
  type: 'location';
-  location: LocationSpec;
+  location: LocationSpec_2;
 };

 // @public
 export type CatalogProcessorParser = (options: {
  data: Buffer;
-  location: LocationSpec;
+  location: LocationSpec_2;
 }) => AsyncIterable<CatalogProcessorResult>;

 // @public (undocumented)
@@ -153,30 +154,26 @@ export type EntityRelationSpec = {
  target: CompoundEntityRef;
 };

-// @public
-export type LocationSpec = {
-  type: string;
-  target: string;
-  presence?: 'optional' | 'required';
-};
+// @public @deprecated
+export type LocationSpec = LocationSpec_2;

 // @public
 export const processingResult: Readonly<{
  readonly notFoundError: (
-    atLocation: LocationSpec,
+    atLocation: LocationSpec_2,
    message: string,
  ) => CatalogProcessorResult;
  readonly inputError: (
-    atLocation: LocationSpec,
+    atLocation: LocationSpec_2,
    message: string,
  ) => CatalogProcessorResult;
  readonly generalError: (
-    atLocation: LocationSpec,
+    atLocation: LocationSpec_2,
    message: string,
  ) => CatalogProcessorResult;
-  readonly location: (newLocation: LocationSpec) => CatalogProcessorResult;
+  readonly location: (newLocation: LocationSpec_2) => CatalogProcessorResult;
  readonly entity: (
-    atLocation: LocationSpec,
+    atLocation: LocationSpec_2,
    newEntity: Entity,
  ) => CatalogProcessorResult;
  readonly relation: (spec: EntityRelationSpec) => CatalogProcessorResult;
@@ -28,6 +28,7 @@
    "@backstage/catalog-client": "workspace:^",
    "@backstage/catalog-model": "workspace:^",
    "@backstage/errors": "workspace:^",
+    "@backstage/plugin-catalog-common": "workspace:^",
    "@backstage/types": "workspace:^"
  },
  "devDependencies": {
@@ -15,6 +15,7 @@
 */

 import { CompoundEntityRef } from '@backstage/catalog-model';
+import { LocationSpec as NonDeprecatedLocationSpec } from '@backstage/plugin-catalog-common';

 /**
 * Holds the entity location information.
@@ -26,12 +27,9 @@ import { CompoundEntityRef } from '@backstage/catalog-model';
 *  default value: 'required'.
 *
 * @public
+ * @deprecated use the same type from `@backstage/plugin-catalog-common` instead
 */
-export type LocationSpec = {
-  type: string;
-  target: string;
-  presence?: 'optional' | 'required';
-};
+export type LocationSpec = NonDeprecatedLocationSpec;

 /**
 * Holds the relation data for entities.
@@ -17,7 +17,8 @@
 import { InputError, NotFoundError } from '@backstage/errors';
 import { Entity } from '@backstage/catalog-model';
 import { CatalogProcessorResult } from './processor';
-import { EntityRelationSpec, LocationSpec } from './common';
+import { EntityRelationSpec } from './common';
+import { LocationSpec } from '@backstage/plugin-catalog-common';

 /**
 * Factory functions for the standard processing result types.
@@ -16,7 +16,8 @@

 import { Entity } from '@backstage/catalog-model';
 import { JsonValue } from '@backstage/types';
-import { EntityRelationSpec, LocationSpec } from './common';
+import { EntityRelationSpec } from './common';
+import { LocationSpec } from '@backstage/plugin-catalog-common';

 /**
 * @public
@@ -4576,6 +4576,7 @@ __metadata:
    "@backstage/backend-plugin-api": "workspace:^"
    "@backstage/backend-tasks": "workspace:^"
    "@backstage/backend-test-utils": "workspace:^"
+    "@backstage/catalog-client": "workspace:^"
    "@backstage/catalog-model": "workspace:^"
    "@backstage/cli": "workspace:^"
    "@backstage/config": "workspace:^"
@@ -4585,7 +4586,9 @@ __metadata:
    "@backstage/plugin-catalog-node": "workspace:^"
    "@backstage/types": "workspace:^"
    "@octokit/graphql": ^5.0.0
+    "@octokit/rest": ^19.0.3
    "@types/lodash": ^4.14.151
+    git-url-parse: ^13.0.0
    lodash: ^4.17.21
    msw: ^0.47.0
    node-fetch: ^2.6.7
@@ -4750,6 +4753,7 @@ __metadata:
  version: 0.0.0-use.local
  resolution: "@backstage/plugin-catalog-common@workspace:plugins/catalog-common"
  dependencies:
+    "@backstage/catalog-model": "workspace:^"
    "@backstage/cli": "workspace:^"
    "@backstage/plugin-permission-common": "workspace:^"
    "@backstage/plugin-search-common": "workspace:^"
@@ -4831,6 +4835,7 @@ __metadata:
    "@backstage/errors": "workspace:^"
    "@backstage/integration": "workspace:^"
    "@backstage/integration-react": "workspace:^"
+    "@backstage/plugin-catalog-common": "workspace:^"
    "@backstage/plugin-catalog-react": "workspace:^"
    "@backstage/test-utils": "workspace:^"
    "@material-ui/core": ^4.12.2
@@ -4867,6 +4872,7 @@ __metadata:
    "@backstage/catalog-model": "workspace:^"
    "@backstage/cli": "workspace:^"
    "@backstage/errors": "workspace:^"
+    "@backstage/plugin-catalog-common": "workspace:^"
    "@backstage/types": "workspace:^"
  languageName: unknown
  linkType: soft