From dbadbe7982a5885199f86887ab7eadfab528d437 Mon Sep 17 00:00:00 2001 From: Marek Calus Date: Wed, 28 Oct 2020 23:45:43 +0100 Subject: [PATCH] Renames and new types --- packages/backend/src/plugins/catalog.ts | 4 +- packages/catalog-model/src/location/index.ts | 2 +- .../catalog-model/src/location/validation.ts | 6 +- ...ConfigGenerator.ts => LocationAnalyzer.ts} | 30 ++++---- .../catalog-backend/src/ingestion/types.ts | 63 +++++++++++++++- .../src/service/CatalogBuilder.ts | 10 +-- plugins/catalog-backend/src/service/router.ts | 19 +++-- yarn.lock | 75 ++++--------------- 8 files changed, 115 insertions(+), 94 deletions(-) rename plugins/catalog-backend/src/ingestion/{ConfigGenerator.ts => LocationAnalyzer.ts} (60%) diff --git a/packages/backend/src/plugins/catalog.ts b/packages/backend/src/plugins/catalog.ts index 8d6c823dce..1338fab440 100644 --- a/packages/backend/src/plugins/catalog.ts +++ b/packages/backend/src/plugins/catalog.ts @@ -28,7 +28,7 @@ export default async function createPlugin(env: PluginEnvironment) { entitiesCatalog, locationsCatalog, higherOrderOperation, - configGenerator, + locationAnalyzer, } = await builder.build(); useHotCleanup( @@ -40,7 +40,7 @@ export default async function createPlugin(env: PluginEnvironment) { entitiesCatalog, locationsCatalog, higherOrderOperation, - configGenerator, + locationAnalyzer, logger: env.logger, }); } diff --git a/packages/catalog-model/src/location/index.ts b/packages/catalog-model/src/location/index.ts index 1b7587c1f0..ce64b988a6 100644 --- a/packages/catalog-model/src/location/index.ts +++ b/packages/catalog-model/src/location/index.ts @@ -18,6 +18,6 @@ export type { Location, LocationSpec } from './types'; export { locationSchema, locationSpecSchema, - repoPathSchema, + analyzeLocationSchema, } from './validation'; export { LOCATION_ANNOTATION } from './annotation'; diff --git a/packages/catalog-model/src/location/validation.ts b/packages/catalog-model/src/location/validation.ts index 28662119b0..4d2e602862 100644 --- a/packages/catalog-model/src/location/validation.ts +++ b/packages/catalog-model/src/location/validation.ts @@ -35,9 +35,9 @@ export const locationSchema = yup .noUnknown() .required(); -export const repoPathSchema = yup - .object<{ repoPath: string }>({ - repoPath: yup.string().required(), +export const analyzeLocationSchema = yup + .object<{ location: LocationSpec }>({ + location: locationSpecSchema, }) .noUnknown() .required(); diff --git a/plugins/catalog-backend/src/ingestion/ConfigGenerator.ts b/plugins/catalog-backend/src/ingestion/LocationAnalyzer.ts similarity index 60% rename from plugins/catalog-backend/src/ingestion/ConfigGenerator.ts rename to plugins/catalog-backend/src/ingestion/LocationAnalyzer.ts index d4d586b4de..f6991b1c90 100644 --- a/plugins/catalog-backend/src/ingestion/ConfigGenerator.ts +++ b/plugins/catalog-backend/src/ingestion/LocationAnalyzer.ts @@ -14,32 +14,36 @@ * limitations under the License. */ -import { Entity } from '@backstage/catalog-model'; import { Logger } from 'winston'; -import { ConfigGenerator } from './types'; +import { + AnalyzeLocationRequest, + AnalyzeLocationResponse, + LocationAnalyzer, +} from './types'; -export class ConfigGeneratorClient implements ConfigGenerator { - logger: Logger; +export class LocationAnalyzerClient implements LocationAnalyzer { + private readonly logger: Logger; constructor(logger: Logger) { this.logger = logger; } - async generateConfig(repoPath: string): Promise { - const [ownerName, repoName] = [ - repoPath.split('/')[0], - repoPath.split('/')[1], - ]; - const config = { + async generateConfig( + request: AnalyzeLocationRequest, + ): Promise { + const [ownerName, repoName] = request.location.target.split('/').slice(-2); + const entity = { apiVersion: 'backstage.io/v1alpha1', kind: 'Component', metadata: { name: repoName, - description: '', - annotations: { 'github.com/project-slug': repoPath }, + annotations: { 'github.com/project-slug': `${ownerName}/${repoName}` }, }, spec: { type: 'service', owner: ownerName, lifecycle: 'experimental' }, }; - return [config]; + return { + existingEntityFiles: [], + generateEntities: [{ entity, fields: [] }], + }; } } diff --git a/plugins/catalog-backend/src/ingestion/types.ts b/plugins/catalog-backend/src/ingestion/types.ts index 2d7165c140..19d472b3ab 100644 --- a/plugins/catalog-backend/src/ingestion/types.ts +++ b/plugins/catalog-backend/src/ingestion/types.ts @@ -20,6 +20,7 @@ import type { Location, LocationSpec, } from '@backstage/catalog-model'; +import { RecursivePartial } from './processors/ldap/util'; // // HigherOrderOperation @@ -70,6 +71,64 @@ export type ReadLocationError = { // ConfigGenerator // -export type ConfigGenerator = { - generateConfig(repoPath: string): Promise; +export type LocationAnalyzer = { + generateConfig( + location: AnalyzeLocationRequest, + ): Promise; +}; + +export type AnalyzeLocationRequest = { + location: LocationSpec; +}; + +export type AnalyzeLocationResponse = { + existingEntityFiles: AnalyzeLocationExistingEntity[]; + generateEntities: AnalyzeLocationGenerateEntity[]; +}; + +// If the folder pointed to already contained catalog info yaml files, they are +// read and emitted like this so that the frontend can inform the user that it +// located them and can make sure to register them as well if they weren't +// already +type AnalyzeLocationExistingEntity = { + location: LocationSpec; + isRegistered: boolean; + entity: Entity; +}; + +// This is some form of representation of what the analyzer could deduce. +// We should probably have a chat about how this can best be conveyed to +// the frontend. It'll probably contain a (possibly incomplete) entity, plus +// enough info for the frontend to know what form data to show to the user +// for overriding/completing the info. +type AnalyzeLocationGenerateEntity = { + // Some form of partial representation of the entity + entity: RecursivePartial; + // Lists the suggestions that the user may want to override + fields: AnalyzeLocationEntityField[]; +}; + +// This is where I get really vague. Something like this perhaps? Or it could be +// something like a json-schema that contains enough info for the frontend to +// be able to present a form and explanations +type AnalyzeLocationEntityField = { + // e.g. "spec.owner"? The frontend needs to know how to "inject" the field into the + // entity again if the user wants to change it + field: string; + + // The outcome of the analysis for this particular field + state: + | 'analysis_suggested_value' + | 'analysis_suggested_no_value' + | 'needs_user_input'; + + // If the analysis did suggest a value, this is where it would be. Not sure if we want + // to limit this to strings or if we want it to be any JsonValue + value: string | null; + + // A text to show to the user to inform about the choices made. Like, it could say + // "Found a CODEOWNERS file that covers this target, so we suggest leaving this + // field empty; which would currently make it owned by X" where X is taken from the + // codeowners file. + description: string; }; diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index 39417dfd6f..439b85591f 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -52,7 +52,7 @@ import { UrlReaderProcessor, } from '../ingestion'; import { CatalogRulesEnforcer } from '../ingestion/CatalogRules'; -import { ConfigGeneratorClient } from '../ingestion/ConfigGenerator'; +import { LocationAnalyzerClient } from '../ingestion/LocationAnalyzer'; import { BuiltinKindsEntityProcessor } from '../ingestion/processors/BuiltinKindsEntityProcessor'; import { LdapOrgReaderProcessor } from '../ingestion/processors/LdapOrgReaderProcessor'; import { @@ -60,7 +60,7 @@ import { textPlaceholderResolver, yamlPlaceholderResolver, } from '../ingestion/processors/PlaceholderProcessor'; -import { ConfigGenerator } from '../ingestion/types'; +import { LocationAnalyzer } from '../ingestion/types'; export type CatalogEnvironment = { logger: Logger; @@ -204,7 +204,7 @@ export class CatalogBuilder { entitiesCatalog: EntitiesCatalog; locationsCatalog: LocationsCatalog; higherOrderOperation: HigherOrderOperation; - configGenerator: ConfigGenerator; + locationAnalyzer: LocationAnalyzer; }> { const { config, database, logger } = this.env; @@ -232,13 +232,13 @@ export class CatalogBuilder { locationReader, logger, ); - const configGenerator = new ConfigGeneratorClient(logger); + const locationAnalyzer = new LocationAnalyzerClient(logger); return { entitiesCatalog, locationsCatalog, higherOrderOperation, - configGenerator, + locationAnalyzer, }; } diff --git a/plugins/catalog-backend/src/service/router.ts b/plugins/catalog-backend/src/service/router.ts index b11572b180..8d5fb73c37 100644 --- a/plugins/catalog-backend/src/service/router.ts +++ b/plugins/catalog-backend/src/service/router.ts @@ -15,13 +15,16 @@ */ import { errorHandler } from '@backstage/backend-common'; -import { locationSpecSchema, repoPathSchema } from '@backstage/catalog-model'; +import { + locationSpecSchema, + analyzeLocationSchema, +} from '@backstage/catalog-model'; import type { Entity } from '@backstage/catalog-model'; import express from 'express'; import Router from 'express-promise-router'; import { Logger } from 'winston'; import { EntitiesCatalog, LocationsCatalog } from '../catalog'; -import { ConfigGenerator, HigherOrderOperation } from '../ingestion/types'; +import { LocationAnalyzer, HigherOrderOperation } from '../ingestion/types'; import { translateQueryToEntityFilters, translateQueryToFieldMapper, @@ -32,7 +35,7 @@ export interface RouterOptions { entitiesCatalog?: EntitiesCatalog; locationsCatalog?: LocationsCatalog; higherOrderOperation?: HigherOrderOperation; - configGenerator?: ConfigGenerator; + locationAnalyzer?: LocationAnalyzer; logger: Logger; } @@ -43,7 +46,7 @@ export async function createRouter( entitiesCatalog, locationsCatalog, higherOrderOperation, - configGenerator, + locationAnalyzer: locationAnalyzer, } = options; const router = Router(); @@ -135,10 +138,10 @@ export async function createRouter( }); } - if (configGenerator) { - router.post('/generate-config', async (req, res) => { - const input = await validateRequestBody(req, repoPathSchema); - const output = await configGenerator.generateConfig(input.repoPath); + if (locationAnalyzer) { + router.post('/analyze-location', async (req, res) => { + const input = await validateRequestBody(req, analyzeLocationSchema); + const output = await locationAnalyzer.generateConfig(input); res.status(201).send(output); }); } diff --git a/yarn.lock b/yarn.lock index 7f85f649c9..1c6125854e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3691,50 +3691,18 @@ react-router "^6.0.0-beta.0" react-use "^15.3.3" -"@roadiehq/backstage-plugin-github-pull-requests@^0.5.2": - version "0.5.2" - resolved "https://registry.npmjs.org/@roadiehq/backstage-plugin-github-pull-requests/-/backstage-plugin-github-pull-requests-0.5.2.tgz#455355fe5cdcc43c22b80a8ef3697378084680a1" - integrity sha512-GM+nWQZ+aehMclN70Pp35UQe3I46SQCjWiGZ6/0TfDzrueHln39L2UspUj1L/713xDDtDK8ZnTJdFn/qKx7j4A== +"@rollup/plugin-commonjs@^14.0.0": + version "14.0.0" + resolved "https://registry.npmjs.org/@rollup/plugin-commonjs/-/plugin-commonjs-14.0.0.tgz#4285f9ec2db686a31129e5a2b415c94aa1f836f0" + integrity sha512-+PSmD9ePwTAeU106i9FRdc+Zb3XUWyW26mo5Atr2mk82hor8+nPwkztEjFo8/B1fJKfaQDg9aM2bzQkjhi7zOw== dependencies: - "@backstage/catalog-model" "^0.1.1-alpha.25" - "@backstage/core" "^0.1.1-alpha.25" - "@backstage/plugin-catalog" "^0.1.1-alpha.25" - "@backstage/theme" "^0.1.1-alpha.25" - "@material-ui/core" "^4.11.0" - "@material-ui/icons" "^4.9.1" - "@material-ui/lab" "^4.0.0-alpha.56" - "@octokit/rest" "^18.0.0" - "@octokit/types" "^5.0.1" - "@types/react-dom" "^16.9.8" - history "^5.0.0" - moment "^2.27.0" - react "^16.13.1" - react-dom "^16.13.1" - react-router "6.0.0-beta.0" - react-use "^15.3.3" - -"@roadiehq/backstage-plugin-travis-ci@^0.2.5": - version "0.2.5" - resolved "https://registry.npmjs.org/@roadiehq/backstage-plugin-travis-ci/-/backstage-plugin-travis-ci-0.2.5.tgz#588cd8b4a4dd4bd426c0012f8d35eae4504b55f4" - integrity sha512-4aJizmvIeunpl8D/YDvzagNzikPU51DSZFz1tN0XCwRR+buTPknV141qsKP+yxUbs6nx3Xuh+T/vNkb9lk6FlA== - dependencies: - "@backstage/catalog-model" "^0.1.1-alpha.24" - "@backstage/core" "^0.1.1-alpha.24" - "@backstage/core-api" "^0.1.1-alpha.24" - "@backstage/plugin-catalog" "^0.1.1-alpha.24" - "@backstage/theme" "^0.1.1-alpha.24" - "@material-ui/core" "^4.9.1" - "@material-ui/icons" "^4.9.1" - "@material-ui/lab" "4.0.0-alpha.45" - date-fns "^2.15.0" - history "^5.0.0" - moment "^2.27.0" - react "^16.13.1" - react-dom "^16.13.1" - react-lazylog "^4.5.2" - react-router "6.0.0-beta.0" - react-router-dom "6.0.0-beta.0" - react-use "^15.3.3" + "@rollup/pluginutils" "^3.0.8" + commondir "^1.0.1" + estree-walker "^1.0.1" + glob "^7.1.2" + is-reference "^1.1.2" + magic-string "^0.25.2" + resolve "^1.11.0" "@rollup/plugin-commonjs@^16.0.0": version "16.0.0" @@ -3749,19 +3717,6 @@ magic-string "^0.25.7" resolve "^1.17.0" -"@rollup/plugin-commonjs@^14.0.0": - version "14.0.0" - resolved "https://registry.npmjs.org/@rollup/plugin-commonjs/-/plugin-commonjs-14.0.0.tgz#4285f9ec2db686a31129e5a2b415c94aa1f836f0" - integrity sha512-+PSmD9ePwTAeU106i9FRdc+Zb3XUWyW26mo5Atr2mk82hor8+nPwkztEjFo8/B1fJKfaQDg9aM2bzQkjhi7zOw== - dependencies: - "@rollup/pluginutils" "^3.0.8" - commondir "^1.0.1" - estree-walker "^1.0.1" - glob "^7.1.2" - is-reference "^1.1.2" - magic-string "^0.25.2" - resolve "^1.11.0" - "@rollup/plugin-json@^4.0.2": version "4.1.0" resolved "https://registry.npmjs.org/@rollup/plugin-json/-/plugin-json-4.1.0.tgz#54e09867ae6963c593844d8bd7a9c718294496f3" @@ -9881,7 +9836,7 @@ dateformat@^3.0.0: resolved "https://registry.npmjs.org/dateformat/-/dateformat-3.0.3.tgz#a6e37499a4d9a9cf85ef5872044d62901c9889ae" integrity sha512-jyCETtSl3VMZMWeRo7iY1FL19ges1t55hMo5yaam4Jrsm5EPL89UQkoQRyiI+Yf4k8r2ZpdngkV8hr1lIdjb3Q== -dayjs@^1.9.4: +dayjs@^1.9.1, dayjs@^1.9.4: version "1.9.4" resolved "https://registry.npmjs.org/dayjs/-/dayjs-1.9.4.tgz#fcde984e227f4296f04e7b05720adad2e1071f1b" integrity sha512-ABSF3alrldf7nM9sQ2U+Ln67NRwmzlLOqG7kK03kck0mw3wlSSEKv/XhKGGxUjQcS57QeiCyNdrFgtj9nWlrng== @@ -14227,7 +14182,7 @@ is-promise@^2.1, is-promise@^2.1.0: resolved "https://registry.npmjs.org/is-promise/-/is-promise-2.2.2.tgz#39ab959ccbf9a774cf079f7b40c7a26f763135f1" integrity sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ== -is-reference@^1.2.1: +is-reference@^1.1.2, is-reference@^1.2.1: version "1.2.1" resolved "https://registry.npmjs.org/is-reference/-/is-reference-1.2.1.tgz#8b2dac0b371f4bc994fdeaba9eb542d03002d0b7" integrity sha512-U82MsXXiFIrjCK4otLT+o2NA2Cd2g5MLoOVXUZjIOhLurrRxpEXzI8O0KZHr3IjLvlAH1kTPYSuqer5T9ZVBKQ== @@ -16125,7 +16080,7 @@ macos-release@^2.2.0: resolved "https://registry.npmjs.org/macos-release/-/macos-release-2.3.0.tgz#eb1930b036c0800adebccd5f17bc4c12de8bb71f" integrity sha512-OHhSbtcviqMPt7yfw5ef5aghS2jzFVKEFyCJndQt2YpSQ9qRVSEv2axSJI1paVThEu+FFGs584h/1YhxjVqajA== -magic-string@^0.25.7: +magic-string@^0.25.2, magic-string@^0.25.7: version "0.25.7" resolved "https://registry.npmjs.org/magic-string/-/magic-string-0.25.7.tgz#3f497d6fd34c669c6798dcb821f2ef31f5445051" integrity sha512-4CrMT5DOHTDk4HYDlzmwu4FVCcIYI8gauveasrdCu2IKIFOJ3f0v/8MDGJCDL9oD2ppz/Av1b0Nj345H9M+XIA== @@ -20782,7 +20737,7 @@ resolve@1.17.0, resolve@^1.1.6, resolve@^1.1.7, resolve@^1.10.0, resolve@^1.12.0 dependencies: path-parse "^1.0.6" -resolve@^1.18.1: +resolve@^1.11.0, resolve@^1.18.1: version "1.18.1" resolved "https://registry.npmjs.org/resolve/-/resolve-1.18.1.tgz#018fcb2c5b207d2a6424aee361c5a266da8f4130" integrity sha512-lDfCPaMKfOJXjy0dPayzPdF1phampNWr3qFCjAu+rw/qbQmr5jWH5xN2hwh9QKfw9E5v4hwV7A+jrCmL8yjjqA==