diff --git a/.changeset/repo-tools-yo-dawg.md b/.changeset/repo-tools-yo-dawg.md new file mode 100644 index 0000000000..85757fb172 --- /dev/null +++ b/.changeset/repo-tools-yo-dawg.md @@ -0,0 +1,5 @@ +--- +'@backstage/repo-tools': patch +--- + +Introducing a new, experimental command `backstage-repo-tools generate-catalog-info`, which can be used to create standardized `catalog-info.yaml` files for each Backstage package in a Backstage monorepo. It can also be used to automatically fix existing `catalog-info.yaml` files with the correct metadata (including `metadata.name`, `metadata.title`, and `metadata.description` introspected from the package's `package.json`, as well as `spec.owner` introspected from `CODEOWNERS`), e.g. in a post-commit hook. diff --git a/packages/repo-tools/cli-report.md b/packages/repo-tools/cli-report.md index a0df5e721c..9c87bd8dff 100644 --- a/packages/repo-tools/cli-report.md +++ b/packages/repo-tools/cli-report.md @@ -14,6 +14,7 @@ Options: Commands: api-reports [options] [paths...] type-deps + generate-catalog-info [options] schema [command] help [command] ``` @@ -36,6 +37,16 @@ Options: -h, --help ``` +### `backstage-repo-tools generate-catalog-info` + +``` +Usage: backstage-repo-tools generate-catalog-info [options] + +Options: + --dry-run + -h, --help +``` + ### `backstage-repo-tools schema` ``` diff --git a/packages/repo-tools/package.json b/packages/repo-tools/package.json index 8a043217bb..41ea703109 100644 --- a/packages/repo-tools/package.json +++ b/packages/repo-tools/package.json @@ -32,6 +32,7 @@ "dependencies": { "@apidevtools/swagger-parser": "^10.1.0", "@apisyouwonthate/style-guide": "^1.4.0", + "@backstage/catalog-model": "workspace:^", "@backstage/cli-common": "workspace:^", "@backstage/cli-node": "workspace:^", "@backstage/errors": "workspace:^", @@ -46,6 +47,7 @@ "@stoplight/spectral-runtime": "^1.1.2", "@stoplight/types": "^13.14.0", "chalk": "^4.0.0", + "codeowners-utils": "^1.0.2", "commander": "^9.1.0", "fs-extra": "10.1.0", "glob": "^8.0.3", @@ -54,7 +56,8 @@ "lodash": "^4.17.21", "minimatch": "^5.1.1", "p-limit": "^3.0.2", - "ts-node": "^10.0.0" + "ts-node": "^10.0.0", + "yaml-diff-patch": "^2.0.0" }, "devDependencies": { "@backstage/cli": "workspace:^", diff --git a/packages/repo-tools/src/commands/generate-catalog-info/codeowners.ts b/packages/repo-tools/src/commands/generate-catalog-info/codeowners.ts new file mode 100644 index 0000000000..f485e872e9 --- /dev/null +++ b/packages/repo-tools/src/commands/generate-catalog-info/codeowners.ts @@ -0,0 +1,69 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + CodeOwnersEntry, + CODEOWNERS_PATHS, + matchFile as matchCodeowner, + parse as parseCodeowners, +} from 'codeowners-utils'; +import { relative as relativePath, resolve as resolvePath } from 'path'; +import { readFile } from './utils'; + +export async function loadCodeowners(): Promise { + const maybeFiles = await Promise.allSettled( + CODEOWNERS_PATHS.map(async path => + readFile(resolvePath('.', path), { encoding: 'utf-8' }), + ), + ); + const file = maybeFiles.find( + maybeFile => maybeFile.status === 'fulfilled', + ) as PromiseFulfilledResult | undefined; + + if (!file) { + throw new Error( + 'This utility expects a CODEOWNERS file, but no such file was found.', + ); + } + + return parseCodeowners(file.value); +} + +export function getPossibleCodeowners( + codeowners: CodeOwnersEntry[], + relPath: string, +): string[] { + const codeownerMaybe = matchCodeowner(relPath, codeowners); + return codeownerMaybe + ? codeownerMaybe.owners.map( + owner => (owner.match(/(?:\@[^\/]+\/)?([^\@\/]*)$/) || [])[1], + ) + : []; +} + +export function getOwnerFromCodeowners( + codeowners: CodeOwnersEntry[], + absPath: string, +): string { + const relPath = relativePath('.', absPath); + const possibleOwners = getPossibleCodeowners(codeowners, relPath); + const owner = possibleOwners.slice(-1)[0]; + + if (!owner) { + throw new Error(`${relPath} isn't owned by anyone in CODEOWNERS`); + } + + return owner; +} diff --git a/packages/repo-tools/src/commands/generate-catalog-info/generate-catalog-info.ts b/packages/repo-tools/src/commands/generate-catalog-info/generate-catalog-info.ts new file mode 100644 index 0000000000..08f73898c8 --- /dev/null +++ b/packages/repo-tools/src/commands/generate-catalog-info/generate-catalog-info.ts @@ -0,0 +1,200 @@ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import YAML from 'js-yaml'; +import pLimit from 'p-limit'; +import { relative as relativePath, resolve as resolvePath } from 'path'; +import { yamlOverwrite } from 'yaml-diff-patch'; +import chalk from 'chalk'; +import { PackageGraph, PackageRole } from '@backstage/cli-node'; +import { Entity } from '@backstage/catalog-model'; +import { + getOwnerFromCodeowners, + getPossibleCodeowners, + loadCodeowners, +} from './codeowners'; +import { + BackstagePackageJson, + isBackstagePackage, + readFile, + writeFile, +} from './utils'; +import { CodeOwnersEntry } from 'codeowners-utils'; + +type CreateFixPackageInfoYamlsOptions = { + dryRun?: boolean; +}; + +export default async (opts: CreateFixPackageInfoYamlsOptions) => { + const { dryRun = false } = opts; + const packages = await PackageGraph.listTargetPackages(); + const codeowners = await loadCodeowners(); + const limit = pLimit(10); + + const results = await Promise.allSettled( + packages.map(({ packageJson, dir }) => + limit(async () => { + if (!isBackstagePackage(packageJson)) { + return; + } + + // Check if there is already a corresponding catalog-info.yaml + const infoYamlPath = resolvePath(dir, 'catalog-info.yaml'); + let yamlString = ''; + + try { + yamlString = await readFile(infoYamlPath, { encoding: 'utf-8' }); + } catch (e) { + if (e.code === 'ENOENT') { + await createCatalogInfoYaml({ + yamlPath: infoYamlPath, + packageJson, + codeowners, + dryRun, + }); + return; + } + + throw e; + } + + await fixCatalogInfoYaml({ + yamlPath: infoYamlPath, + packageJson, + codeowners, + yamlString, + dryRun, + }); + }), + ), + ); + + const rejects = results.filter( + r => r.status === 'rejected', + ) as PromiseRejectedResult[]; + if (rejects.length > 0) { + // Problems encountered. Print details here. + console.error( + chalk.red('Unable to create or fix catalog-info.yaml files\n'), + ); + rejects.forEach(reject => console.error(` ${reject.reason}`)); + console.error(); + process.exit(1); + } +}; + +type CreateOptions = { + yamlPath: string; + packageJson: BackstagePackageJson; + codeowners: CodeOwnersEntry[]; + dryRun: boolean; +}; + +type FixOptions = CreateOptions & { + yamlString: string; +}; + +type BackstagePackageEntity = Entity & { + spec: { + type: 'backstage-package'; + backstageRole: PackageRole; + lifecycle: string; + owner: string; + [key: string]: any; + }; +}; + +function createCatalogInfoYaml(options: CreateOptions) { + const { codeowners, dryRun, packageJson, yamlPath } = options; + const owner = getOwnerFromCodeowners(codeowners, yamlPath); + const entity = createOrMergeEntity(packageJson, owner); + + return dryRun + ? Promise.resolve(console.error(`Create ${relativePath('.', yamlPath)}`)) + : writeFile(yamlPath, YAML.dump(entity)); +} + +function fixCatalogInfoYaml(options: FixOptions) { + const { codeowners, dryRun, packageJson, yamlPath, yamlString } = options; + const possibleOwners = getPossibleCodeowners( + codeowners, + relativePath('.', yamlPath), + ); + const safeName = packageJson.name + .replace(/[^a-z0-9_\-\.]+/g, '-') + .replace(/^[^a-z0-9]|[^a-z0-9]$/g, ''); + let yamlJson: BackstagePackageEntity; + + try { + yamlJson = YAML.load(yamlString) as BackstagePackageEntity; + } catch (e) { + throw new Error(`Unable to parse ${relativePath('.', yamlPath)}: ${e}`); + } + + const badOwner = !possibleOwners.includes(yamlJson.spec?.owner); + const badTitle = yamlJson.metadata.title !== packageJson.name; + const badName = yamlJson.metadata.name !== safeName; + const badType = + yamlJson.spec?.type !== `backstage-${packageJson.backstage.role}`; + const badDesc = yamlJson.metadata.description !== packageJson.description; + + if (badOwner || badTitle || badName || badType || badDesc) { + const owner = badOwner + ? getOwnerFromCodeowners(codeowners, yamlPath) + : yamlJson.spec?.owner; + const newJson = createOrMergeEntity(packageJson, owner, yamlJson); + return dryRun + ? Promise.resolve(console.error(`Update ${relativePath('.', yamlPath)}`)) + : writeFile(yamlPath, yamlOverwrite(yamlString, newJson)); + } + + return Promise.resolve(); +} + +/** + * Canonical representation on how to create (or update) a backstage-package + * component. + */ +function createOrMergeEntity( + packageJson: BackstagePackageJson, + owner: string, + existingEntity: BackstagePackageEntity | Record = {}, +): BackstagePackageEntity { + const safeEntityName = packageJson.name + .replace(/[^a-z0-9_\-\.]+/g, '-') + .replace(/^[^a-z0-9]|[^a-z0-9]$/g, ''); + + return { + ...existingEntity, + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + ...existingEntity.metadata, + // Provide default name/title/description values. + name: safeEntityName, + title: packageJson.name, + ...(packageJson.description + ? { description: packageJson.description } + : undefined), + }, + spec: { + lifecycle: 'experimental', + ...existingEntity.spec, + type: `backstage-${packageJson.backstage.role}`, + owner, + }, + }; +} diff --git a/packages/repo-tools/src/commands/generate-catalog-info/utils.ts b/packages/repo-tools/src/commands/generate-catalog-info/utils.ts new file mode 100644 index 0000000000..f446f4e867 --- /dev/null +++ b/packages/repo-tools/src/commands/generate-catalog-info/utils.ts @@ -0,0 +1,42 @@ +/* + * Copyright 2023 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import fs from 'fs'; +import { Package } from '@manypkg/get-packages'; +import { + BackstagePackageJson as BackstagePackageJsonActual, + PackageRole, +} from '@backstage/cli-node'; +import { promisify } from 'util'; + +export const readFile = promisify(fs.readFile); +export const writeFile = promisify(fs.writeFile); + +export type BackstagePackageJson = BackstagePackageJsonActual & { + description?: string; + backstage: { + role: PackageRole; + }; +}; + +export function isBackstagePackage( + packageJson: Package['packageJson'], +): packageJson is BackstagePackageJson { + return ( + packageJson && + packageJson.hasOwnProperty('backstage') && + (packageJson as any)?.backstage?.role !== 'undefined' + ); +} diff --git a/packages/repo-tools/src/commands/index.ts b/packages/repo-tools/src/commands/index.ts index e19ffde4b8..7aa574d291 100644 --- a/packages/repo-tools/src/commands/index.ts +++ b/packages/repo-tools/src/commands/index.ts @@ -96,6 +96,21 @@ export function registerCommands(program: Command) { .description('Find inconsistencies in types of all packages and plugins') .action(lazy(() => import('./type-deps/type-deps').then(m => m.default))); + program + .command('generate-catalog-info') + .option( + '--dry-run', + 'Shows what would happen without actually writing any yaml.', + ) + .description('Create or fix info yaml files for all backstage packages') + .action( + lazy(() => + import('./generate-catalog-info/generate-catalog-info').then( + m => m.default, + ), + ), + ); + registerSchemaCommand(program); } diff --git a/yarn.lock b/yarn.lock index 68d0c1a922..b41fc1464e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9538,6 +9538,7 @@ __metadata: dependencies: "@apidevtools/swagger-parser": ^10.1.0 "@apisyouwonthate/style-guide": ^1.4.0 + "@backstage/catalog-model": "workspace:^" "@backstage/cli": "workspace:^" "@backstage/cli-common": "workspace:^" "@backstage/cli-node": "workspace:^" @@ -9557,6 +9558,7 @@ __metadata: "@types/mock-fs": ^4.13.0 "@types/node": ^16.11.26 chalk: ^4.0.0 + codeowners-utils: ^1.0.2 commander: ^9.1.0 fs-extra: 10.1.0 glob: ^8.0.3 @@ -9567,6 +9569,7 @@ __metadata: mock-fs: ^5.1.0 p-limit: ^3.0.2 ts-node: ^10.0.0 + yaml-diff-patch: ^2.0.0 peerDependencies: "@microsoft/api-extractor-model": "*" "@microsoft/tsdoc": "*" @@ -25565,7 +25568,7 @@ __metadata: languageName: node linkType: hard -"fast-json-patch@npm:^3.0.0-1": +"fast-json-patch@npm:^3.0.0-1, fast-json-patch@npm:^3.1.0": version: 3.1.1 resolution: "fast-json-patch@npm:3.1.1" checksum: c4525b61b2471df60d4b025b4118b036d99778a93431aa44d1084218182841d82ce93056f0f3bbd731a24e6a8e69820128adf1873eb2199a26c62ef58d137833 @@ -33855,6 +33858,15 @@ __metadata: languageName: node linkType: hard +"oppa@npm:^0.4.0": + version: 0.4.0 + resolution: "oppa@npm:0.4.0" + dependencies: + chalk: ^4.1.1 + checksum: ecc43e63ede05c3ccb10e0f2c3f3020a6d72e1a3b318f3e37b8cc8a1a279e300991c043e5385d560c1eebb54a56c7f9b69bf0db0d1933acf350bcd2980c96055 + languageName: node + linkType: hard + "optimism@npm:^0.17.5": version: 0.17.5 resolution: "optimism@npm:0.17.5" @@ -42567,6 +42579,21 @@ __metadata: languageName: node linkType: hard +"yaml-diff-patch@npm:^2.0.0": + version: 2.0.0 + resolution: "yaml-diff-patch@npm:2.0.0" + dependencies: + fast-json-patch: ^3.1.0 + oppa: ^0.4.0 + yaml: ^2.0.0-10 + bin: + yaml-diff-patch: dist/bin/yaml-patch.js + yaml-overwrite: dist/bin/yaml-patch.js + yaml-patch: dist/bin/yaml-patch.js + checksum: 5207d8523584eb6088fe32a0c6010599260ecfa5f959d120a1bad02f19143d1ddeafe10c37ccf125ac04d079072a5ead92b55c6787fd64d12f5acbb0d172e7ec + languageName: node + linkType: hard + "yaml@npm:^1.10.0, yaml@npm:^1.10.2, yaml@npm:^1.7.2": version: 1.10.2 resolution: "yaml@npm:1.10.2" @@ -42574,10 +42601,10 @@ __metadata: languageName: node linkType: hard -"yaml@npm:^2.0.0, yaml@npm:^2.1.1, yaml@npm:^2.2.1, yaml@npm:^2.2.2": - version: 2.2.2 - resolution: "yaml@npm:2.2.2" - checksum: d90c235e099e30094dcff61ba3350437aef53325db4a6bcd04ca96e1bfe7e348b191f6a7a52b5211e2dbc4eeedb22a00b291527da030de7c189728ef3f2b4eb3 +"yaml@npm:^2.0.0, yaml@npm:^2.0.0-10, yaml@npm:^2.1.1, yaml@npm:^2.2.1, yaml@npm:^2.2.2": + version: 2.3.1 + resolution: "yaml@npm:2.3.1" + checksum: 2c7bc9a7cd4c9f40d3b0b0a98e370781b68b8b7c4515720869aced2b00d92f5da1762b4ffa947f9e795d6cd6b19f410bd4d15fdd38aca7bd96df59bd9486fb54 languageName: node linkType: hard