Merge pull request #18354 from backstage/catalog-backstage-plugins-command

[repo-tools] Command to create/update `catalog-info.yaml` files for Backstage Packages

.changeset/repo-tools-yo-dawg.md

@@ -0,0 +1,5 @@
+---
+'@backstage/repo-tools': patch
+---
+
+Introducing a new, experimental command `backstage-repo-tools generate-catalog-info`, which can be used to create standardized `catalog-info.yaml` files for each Backstage package in a Backstage monorepo. It can also be used to automatically fix existing `catalog-info.yaml` files with the correct metadata (including `metadata.name`, `metadata.title`, and `metadata.description` introspected from the package's `package.json`, as well as `spec.owner` introspected from `CODEOWNERS`), e.g. in a post-commit hook.

packages/repo-tools/cli-report.md

@@ -14,6 +14,7 @@ Options:
 Commands:
   api-reports [options] [paths...]
   type-deps
+  generate-catalog-info [options]
   schema [command]
   help [command]
 ```
@@ -36,6 +37,16 @@ Options:
   -h, --help
 ```
 
+### `backstage-repo-tools generate-catalog-info`
+
+```
+Usage: backstage-repo-tools generate-catalog-info [options]
+
+Options:
+  --dry-run
+  -h, --help
+```
+
 ### `backstage-repo-tools schema`
 
 ```

packages/repo-tools/package.json

@@ -32,6 +32,7 @@
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.1.0",
     "@apisyouwonthate/style-guide": "^1.4.0",
+    "@backstage/catalog-model": "workspace:^",
     "@backstage/cli-common": "workspace:^",
     "@backstage/cli-node": "workspace:^",
     "@backstage/errors": "workspace:^",
@@ -46,6 +47,7 @@
     "@stoplight/spectral-runtime": "^1.1.2",
     "@stoplight/types": "^13.14.0",
     "chalk": "^4.0.0",
+    "codeowners-utils": "^1.0.2",
     "commander": "^9.1.0",
     "fs-extra": "10.1.0",
     "glob": "^8.0.3",
@@ -54,7 +56,8 @@
     "lodash": "^4.17.21",
     "minimatch": "^5.1.1",
     "p-limit": "^3.0.2",
-    "ts-node": "^10.0.0"
+    "ts-node": "^10.0.0",
+    "yaml-diff-patch": "^2.0.0"
   },
   "devDependencies": {
     "@backstage/cli": "workspace:^",

packages/repo-tools/src/commands/generate-catalog-info/codeowners.ts

@@ -0,0 +1,69 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import {
+  CodeOwnersEntry,
+  CODEOWNERS_PATHS,
+  matchFile as matchCodeowner,
+  parse as parseCodeowners,
+} from 'codeowners-utils';
+import { relative as relativePath, resolve as resolvePath } from 'path';
+import { readFile } from './utils';
+
+export async function loadCodeowners(): Promise<CodeOwnersEntry[]> {
+  const maybeFiles = await Promise.allSettled(
+    CODEOWNERS_PATHS.map(async path =>
+      readFile(resolvePath('.', path), { encoding: 'utf-8' }),
+    ),
+  );
+  const file = maybeFiles.find(
+    maybeFile => maybeFile.status === 'fulfilled',
+  ) as PromiseFulfilledResult<string> | undefined;
+
+  if (!file) {
+    throw new Error(
+      'This utility expects a CODEOWNERS file, but no such file was found.',
+    );
+  }
+
+  return parseCodeowners(file.value);
+}
+
+export function getPossibleCodeowners(
+  codeowners: CodeOwnersEntry[],
+  relPath: string,
+): string[] {
+  const codeownerMaybe = matchCodeowner(relPath, codeowners);
+  return codeownerMaybe
+    ? codeownerMaybe.owners.map(
+        owner => (owner.match(/(?:\@[^\/]+\/)?([^\@\/]*)$/) || [])[1],
+      )
+    : [];
+}
+
+export function getOwnerFromCodeowners(
+  codeowners: CodeOwnersEntry[],
+  absPath: string,
+): string {
+  const relPath = relativePath('.', absPath);
+  const possibleOwners = getPossibleCodeowners(codeowners, relPath);
+  const owner = possibleOwners.slice(-1)[0];
+
+  if (!owner) {
+    throw new Error(`${relPath} isn't owned by anyone in CODEOWNERS`);
+  }
+
+  return owner;
+}

packages/repo-tools/src/commands/generate-catalog-info/generate-catalog-info.ts

@@ -0,0 +1,200 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import YAML from 'js-yaml';
+import pLimit from 'p-limit';
+import { relative as relativePath, resolve as resolvePath } from 'path';
+import { yamlOverwrite } from 'yaml-diff-patch';
+import chalk from 'chalk';
+import { PackageGraph, PackageRole } from '@backstage/cli-node';
+import { Entity } from '@backstage/catalog-model';
+import {
+  getOwnerFromCodeowners,
+  getPossibleCodeowners,
+  loadCodeowners,
+} from './codeowners';
+import {
+  BackstagePackageJson,
+  isBackstagePackage,
+  readFile,
+  writeFile,
+} from './utils';
+import { CodeOwnersEntry } from 'codeowners-utils';
+
+type CreateFixPackageInfoYamlsOptions = {
+  dryRun?: boolean;
+};
+
+export default async (opts: CreateFixPackageInfoYamlsOptions) => {
+  const { dryRun = false } = opts;
+  const packages = await PackageGraph.listTargetPackages();
+  const codeowners = await loadCodeowners();
+  const limit = pLimit(10);
+
+  const results = await Promise.allSettled<void>(
+    packages.map(({ packageJson, dir }) =>
+      limit(async () => {
+        if (!isBackstagePackage(packageJson)) {
+          return;
+        }
+
+        // Check if there is already a corresponding catalog-info.yaml
+        const infoYamlPath = resolvePath(dir, 'catalog-info.yaml');
+        let yamlString = '';
+
+        try {
+          yamlString = await readFile(infoYamlPath, { encoding: 'utf-8' });
+        } catch (e) {
+          if (e.code === 'ENOENT') {
+            await createCatalogInfoYaml({
+              yamlPath: infoYamlPath,
+              packageJson,
+              codeowners,
+              dryRun,
+            });
+            return;
+          }
+
+          throw e;
+        }
+
+        await fixCatalogInfoYaml({
+          yamlPath: infoYamlPath,
+          packageJson,
+          codeowners,
+          yamlString,
+          dryRun,
+        });
+      }),
+    ),
+  );
+
+  const rejects = results.filter(
+    r => r.status === 'rejected',
+  ) as PromiseRejectedResult[];
+  if (rejects.length > 0) {
+    // Problems encountered. Print details here.
+    console.error(
+      chalk.red('Unable to create or fix catalog-info.yaml files\n'),
+    );
+    rejects.forEach(reject => console.error(`  ${reject.reason}`));
+    console.error();
+    process.exit(1);
+  }
+};
+
+type CreateOptions = {
+  yamlPath: string;
+  packageJson: BackstagePackageJson;
+  codeowners: CodeOwnersEntry[];
+  dryRun: boolean;
+};
+
+type FixOptions = CreateOptions & {
+  yamlString: string;
+};
+
+type BackstagePackageEntity = Entity & {
+  spec: {
+    type: 'backstage-package';
+    backstageRole: PackageRole;
+    lifecycle: string;
+    owner: string;
+    [key: string]: any;
+  };
+};
+
+function createCatalogInfoYaml(options: CreateOptions) {
+  const { codeowners, dryRun, packageJson, yamlPath } = options;
+  const owner = getOwnerFromCodeowners(codeowners, yamlPath);
+  const entity = createOrMergeEntity(packageJson, owner);
+
+  return dryRun
+    ? Promise.resolve(console.error(`Create ${relativePath('.', yamlPath)}`))
+    : writeFile(yamlPath, YAML.dump(entity));
+}
+
+function fixCatalogInfoYaml(options: FixOptions) {
+  const { codeowners, dryRun, packageJson, yamlPath, yamlString } = options;
+  const possibleOwners = getPossibleCodeowners(
+    codeowners,
+    relativePath('.', yamlPath),
+  );
+  const safeName = packageJson.name
+    .replace(/[^a-z0-9_\-\.]+/g, '-')
+    .replace(/^[^a-z0-9]|[^a-z0-9]$/g, '');
+  let yamlJson: BackstagePackageEntity;
+
+  try {
+    yamlJson = YAML.load(yamlString) as BackstagePackageEntity;
+  } catch (e) {
+    throw new Error(`Unable to parse ${relativePath('.', yamlPath)}: ${e}`);
+  }
+
+  const badOwner = !possibleOwners.includes(yamlJson.spec?.owner);
+  const badTitle = yamlJson.metadata.title !== packageJson.name;
+  const badName = yamlJson.metadata.name !== safeName;
+  const badType =
+    yamlJson.spec?.type !== `backstage-${packageJson.backstage.role}`;
+  const badDesc = yamlJson.metadata.description !== packageJson.description;
+
+  if (badOwner || badTitle || badName || badType || badDesc) {
+    const owner = badOwner
+      ? getOwnerFromCodeowners(codeowners, yamlPath)
+      : yamlJson.spec?.owner;
+    const newJson = createOrMergeEntity(packageJson, owner, yamlJson);
+    return dryRun
+      ? Promise.resolve(console.error(`Update ${relativePath('.', yamlPath)}`))
+      : writeFile(yamlPath, yamlOverwrite(yamlString, newJson));
+  }
+
+  return Promise.resolve();
+}
+
+/**
+ * Canonical representation on how to create (or update) a backstage-package
+ * component.
+ */
+function createOrMergeEntity(
+  packageJson: BackstagePackageJson,
+  owner: string,
+  existingEntity: BackstagePackageEntity | Record<string, any> = {},
+): BackstagePackageEntity {
+  const safeEntityName = packageJson.name
+    .replace(/[^a-z0-9_\-\.]+/g, '-')
+    .replace(/^[^a-z0-9]|[^a-z0-9]$/g, '');
+
+  return {
+    ...existingEntity,
+    apiVersion: 'backstage.io/v1alpha1',
+    kind: 'Component',
+    metadata: {
+      ...existingEntity.metadata,
+      // Provide default name/title/description values.
+      name: safeEntityName,
+      title: packageJson.name,
+      ...(packageJson.description
+        ? { description: packageJson.description }
+        : undefined),
+    },
+    spec: {
+      lifecycle: 'experimental',
+      ...existingEntity.spec,
+      type: `backstage-${packageJson.backstage.role}`,
+      owner,
+    },
+  };
+}

packages/repo-tools/src/commands/generate-catalog-info/utils.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2023 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import fs from 'fs';
+import { Package } from '@manypkg/get-packages';
+import {
+  BackstagePackageJson as BackstagePackageJsonActual,
+  PackageRole,
+} from '@backstage/cli-node';
+import { promisify } from 'util';
+
+export const readFile = promisify(fs.readFile);
+export const writeFile = promisify(fs.writeFile);
+
+export type BackstagePackageJson = BackstagePackageJsonActual & {
+  description?: string;
+  backstage: {
+    role: PackageRole;
+  };
+};
+
+export function isBackstagePackage(
+  packageJson: Package['packageJson'],
+): packageJson is BackstagePackageJson {
+  return (
+    packageJson &&
+    packageJson.hasOwnProperty('backstage') &&
+    (packageJson as any)?.backstage?.role !== 'undefined'
+  );
+}

packages/repo-tools/src/commands/index.ts

@@ -96,6 +96,21 @@ export function registerCommands(program: Command) {
     .description('Find inconsistencies in types of all packages and plugins')
     .action(lazy(() => import('./type-deps/type-deps').then(m => m.default)));
 
+  program
+    .command('generate-catalog-info')
+    .option(
+      '--dry-run',
+      'Shows what would happen without actually writing any yaml.',
+    )
+    .description('Create or fix info yaml files for all backstage packages')
+    .action(
+      lazy(() =>
+        import('./generate-catalog-info/generate-catalog-info').then(
+          m => m.default,
+        ),
+      ),
+    );
+
   registerSchemaCommand(program);
 }
 

yarn.lock

@@ -9538,6 +9538,7 @@ __metadata:
   dependencies:
     "@apidevtools/swagger-parser": ^10.1.0
     "@apisyouwonthate/style-guide": ^1.4.0
+    "@backstage/catalog-model": "workspace:^"
     "@backstage/cli": "workspace:^"
     "@backstage/cli-common": "workspace:^"
     "@backstage/cli-node": "workspace:^"
@@ -9557,6 +9558,7 @@ __metadata:
     "@types/mock-fs": ^4.13.0
     "@types/node": ^16.11.26
     chalk: ^4.0.0
+    codeowners-utils: ^1.0.2
     commander: ^9.1.0
     fs-extra: 10.1.0
     glob: ^8.0.3
@@ -9567,6 +9569,7 @@ __metadata:
     mock-fs: ^5.1.0
     p-limit: ^3.0.2
     ts-node: ^10.0.0
+    yaml-diff-patch: ^2.0.0
   peerDependencies:
     "@microsoft/api-extractor-model": "*"
     "@microsoft/tsdoc": "*"
@@ -25565,7 +25568,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"fast-json-patch@npm:^3.0.0-1":
+"fast-json-patch@npm:^3.0.0-1, fast-json-patch@npm:^3.1.0":
   version: 3.1.1
   resolution: "fast-json-patch@npm:3.1.1"
   checksum: c4525b61b2471df60d4b025b4118b036d99778a93431aa44d1084218182841d82ce93056f0f3bbd731a24e6a8e69820128adf1873eb2199a26c62ef58d137833
@@ -33855,6 +33858,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"oppa@npm:^0.4.0":
+  version: 0.4.0
+  resolution: "oppa@npm:0.4.0"
+  dependencies:
+    chalk: ^4.1.1
+  checksum: ecc43e63ede05c3ccb10e0f2c3f3020a6d72e1a3b318f3e37b8cc8a1a279e300991c043e5385d560c1eebb54a56c7f9b69bf0db0d1933acf350bcd2980c96055
+  languageName: node
+  linkType: hard
+
 "optimism@npm:^0.17.5":
   version: 0.17.5
   resolution: "optimism@npm:0.17.5"
@@ -42567,6 +42579,21 @@ __metadata:
   languageName: node
   linkType: hard
 
+"yaml-diff-patch@npm:^2.0.0":
+  version: 2.0.0
+  resolution: "yaml-diff-patch@npm:2.0.0"
+  dependencies:
+    fast-json-patch: ^3.1.0
+    oppa: ^0.4.0
+    yaml: ^2.0.0-10
+  bin:
+    yaml-diff-patch: dist/bin/yaml-patch.js
+    yaml-overwrite: dist/bin/yaml-patch.js
+    yaml-patch: dist/bin/yaml-patch.js
+  checksum: 5207d8523584eb6088fe32a0c6010599260ecfa5f959d120a1bad02f19143d1ddeafe10c37ccf125ac04d079072a5ead92b55c6787fd64d12f5acbb0d172e7ec
+  languageName: node
+  linkType: hard
+
 "yaml@npm:^1.10.0, yaml@npm:^1.10.2, yaml@npm:^1.7.2":
   version: 1.10.2
   resolution: "yaml@npm:1.10.2"
@@ -42574,10 +42601,10 @@ __metadata:
   languageName: node
   linkType: hard
 
-"yaml@npm:^2.0.0, yaml@npm:^2.1.1, yaml@npm:^2.2.1, yaml@npm:^2.2.2":
-  version: 2.2.2
-  resolution: "yaml@npm:2.2.2"
-  checksum: d90c235e099e30094dcff61ba3350437aef53325db4a6bcd04ca96e1bfe7e348b191f6a7a52b5211e2dbc4eeedb22a00b291527da030de7c189728ef3f2b4eb3
+"yaml@npm:^2.0.0, yaml@npm:^2.0.0-10, yaml@npm:^2.1.1, yaml@npm:^2.2.1, yaml@npm:^2.2.2":
+  version: 2.3.1
+  resolution: "yaml@npm:2.3.1"
+  checksum: 2c7bc9a7cd4c9f40d3b0b0a98e370781b68b8b7c4515720869aced2b00d92f5da1762b4ffa947f9e795d6cd6b19f410bd4d15fdd38aca7bd96df59bd9486fb54
   languageName: node
   linkType: hard