From 9ebc89735160cce7b18c89d1a3bc69ceddf268d2 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Wed, 25 Nov 2020 01:56:55 +0100 Subject: [PATCH 001/188] TechDocs: Refactor Publisher to pave way for external publishers --- app-config.yaml | 5 + packages/backend/src/plugins/techdocs.ts | 4 +- .../packages/backend/src/plugins/techdocs.ts | 4 +- .../techdocs-backend/src/service/router.ts | 11 +- .../src/service/standaloneServer.ts | 17 ++- .../src/techdocs/stages/publish/index.ts | 5 +- .../src/techdocs/stages/publish/local.ts | 41 +++++-- .../src/techdocs/stages/publish/publish.ts | 111 ++++++++++++++++++ .../src/techdocs/stages/publish/types.ts | 18 +-- 9 files changed, 172 insertions(+), 44 deletions(-) create mode 100644 plugins/techdocs-backend/src/techdocs/stages/publish/publish.ts diff --git a/app-config.yaml b/app-config.yaml index 3b476981e4..f381f71ead 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -60,6 +60,11 @@ techdocs: requestUrl: http://localhost:7000/api/techdocs generators: techdocs: 'docker' + publisher: + # Ref: + type: 'local' # other options - google_gcs, aws_s3, azure_storage, etc. + # if type set to google_gcs + # google_token: 'abc' # with write access sentry: organization: my-company diff --git a/packages/backend/src/plugins/techdocs.ts b/packages/backend/src/plugins/techdocs.ts index de48280e64..3669911c0c 100644 --- a/packages/backend/src/plugins/techdocs.ts +++ b/packages/backend/src/plugins/techdocs.ts @@ -19,10 +19,10 @@ import { DirectoryPreparer, Preparers, Generators, - LocalPublish, TechdocsGenerator, CommonGitPreparer, UrlPreparer, + Publisher, } from '@backstage/plugin-techdocs-backend'; import { PluginEnvironment } from '../types'; import Docker from 'dockerode'; @@ -50,7 +50,7 @@ export default async function createPlugin({ const urlPreparer = new UrlPreparer(reader, logger); preparers.register('url', urlPreparer); - const publisher = new LocalPublish(logger, discovery); + const publisher = new Publisher(logger, config, discovery); const dockerClient = new Docker(); diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts index ac4d81a8e8..3b8df8ef21 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts @@ -4,7 +4,7 @@ import { CommonGitPreparer, Preparers, Generators, - LocalPublish, + Publisher, TechdocsGenerator, } from '@backstage/plugin-techdocs-backend'; import { PluginEnvironment } from '../types'; @@ -28,7 +28,7 @@ export default async function createPlugin({ preparers.register('github', commonGitPreparer); preparers.register('gitlab', commonGitPreparer); - const publisher = new LocalPublish(logger, discovery); + const publisher = new Publisher(logger, config, discovery); const dockerClient = new Docker(); diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index 625252dd7e..61f0fa2e9f 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -20,12 +20,7 @@ import Knex from 'knex'; import fetch from 'cross-fetch'; import { Config } from '@backstage/config'; import Docker from 'dockerode'; -import { - GeneratorBuilder, - PreparerBuilder, - PublisherBase, - LocalPublish, -} from '../techdocs'; +import { GeneratorBuilder, PreparerBuilder, PublisherBase } from '../techdocs'; import { PluginEndpointDiscovery, resolvePackagePath, @@ -63,7 +58,7 @@ export async function createRouter({ router.get('/metadata/techdocs/*', async (req, res) => { let storageUrl = config.getString('techdocs.storageUrl'); - if (publisher instanceof LocalPublish) { + if (publisher.isLocalPublisher()) { storageUrl = new URL( new URL(storageUrl).pathname, await discovery.getBaseUrl('techdocs'), @@ -140,7 +135,7 @@ export async function createRouter({ res.redirect(`${storageUrl}${req.path.replace('/docs', '')}`); }); - if (publisher instanceof LocalPublish) { + if (publisher.isLocalPublisher()) { router.use('/static/docs', express.static(staticDocsDir)); } diff --git a/plugins/techdocs-backend/src/service/standaloneServer.ts b/plugins/techdocs-backend/src/service/standaloneServer.ts index 4082a36172..89b49467d4 100644 --- a/plugins/techdocs-backend/src/service/standaloneServer.ts +++ b/plugins/techdocs-backend/src/service/standaloneServer.ts @@ -27,7 +27,7 @@ import { DirectoryPreparer, Generators, TechdocsGenerator, - LocalPublish, + Publisher, } from '../techdocs'; import { ConfigReader } from '@backstage/config'; @@ -41,7 +41,18 @@ export async function startStandaloneServer( options: ServerOptions, ): Promise { const logger = options.logger.child({ service: 'techdocs-backend' }); - const config = ConfigReader.fromConfigs([]); + const config = ConfigReader.fromConfigs([ + { + context: '', + data: { + techdocs: { + publisher: { + type: 'local', + }, + }, + }, + }, + ]); const discovery = SingleHostDiscovery.fromConfig(config); logger.debug('Creating application...'); @@ -53,7 +64,7 @@ export async function startStandaloneServer( const techdocsGenerator = new TechdocsGenerator(logger, config); generators.register('techdocs', techdocsGenerator); - const publisher = new LocalPublish(logger, discovery); + const publisher = new Publisher(logger, config, discovery); const dockerClient = new Docker(); diff --git a/plugins/techdocs-backend/src/techdocs/stages/publish/index.ts b/plugins/techdocs-backend/src/techdocs/stages/publish/index.ts index 0029ea5c4e..9a21217083 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/publish/index.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/publish/index.ts @@ -13,5 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -export { LocalPublish } from './local'; -export type { PublisherBase } from './types'; +export { Publisher } from './publish'; +export type { PublisherBase } from './publish'; +export type { PublisherType } from './types'; diff --git a/plugins/techdocs-backend/src/techdocs/stages/publish/local.ts b/plugins/techdocs-backend/src/techdocs/stages/publish/local.ts index 464c4b90a0..712bf8266d 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/publish/local.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/publish/local.ts @@ -16,13 +16,38 @@ import fs from 'fs-extra'; import { Logger } from 'winston'; import { Entity } from '@backstage/catalog-model'; -import { PublisherBase } from './types'; import { resolvePackagePath, PluginEndpointDiscovery, } from '@backstage/backend-common'; -export class LocalPublish implements PublisherBase { +export type LocalPublishParams = { + entity: Entity; + directory: string; +}; + +export type LocalPublishReturn = + | Promise<{ remoteUrl: string }> + | { remoteUrl: string }; + +/** + * Type for the local publisher which uses local filesystem to store the generated static files. + * + * It uses a directory called "static" at the root of techdocs-backend plugin. + */ +export interface LocalPublisher { + /** + * Store the generated files inside a static folder in local filesystem. + * + * @param {LocalPublishParams} opts Object containing the entity from the service + * catalog, and the directory that contains the generated static files from TechDocs. + * @returns {LocalPublishReturn} Either a promise or an object with `remoteUrl` which is the URL + * which serves files from the local publisher's static directory. + */ + publish(opts: LocalPublishParams): LocalPublishReturn; +} + +export class LocalPublish { private readonly logger: Logger; private readonly discovery: PluginEndpointDiscovery; @@ -31,17 +56,7 @@ export class LocalPublish implements PublisherBase { this.discovery = discovery; } - publish({ - entity, - directory, - }: { - entity: Entity; - directory: string; - }): - | Promise<{ - remoteUrl: string; - }> - | { remoteUrl: string } { + publish({ entity, directory }: LocalPublishParams): LocalPublishReturn { const entityNamespace = entity.metadata.namespace ?? 'default'; const publishDir = resolvePackagePath( diff --git a/plugins/techdocs-backend/src/techdocs/stages/publish/publish.ts b/plugins/techdocs-backend/src/techdocs/stages/publish/publish.ts new file mode 100644 index 0000000000..bf01812fa2 --- /dev/null +++ b/plugins/techdocs-backend/src/techdocs/stages/publish/publish.ts @@ -0,0 +1,111 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Logger } from 'winston'; +import { Config } from '@backstage/config'; +import { PluginEndpointDiscovery } from '@backstage/backend-common'; +import { Entity } from '@backstage/catalog-model'; +import { PublisherType } from './types'; +import { LocalPublish } from './local'; + +export type PublisherBaseParams = { + entity: Entity; + directory: string; +}; + +export type PublisherBaseReturn = + | Promise<{ remoteUrl: string }> + | { remoteUrl: string }; + +/** + * Type for the publisher instance registered with backend which manages publishing of the + * generated static files after the prepare and generate steps of TechDocs. + * + * Depending upon the value of techdocs.publisher.type in app config, this instance creates + * and uses a publisher of the particular type i.e. local, google_gcs, aws_s3, etc. It defaults + * to use the local publisher. + */ +export interface PublisherBase { + /** + * Invoke the app's configured publisher's publish method. + * + * @param {PublisherBaseParams} opts Object containing the entity from the service + * catalog, and the directory that contains the generated static files from TechDocs. + */ + publish(opts: PublisherBaseParams): PublisherBaseReturn; + + /** + * Return true if local filesystem is being used to store generated files for TechDocs. + */ + isLocalPublisher(): boolean; + + /** + * Return true if an external cloud storage (GCS, S3, SFTP server, etc.) is being used to + * store generated files for TechDocs. + */ + isExternalPublisher(): boolean; +} + +export class Publisher implements PublisherBase { + private readonly logger: Logger; + private readonly config: Config; + private readonly discovery: PluginEndpointDiscovery; + private readonly publisherType: PublisherType; + private readonly publisher: any; + + constructor( + logger: Logger, + config: Config, + discovery: PluginEndpointDiscovery, + ) { + this.logger = logger; + this.config = config; + this.discovery = discovery; + + this.publisherType = + (this.config.getOptionalString( + 'techdocs.publisher.type', + ) as PublisherType) ?? 'local'; + + switch (this.publisherType) { + case 'google_gcs': + this.logger.info( + 'Creating Google Storage Bucket publisher for TechDocs', + ); + this.publisher = new LocalPublish(this.logger, this.discovery); + break; + case 'local': + this.logger.info('Creating Local publisher for TechDocs'); + this.publisher = new LocalPublish(this.logger, this.discovery); + break; + default: + this.logger.info('Creating Local publisher for TechDocs'); + this.publisher = new LocalPublish(this.logger, this.discovery); + break; + } + } + + publish({ entity, directory }: PublisherBaseParams): PublisherBaseReturn { + return this.publisher.publish({ entity, directory }); + } + + isLocalPublisher(): boolean { + return this.publisherType === 'local'; + } + + isExternalPublisher(): boolean { + return this.publisherType !== 'local'; + } +} diff --git a/plugins/techdocs-backend/src/techdocs/stages/publish/types.ts b/plugins/techdocs-backend/src/techdocs/stages/publish/types.ts index ca85d9f56e..2c2348e37a 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/publish/types.ts +++ b/plugins/techdocs-backend/src/techdocs/stages/publish/types.ts @@ -13,20 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Entity } from '@backstage/catalog-model'; /** - * Publisher is in charge of taking a folder created by - * the builder, and pushing it to storage + * Key for all the different types of TechDocs publishers available. */ -export type PublisherBase = { - /** - * - * @param opts object containing the entity from the service - * catalog, and the directory that has been generated - */ - publish(opts: { - entity: Entity; - directory: string; - }): Promise<{ remoteUrl: string }> | { remoteUrl: string }; -}; +export type PublisherType = 'local' | 'google_gcs'; + +export interface ExternalPublisher {} From ff348df9fefb4f150d7225fa561e8de3d62eae39 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Wed, 25 Nov 2020 19:21:21 +0100 Subject: [PATCH 002/188] techdocs-common: Initialize package and move common code from techdocs-backend --- packages/backend/package.json | 1 + packages/backend/src/plugins/techdocs.ts | 4 +- packages/techdocs-common/.eslintrc.js | 3 + packages/techdocs-common/README.md | 7 +++ packages/techdocs-common/package.json | 55 +++++++++++++++++++ .../techdocs-common}/src/default-branch.ts | 0 .../techdocs-common}/src/git-auth.ts | 0 .../techdocs-common}/src/helpers.test.ts | 0 .../techdocs-common}/src/helpers.ts | 2 +- .../techdocs-common/src}/index.ts | 3 + .../stages/generate/__fixtures__/mkdocs.yml | 0 .../__fixtures__/mkdocs_with_repo_url.yml | 0 .../src}/stages/generate/generators.test.ts | 2 +- .../src}/stages/generate/generators.ts | 0 .../src}/stages/generate/helpers.test.ts | 2 +- .../src}/stages/generate/helpers.ts | 2 +- .../src}/stages/generate/index.ts | 0 .../src}/stages/generate/techdocs.ts | 0 .../src}/stages/generate/types.ts | 2 +- .../techdocs-common/src}/stages/index.ts | 0 .../src}/stages/prepare/commonGit.test.ts | 6 +- .../src}/stages/prepare/commonGit.ts | 5 +- .../src}/stages/prepare/dir.test.ts | 6 +- .../src}/stages/prepare/dir.ts | 5 +- .../src}/stages/prepare/index.ts | 0 .../src}/stages/prepare/preparers.ts | 2 +- .../src}/stages/prepare/types.ts | 0 .../src}/stages/prepare/url.ts | 3 +- .../src}/stages/publish/index.ts | 0 .../src}/stages/publish/local.test.ts | 4 +- .../src}/stages/publish/local.ts | 0 .../src}/stages/publish/publish.ts | 0 .../test-component-folder-AP2Wti/mock-file | 0 .../src}/stages/publish/types.ts | 0 plugins/techdocs-backend/package.json | 7 +-- plugins/techdocs-backend/src/index.ts | 1 - .../techdocs-backend/src/service/helpers.ts | 5 +- .../techdocs-backend/src/service/router.ts | 8 ++- .../src/service/standaloneServer.ts | 2 +- plugins/techdocs/package.json | 1 + 40 files changed, 100 insertions(+), 38 deletions(-) create mode 100644 packages/techdocs-common/.eslintrc.js create mode 100644 packages/techdocs-common/README.md create mode 100644 packages/techdocs-common/package.json rename {plugins/techdocs-backend => packages/techdocs-common}/src/default-branch.ts (100%) rename {plugins/techdocs-backend => packages/techdocs-common}/src/git-auth.ts (100%) rename {plugins/techdocs-backend => packages/techdocs-common}/src/helpers.test.ts (100%) rename {plugins/techdocs-backend => packages/techdocs-common}/src/helpers.ts (98%) rename {plugins/techdocs-backend/src/techdocs => packages/techdocs-common/src}/index.ts (87%) rename {plugins/techdocs-backend/src/techdocs => packages/techdocs-common/src}/stages/generate/__fixtures__/mkdocs.yml (100%) rename {plugins/techdocs-backend/src/techdocs => packages/techdocs-common/src}/stages/generate/__fixtures__/mkdocs_with_repo_url.yml (100%) rename {plugins/techdocs-backend/src/techdocs => packages/techdocs-common/src}/stages/generate/generators.test.ts (96%) rename {plugins/techdocs-backend/src/techdocs => packages/techdocs-common/src}/stages/generate/generators.ts (100%) rename {plugins/techdocs-backend/src/techdocs => packages/techdocs-common/src}/stages/generate/helpers.test.ts (99%) rename {plugins/techdocs-backend/src/techdocs => packages/techdocs-common/src}/stages/generate/helpers.ts (99%) rename {plugins/techdocs-backend/src/techdocs => packages/techdocs-common/src}/stages/generate/index.ts (100%) rename {plugins/techdocs-backend/src/techdocs => packages/techdocs-common/src}/stages/generate/techdocs.ts (100%) rename {plugins/techdocs-backend/src/techdocs => packages/techdocs-common/src}/stages/generate/types.ts (97%) rename {plugins/techdocs-backend/src/techdocs => packages/techdocs-common/src}/stages/index.ts (100%) rename {plugins/techdocs-backend/src/techdocs => packages/techdocs-common/src}/stages/prepare/commonGit.test.ts (95%) rename {plugins/techdocs-backend/src/techdocs => packages/techdocs-common/src}/stages/prepare/commonGit.ts (94%) rename {plugins/techdocs-backend/src/techdocs => packages/techdocs-common/src}/stages/prepare/dir.test.ts (95%) rename {plugins/techdocs-backend/src/techdocs => packages/techdocs-common/src}/stages/prepare/dir.ts (96%) rename {plugins/techdocs-backend/src/techdocs => packages/techdocs-common/src}/stages/prepare/index.ts (100%) rename {plugins/techdocs-backend/src/techdocs => packages/techdocs-common/src}/stages/prepare/preparers.ts (95%) rename {plugins/techdocs-backend/src/techdocs => packages/techdocs-common/src}/stages/prepare/types.ts (100%) rename {plugins/techdocs-backend/src/techdocs => packages/techdocs-common/src}/stages/prepare/url.ts (95%) rename {plugins/techdocs-backend/src/techdocs => packages/techdocs-common/src}/stages/publish/index.ts (100%) rename {plugins/techdocs-backend/src/techdocs => packages/techdocs-common/src}/stages/publish/local.test.ts (90%) rename {plugins/techdocs-backend/src/techdocs => packages/techdocs-common/src}/stages/publish/local.ts (100%) rename {plugins/techdocs-backend/src/techdocs => packages/techdocs-common/src}/stages/publish/publish.ts (100%) create mode 100644 packages/techdocs-common/src/stages/publish/test-component-folder-AP2Wti/mock-file rename {plugins/techdocs-backend/src/techdocs => packages/techdocs-common/src}/stages/publish/types.ts (100%) diff --git a/packages/backend/package.json b/packages/backend/package.json index beb2567bd7..158ff043b2 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -31,6 +31,7 @@ "@backstage/plugin-scaffolder-backend": "^0.3.2", "@backstage/plugin-sentry-backend": "^0.1.3", "@backstage/plugin-techdocs-backend": "^0.3.0", + "@backstage/techdocs-common": "^0.1.1", "@gitbeaker/node": "^25.2.0", "@octokit/rest": "^18.0.0", "azure-devops-node-api": "^10.1.1", diff --git a/packages/backend/src/plugins/techdocs.ts b/packages/backend/src/plugins/techdocs.ts index 3669911c0c..25d17801ea 100644 --- a/packages/backend/src/plugins/techdocs.ts +++ b/packages/backend/src/plugins/techdocs.ts @@ -14,8 +14,8 @@ * limitations under the License. */ +import { createRouter } from '@backstage/plugin-techdocs-backend'; import { - createRouter, DirectoryPreparer, Preparers, Generators, @@ -23,7 +23,7 @@ import { CommonGitPreparer, UrlPreparer, Publisher, -} from '@backstage/plugin-techdocs-backend'; +} from '@backstage/techdocs-common'; import { PluginEnvironment } from '../types'; import Docker from 'dockerode'; diff --git a/packages/techdocs-common/.eslintrc.js b/packages/techdocs-common/.eslintrc.js new file mode 100644 index 0000000000..16a033dbc6 --- /dev/null +++ b/packages/techdocs-common/.eslintrc.js @@ -0,0 +1,3 @@ +module.exports = { + extends: [require.resolve('@backstage/cli/config/eslint.backend')], +}; diff --git a/packages/techdocs-common/README.md b/packages/techdocs-common/README.md new file mode 100644 index 0000000000..acf267bbfb --- /dev/null +++ b/packages/techdocs-common/README.md @@ -0,0 +1,7 @@ +# @backstage/techdocs-common + +Common functionalities for TechDocs, to be shared between techdocs plugins and techdocs-cli + +## Usage + +TODO: List supported APIs diff --git a/packages/techdocs-common/package.json b/packages/techdocs-common/package.json new file mode 100644 index 0000000000..6edb1fc589 --- /dev/null +++ b/packages/techdocs-common/package.json @@ -0,0 +1,55 @@ +{ + "name": "@backstage/techdocs-common", + "description": "Common functionalities for TechDocs, to be shared between techdocs plugins and techdocs-cli", + "version": "0.1.1", + "main": "src/index.ts", + "types": "src/index.ts", + "private": false, + "publishConfig": { + "access": "public", + "main": "dist/index.cjs.js", + "types": "dist/index.d.ts" + }, + "homepage": "https://backstage.io", + "repository": { + "type": "git", + "url": "https://github.com/backstage/backstage", + "directory": "packages/techdocs-common" + }, + "keywords": [ + "techdocs", + "backstage" + ], + "license": "Apache-2.0", + "files": [ + "dist" + ], + "scripts": { + "build": "backstage-cli build --outputs cjs,types", + "lint": "backstage-cli lint", + "test": "backstage-cli test", + "prepack": "backstage-cli prepack", + "postpack": "backstage-cli postpack", + "clean": "backstage-cli clean" + }, + "bugs": { + "url": "https://github.com/backstage/backstage/issues" + }, + "dependencies": { + "@backstage/backend-common": "^0.3.2", + "@backstage/catalog-model": "^0.3.1", + "@backstage/config": "^0.1.1", + "cross-fetch": "^3.0.6", + "dockerode": "^3.2.1", + "fs-extra": "^9.0.1", + "git-url-parse": "^11.4.0", + "js-yaml": "^3.14.0", + "mock-fs": "^4.13.0", + "nodegit": "^0.27.0", + "@types/dockerode": "^2.5.34", + "winston": "^3.2.1" + }, + "devDependencies": { + "@backstage/cli": "^0.3.1" + } +} diff --git a/plugins/techdocs-backend/src/default-branch.ts b/packages/techdocs-common/src/default-branch.ts similarity index 100% rename from plugins/techdocs-backend/src/default-branch.ts rename to packages/techdocs-common/src/default-branch.ts diff --git a/plugins/techdocs-backend/src/git-auth.ts b/packages/techdocs-common/src/git-auth.ts similarity index 100% rename from plugins/techdocs-backend/src/git-auth.ts rename to packages/techdocs-common/src/git-auth.ts diff --git a/plugins/techdocs-backend/src/helpers.test.ts b/packages/techdocs-common/src/helpers.test.ts similarity index 100% rename from plugins/techdocs-backend/src/helpers.test.ts rename to packages/techdocs-common/src/helpers.test.ts diff --git a/plugins/techdocs-backend/src/helpers.ts b/packages/techdocs-common/src/helpers.ts similarity index 98% rename from plugins/techdocs-backend/src/helpers.ts rename to packages/techdocs-common/src/helpers.ts index fcf5a4f194..e08d58d58e 100644 --- a/plugins/techdocs-backend/src/helpers.ts +++ b/packages/techdocs-common/src/helpers.ts @@ -23,7 +23,7 @@ import { getDefaultBranch } from './default-branch'; import { getGitRepoType, getTokenForGitRepo } from './git-auth'; import { Entity } from '@backstage/catalog-model'; import { InputError, UrlReader } from '@backstage/backend-common'; -import { RemoteProtocol } from './techdocs/stages/prepare/types'; +import { RemoteProtocol } from './stages/prepare/types'; import { Logger } from 'winston'; // Enables core.longpaths on windows to prevent crashing when checking out repos with long foldernames and/or deep nesting diff --git a/plugins/techdocs-backend/src/techdocs/index.ts b/packages/techdocs-common/src/index.ts similarity index 87% rename from plugins/techdocs-backend/src/techdocs/index.ts rename to packages/techdocs-common/src/index.ts index 7113525bb8..a6e1831049 100644 --- a/plugins/techdocs-backend/src/techdocs/index.ts +++ b/packages/techdocs-common/src/index.ts @@ -14,3 +14,6 @@ * limitations under the License. */ export * from './stages'; +export * from './helpers'; +export * from './default-branch'; +export * from './git-auth'; diff --git a/plugins/techdocs-backend/src/techdocs/stages/generate/__fixtures__/mkdocs.yml b/packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs.yml similarity index 100% rename from plugins/techdocs-backend/src/techdocs/stages/generate/__fixtures__/mkdocs.yml rename to packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs.yml diff --git a/plugins/techdocs-backend/src/techdocs/stages/generate/__fixtures__/mkdocs_with_repo_url.yml b/packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_with_repo_url.yml similarity index 100% rename from plugins/techdocs-backend/src/techdocs/stages/generate/__fixtures__/mkdocs_with_repo_url.yml rename to packages/techdocs-common/src/stages/generate/__fixtures__/mkdocs_with_repo_url.yml diff --git a/plugins/techdocs-backend/src/techdocs/stages/generate/generators.test.ts b/packages/techdocs-common/src/stages/generate/generators.test.ts similarity index 96% rename from plugins/techdocs-backend/src/techdocs/stages/generate/generators.test.ts rename to packages/techdocs-common/src/stages/generate/generators.test.ts index a9303c3794..7f4a964ec7 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/generate/generators.test.ts +++ b/packages/techdocs-common/src/stages/generate/generators.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { Generators, TechdocsGenerator } from './'; +import { Generators, TechdocsGenerator } from '.'; import { getVoidLogger } from '@backstage/backend-common'; import { ConfigReader } from '@backstage/config'; diff --git a/plugins/techdocs-backend/src/techdocs/stages/generate/generators.ts b/packages/techdocs-common/src/stages/generate/generators.ts similarity index 100% rename from plugins/techdocs-backend/src/techdocs/stages/generate/generators.ts rename to packages/techdocs-common/src/stages/generate/generators.ts diff --git a/plugins/techdocs-backend/src/techdocs/stages/generate/helpers.test.ts b/packages/techdocs-common/src/stages/generate/helpers.test.ts similarity index 99% rename from plugins/techdocs-backend/src/techdocs/stages/generate/helpers.test.ts rename to packages/techdocs-common/src/stages/generate/helpers.test.ts index 54ef0b1e83..83ccbbffab 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/generate/helpers.test.ts +++ b/packages/techdocs-common/src/stages/generate/helpers.test.ts @@ -28,7 +28,7 @@ import { patchMkdocsYmlPreBuild, } from './helpers'; import { RemoteProtocol } from '../prepare/types'; -import { ParsedLocationAnnotation } from '../../../helpers'; +import { ParsedLocationAnnotation } from '../../helpers'; const mockEntity = { apiVersion: 'version', diff --git a/plugins/techdocs-backend/src/techdocs/stages/generate/helpers.ts b/packages/techdocs-common/src/stages/generate/helpers.ts similarity index 99% rename from plugins/techdocs-backend/src/techdocs/stages/generate/helpers.ts rename to packages/techdocs-common/src/stages/generate/helpers.ts index 0f60712f5e..13344b8542 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/generate/helpers.ts +++ b/packages/techdocs-common/src/stages/generate/helpers.ts @@ -22,7 +22,7 @@ import yaml from 'js-yaml'; import { Logger } from 'winston'; import { Entity } from '@backstage/catalog-model'; import { SupportedGeneratorKey } from './types'; -import { ParsedLocationAnnotation } from '../../../helpers'; +import { ParsedLocationAnnotation } from '../../helpers'; import { RemoteProtocol } from '../prepare/types'; // TODO: Implement proper support for more generators. diff --git a/plugins/techdocs-backend/src/techdocs/stages/generate/index.ts b/packages/techdocs-common/src/stages/generate/index.ts similarity index 100% rename from plugins/techdocs-backend/src/techdocs/stages/generate/index.ts rename to packages/techdocs-common/src/stages/generate/index.ts diff --git a/plugins/techdocs-backend/src/techdocs/stages/generate/techdocs.ts b/packages/techdocs-common/src/stages/generate/techdocs.ts similarity index 100% rename from plugins/techdocs-backend/src/techdocs/stages/generate/techdocs.ts rename to packages/techdocs-common/src/stages/generate/techdocs.ts diff --git a/plugins/techdocs-backend/src/techdocs/stages/generate/types.ts b/packages/techdocs-common/src/stages/generate/types.ts similarity index 97% rename from plugins/techdocs-backend/src/techdocs/stages/generate/types.ts rename to packages/techdocs-common/src/stages/generate/types.ts index 6d9ce5afca..65e411c3aa 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/generate/types.ts +++ b/packages/techdocs-common/src/stages/generate/types.ts @@ -16,7 +16,7 @@ import { Writable } from 'stream'; import Docker from 'dockerode'; import { Entity } from '@backstage/catalog-model'; -import { ParsedLocationAnnotation } from '../../../helpers'; +import { ParsedLocationAnnotation } from '../../helpers'; /** * The returned directory from the generator which is ready diff --git a/plugins/techdocs-backend/src/techdocs/stages/index.ts b/packages/techdocs-common/src/stages/index.ts similarity index 100% rename from plugins/techdocs-backend/src/techdocs/stages/index.ts rename to packages/techdocs-common/src/stages/index.ts diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/commonGit.test.ts b/packages/techdocs-common/src/stages/prepare/commonGit.test.ts similarity index 95% rename from plugins/techdocs-backend/src/techdocs/stages/prepare/commonGit.test.ts rename to packages/techdocs-common/src/stages/prepare/commonGit.test.ts index 843df476e9..a4240514b5 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/prepare/commonGit.test.ts +++ b/packages/techdocs-common/src/stages/prepare/commonGit.test.ts @@ -16,7 +16,7 @@ import { getVoidLogger } from '@backstage/backend-common'; import { CommonGitPreparer } from './commonGit'; -import { checkoutGitRepository } from '../../../helpers'; +import { checkoutGitRepository } from '../../helpers'; function normalizePath(path: string) { return path @@ -25,8 +25,8 @@ function normalizePath(path: string) { .join('/'); } -jest.mock('../../../helpers', () => ({ - ...jest.requireActual<{}>('../../../helpers'), +jest.mock('../../helpers', () => ({ + ...jest.requireActual<{}>('../../helpers'), checkoutGitRepository: jest.fn(() => '/tmp/backstage-repo/org/name/branch'), })); diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/commonGit.ts b/packages/techdocs-common/src/stages/prepare/commonGit.ts similarity index 94% rename from plugins/techdocs-backend/src/techdocs/stages/prepare/commonGit.ts rename to packages/techdocs-common/src/stages/prepare/commonGit.ts index d9ba96a031..d79373fba3 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/prepare/commonGit.ts +++ b/packages/techdocs-common/src/stages/prepare/commonGit.ts @@ -17,10 +17,7 @@ import path from 'path'; import { Entity } from '@backstage/catalog-model'; import { PreparerBase } from './types'; import parseGitUrl from 'git-url-parse'; -import { - parseReferenceAnnotation, - checkoutGitRepository, -} from '../../../helpers'; +import { parseReferenceAnnotation, checkoutGitRepository } from '../../helpers'; import { Logger } from 'winston'; diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/dir.test.ts b/packages/techdocs-common/src/stages/prepare/dir.test.ts similarity index 95% rename from plugins/techdocs-backend/src/techdocs/stages/prepare/dir.test.ts rename to packages/techdocs-common/src/stages/prepare/dir.test.ts index dc2b1d7d48..5b51f1f46d 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/prepare/dir.test.ts +++ b/packages/techdocs-common/src/stages/prepare/dir.test.ts @@ -15,7 +15,7 @@ */ import { DirectoryPreparer } from './dir'; import { getVoidLogger } from '@backstage/backend-common'; -import { checkoutGitRepository } from '../../../helpers'; +import { checkoutGitRepository } from '../../helpers'; function normalizePath(path: string) { return path @@ -24,8 +24,8 @@ function normalizePath(path: string) { .join('/'); } -jest.mock('../../../helpers', () => ({ - ...jest.requireActual<{}>('../../../helpers'), +jest.mock('../../helpers', () => ({ + ...jest.requireActual<{}>('../../helpers'), checkoutGitRepository: jest.fn(() => '/tmp/backstage-repo/org/name/branch/'), })); diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/dir.ts b/packages/techdocs-common/src/stages/prepare/dir.ts similarity index 96% rename from plugins/techdocs-backend/src/techdocs/stages/prepare/dir.ts rename to packages/techdocs-common/src/stages/prepare/dir.ts index 1effcf61b5..cebf70e52d 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/prepare/dir.ts +++ b/packages/techdocs-common/src/stages/prepare/dir.ts @@ -16,10 +16,7 @@ import { PreparerBase } from './types'; import { Entity } from '@backstage/catalog-model'; import path from 'path'; -import { - parseReferenceAnnotation, - checkoutGitRepository, -} from '../../../helpers'; +import { parseReferenceAnnotation, checkoutGitRepository } from '../../helpers'; import { InputError } from '@backstage/backend-common'; import parseGitUrl from 'git-url-parse'; import { Logger } from 'winston'; diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/index.ts b/packages/techdocs-common/src/stages/prepare/index.ts similarity index 100% rename from plugins/techdocs-backend/src/techdocs/stages/prepare/index.ts rename to packages/techdocs-common/src/stages/prepare/index.ts diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/preparers.ts b/packages/techdocs-common/src/stages/prepare/preparers.ts similarity index 95% rename from plugins/techdocs-backend/src/techdocs/stages/prepare/preparers.ts rename to packages/techdocs-common/src/stages/prepare/preparers.ts index 2f0df47de4..52a47957e8 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/prepare/preparers.ts +++ b/packages/techdocs-common/src/stages/prepare/preparers.ts @@ -16,7 +16,7 @@ import { PreparerBase, RemoteProtocol, PreparerBuilder } from './types'; import { Entity } from '@backstage/catalog-model'; -import { parseReferenceAnnotation } from '../../../helpers'; +import { parseReferenceAnnotation } from '../../helpers'; export class Preparers implements PreparerBuilder { private preparerMap = new Map(); diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/types.ts b/packages/techdocs-common/src/stages/prepare/types.ts similarity index 100% rename from plugins/techdocs-backend/src/techdocs/stages/prepare/types.ts rename to packages/techdocs-common/src/stages/prepare/types.ts diff --git a/plugins/techdocs-backend/src/techdocs/stages/prepare/url.ts b/packages/techdocs-common/src/stages/prepare/url.ts similarity index 95% rename from plugins/techdocs-backend/src/techdocs/stages/prepare/url.ts rename to packages/techdocs-common/src/stages/prepare/url.ts index 330db05aa4..3407684837 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/prepare/url.ts +++ b/packages/techdocs-common/src/stages/prepare/url.ts @@ -15,8 +15,7 @@ */ import { Entity } from '@backstage/catalog-model'; import { PreparerBase } from './types'; -import { getDocFilesFromRepository } from '../../../helpers'; - +import { getDocFilesFromRepository } from '../../helpers'; import { Logger } from 'winston'; import { UrlReader } from '@backstage/backend-common'; diff --git a/plugins/techdocs-backend/src/techdocs/stages/publish/index.ts b/packages/techdocs-common/src/stages/publish/index.ts similarity index 100% rename from plugins/techdocs-backend/src/techdocs/stages/publish/index.ts rename to packages/techdocs-common/src/stages/publish/index.ts diff --git a/plugins/techdocs-backend/src/techdocs/stages/publish/local.test.ts b/packages/techdocs-common/src/stages/publish/local.test.ts similarity index 90% rename from plugins/techdocs-backend/src/techdocs/stages/publish/local.test.ts rename to packages/techdocs-common/src/stages/publish/local.test.ts index 18344489e1..793c2374d9 100644 --- a/plugins/techdocs-backend/src/techdocs/stages/publish/local.test.ts +++ b/packages/techdocs-common/src/stages/publish/local.test.ts @@ -58,12 +58,12 @@ describe('local publisher', () => { await publisher.publish({ entity: mockEntity, directory: tempDir }); const publishDir = path.resolve( __dirname, - `../../../../static/docs/${mockEntity.metadata.name}`, + `../../../../../plugins/techdocs-backend/static/docs/${mockEntity.metadata.name}`, ); const resultDir = path.resolve( __dirname, - `../../../../static/docs/default/${mockEntity.kind}/${mockEntity.metadata.name}`, + `../../../../../plugins/techdocs-backend/static/docs/default/${mockEntity.kind}/${mockEntity.metadata.name}`, ); expect(fs.existsSync(resultDir)).toBeTruthy(); diff --git a/plugins/techdocs-backend/src/techdocs/stages/publish/local.ts b/packages/techdocs-common/src/stages/publish/local.ts similarity index 100% rename from plugins/techdocs-backend/src/techdocs/stages/publish/local.ts rename to packages/techdocs-common/src/stages/publish/local.ts diff --git a/plugins/techdocs-backend/src/techdocs/stages/publish/publish.ts b/packages/techdocs-common/src/stages/publish/publish.ts similarity index 100% rename from plugins/techdocs-backend/src/techdocs/stages/publish/publish.ts rename to packages/techdocs-common/src/stages/publish/publish.ts diff --git a/packages/techdocs-common/src/stages/publish/test-component-folder-AP2Wti/mock-file b/packages/techdocs-common/src/stages/publish/test-component-folder-AP2Wti/mock-file new file mode 100644 index 0000000000..e69de29bb2 diff --git a/plugins/techdocs-backend/src/techdocs/stages/publish/types.ts b/packages/techdocs-common/src/stages/publish/types.ts similarity index 100% rename from plugins/techdocs-backend/src/techdocs/stages/publish/types.ts rename to packages/techdocs-common/src/stages/publish/types.ts diff --git a/plugins/techdocs-backend/package.json b/plugins/techdocs-backend/package.json index bf67108457..bfa60c4b9c 100644 --- a/plugins/techdocs-backend/package.json +++ b/plugins/techdocs-backend/package.json @@ -23,19 +23,14 @@ "@backstage/backend-common": "^0.3.2", "@backstage/catalog-model": "^0.3.1", "@backstage/config": "^0.1.1", + "@backstage/techdocs-common": "^0.1.1", "@types/dockerode": "^2.5.34", "@types/express": "^4.17.6", - "command-exists-promise": "^2.0.2", "cross-fetch": "^3.0.6", "dockerode": "^3.2.1", "express": "^4.17.1", "express-promise-router": "^3.0.3", - "fs-extra": "^9.0.1", - "git-url-parse": "^11.4.0", - "js-yaml": "^3.14.0", "knex": "^0.21.6", - "mock-fs": "^4.13.0", - "nodegit": "^0.27.0", "winston": "^3.2.1" }, "devDependencies": { diff --git a/plugins/techdocs-backend/src/index.ts b/plugins/techdocs-backend/src/index.ts index 8c65708017..7612c392a2 100644 --- a/plugins/techdocs-backend/src/index.ts +++ b/plugins/techdocs-backend/src/index.ts @@ -15,4 +15,3 @@ */ export * from './service/router'; -export * from './techdocs'; diff --git a/plugins/techdocs-backend/src/service/helpers.ts b/plugins/techdocs-backend/src/service/helpers.ts index 4a736f7388..876b80781e 100644 --- a/plugins/techdocs-backend/src/service/helpers.ts +++ b/plugins/techdocs-backend/src/service/helpers.ts @@ -22,9 +22,10 @@ import { GeneratorBuilder, PreparerBase, GeneratorBase, -} from '../techdocs'; + getLocationForEntity, + getLastCommitTimestamp, +} from '@backstage/techdocs-common'; import { BuildMetadataStorage } from '../storage'; -import { getLocationForEntity, getLastCommitTimestamp } from '../helpers'; const getEntityId = (entity: Entity) => { return `${entity.kind}:${entity.metadata.namespace ?? ''}:${ diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index 61f0fa2e9f..0c5708e09f 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -20,14 +20,18 @@ import Knex from 'knex'; import fetch from 'cross-fetch'; import { Config } from '@backstage/config'; import Docker from 'dockerode'; -import { GeneratorBuilder, PreparerBuilder, PublisherBase } from '../techdocs'; +import { + GeneratorBuilder, + PreparerBuilder, + PublisherBase, + getLocationForEntity, +} from '@backstage/techdocs-common'; import { PluginEndpointDiscovery, resolvePackagePath, } from '@backstage/backend-common'; import { Entity } from '@backstage/catalog-model'; import { DocsBuilder } from './helpers'; -import { getLocationForEntity } from '../helpers'; type RouterOptions = { preparers: PreparerBuilder; diff --git a/plugins/techdocs-backend/src/service/standaloneServer.ts b/plugins/techdocs-backend/src/service/standaloneServer.ts index 89b49467d4..af3136dcda 100644 --- a/plugins/techdocs-backend/src/service/standaloneServer.ts +++ b/plugins/techdocs-backend/src/service/standaloneServer.ts @@ -28,7 +28,7 @@ import { Generators, TechdocsGenerator, Publisher, -} from '../techdocs'; +} from '@backstage/techdocs-common'; import { ConfigReader } from '@backstage/config'; export interface ServerOptions { diff --git a/plugins/techdocs/package.json b/plugins/techdocs/package.json index c8bca7d07e..6deb40fd0d 100644 --- a/plugins/techdocs/package.json +++ b/plugins/techdocs/package.json @@ -25,6 +25,7 @@ "@backstage/core": "^0.3.2", "@backstage/core-api": "^0.2.3", "@backstage/plugin-catalog": "^0.2.4", + "@backstage/techdocs-common": "^0.1.1", "@backstage/test-utils": "^0.1.3", "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", From 748160188356305f67a48b523f2190008d09d9c9 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Sun, 29 Nov 2020 09:49:04 +0100 Subject: [PATCH 003/188] techdocs-backend: Add non-zero tests for the CI to pass --- .../src/storage/BuildMetadataStorage.test.ts | 27 +++++++++++++++++++ .../src/storage/BuildMetadataStorage.ts | 5 ++++ 2 files changed, 32 insertions(+) create mode 100644 plugins/techdocs-backend/src/storage/BuildMetadataStorage.test.ts diff --git a/plugins/techdocs-backend/src/storage/BuildMetadataStorage.test.ts b/plugins/techdocs-backend/src/storage/BuildMetadataStorage.test.ts new file mode 100644 index 0000000000..87de9d1d02 --- /dev/null +++ b/plugins/techdocs-backend/src/storage/BuildMetadataStorage.test.ts @@ -0,0 +1,27 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { BuildMetadataStorage } from './BuildMetadataStorage'; + +describe('BuildMetadataStorage', () => { + it('should return build timestamp', () => { + const newMetadataStorage = new BuildMetadataStorage('123abc'); + newMetadataStorage.storeBuildTimestamp(); + + const timestamp = newMetadataStorage.getTimestamp(); + + expect(timestamp).toBeLessThanOrEqual(Date.now()); + }); +}); diff --git a/plugins/techdocs-backend/src/storage/BuildMetadataStorage.ts b/plugins/techdocs-backend/src/storage/BuildMetadataStorage.ts index b82ec1291a..d6a19764d9 100644 --- a/plugins/techdocs-backend/src/storage/BuildMetadataStorage.ts +++ b/plugins/techdocs-backend/src/storage/BuildMetadataStorage.ts @@ -20,6 +20,11 @@ type buildInfo = { const builds = {} as buildInfo; +/** + * Store timestamps of the most recent TechDocs build of each Entity. This is + * used to invalidate cache if the latest commit in the documentation source + * repository is later than the timestamp. + */ export class BuildMetadataStorage { public entityUid: string; private builds: buildInfo; From 64737f447a165d35d141f81a9e3a9d7c4e37826a Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Sun, 29 Nov 2020 10:32:44 +0100 Subject: [PATCH 004/188] App/Backend should import techdocs plugin only, not common package If a backstage App wants to enable techdocs plugin, it should only have to care about installing @backstage/plugin-techdocs and @backstage/plugin-techdocs-backend. @backstage/plugin-techdocs-backend should re-export necessary APIs from techdocs-common --- packages/backend/package.json | 1 - packages/backend/src/plugins/techdocs.ts | 5 ++--- .../default-app/packages/backend/src/plugins/techdocs.ts | 4 ++-- plugins/techdocs-backend/src/index.ts | 1 + 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 158ff043b2..beb2567bd7 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -31,7 +31,6 @@ "@backstage/plugin-scaffolder-backend": "^0.3.2", "@backstage/plugin-sentry-backend": "^0.1.3", "@backstage/plugin-techdocs-backend": "^0.3.0", - "@backstage/techdocs-common": "^0.1.1", "@gitbeaker/node": "^25.2.0", "@octokit/rest": "^18.0.0", "azure-devops-node-api": "^10.1.1", diff --git a/packages/backend/src/plugins/techdocs.ts b/packages/backend/src/plugins/techdocs.ts index 25d17801ea..346ea37bac 100644 --- a/packages/backend/src/plugins/techdocs.ts +++ b/packages/backend/src/plugins/techdocs.ts @@ -13,9 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -import { createRouter } from '@backstage/plugin-techdocs-backend'; import { + createRouter, DirectoryPreparer, Preparers, Generators, @@ -23,7 +22,7 @@ import { CommonGitPreparer, UrlPreparer, Publisher, -} from '@backstage/techdocs-common'; +} from '@backstage/plugin-techdocs-backend'; import { PluginEnvironment } from '../types'; import Docker from 'dockerode'; diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts index 3b8df8ef21..61af70d981 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts @@ -1,11 +1,11 @@ import { createRouter, DirectoryPreparer, - CommonGitPreparer, Preparers, Generators, - Publisher, TechdocsGenerator, + CommonGitPreparer, + Publisher, } from '@backstage/plugin-techdocs-backend'; import { PluginEnvironment } from '../types'; import Docker from 'dockerode'; diff --git a/plugins/techdocs-backend/src/index.ts b/plugins/techdocs-backend/src/index.ts index 7612c392a2..5c57882788 100644 --- a/plugins/techdocs-backend/src/index.ts +++ b/plugins/techdocs-backend/src/index.ts @@ -15,3 +15,4 @@ */ export * from './service/router'; +export * from '@backstage/techdocs-common'; From e09cff1682c481f9378ba144ff88fd537671fc45 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Wed, 2 Dec 2020 11:45:19 +0100 Subject: [PATCH 005/188] Abstract Publisher APIs from the router 1. Move fetchMetadata and serve static files to local publisher 2. Create factory class to create publishers based on app cofig --- packages/backend/src/plugins/techdocs.ts | 2 +- .../packages/backend/src/plugins/techdocs.ts | 2 +- packages/techdocs-common/package.json | 4 +- .../src/stages/publish/index.ts | 3 +- .../src/stages/publish/local.test.ts | 14 ++- .../src/stages/publish/local.ts | 83 +++++++++++----- .../src/stages/publish/publish.ts | 98 ++++--------------- .../test-component-folder-AP2Wti/mock-file | 0 .../src/stages/publish/types.ts | 42 +++++++- .../techdocs-backend/src/service/helpers.ts | 17 +++- .../techdocs-backend/src/service/router.ts | 42 +++----- .../src/service/standaloneServer.ts | 2 +- yarn.lock | 9 ++ 13 files changed, 173 insertions(+), 145 deletions(-) delete mode 100644 packages/techdocs-common/src/stages/publish/test-component-folder-AP2Wti/mock-file diff --git a/packages/backend/src/plugins/techdocs.ts b/packages/backend/src/plugins/techdocs.ts index 346ea37bac..4720b2568e 100644 --- a/packages/backend/src/plugins/techdocs.ts +++ b/packages/backend/src/plugins/techdocs.ts @@ -49,7 +49,7 @@ export default async function createPlugin({ const urlPreparer = new UrlPreparer(reader, logger); preparers.register('url', urlPreparer); - const publisher = new Publisher(logger, config, discovery); + const publisher = Publisher.fromConfig(config, logger, discovery); const dockerClient = new Docker(); diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts index 61af70d981..1bbb5ff24b 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/techdocs.ts @@ -28,7 +28,7 @@ export default async function createPlugin({ preparers.register('github', commonGitPreparer); preparers.register('gitlab', commonGitPreparer); - const publisher = new Publisher(logger, config, discovery); + const publisher = Publisher.fromConfig(config, logger, discovery); const dockerClient = new Docker(); diff --git a/packages/techdocs-common/package.json b/packages/techdocs-common/package.json index 6edb1fc589..cd5e593ad1 100644 --- a/packages/techdocs-common/package.json +++ b/packages/techdocs-common/package.json @@ -39,14 +39,16 @@ "@backstage/backend-common": "^0.3.2", "@backstage/catalog-model": "^0.3.1", "@backstage/config": "^0.1.1", + "@types/dockerode": "^2.5.34", + "@types/express-serve-static-core": "^4.17.14", "cross-fetch": "^3.0.6", "dockerode": "^3.2.1", + "express": "^4.17.1", "fs-extra": "^9.0.1", "git-url-parse": "^11.4.0", "js-yaml": "^3.14.0", "mock-fs": "^4.13.0", "nodegit": "^0.27.0", - "@types/dockerode": "^2.5.34", "winston": "^3.2.1" }, "devDependencies": { diff --git a/packages/techdocs-common/src/stages/publish/index.ts b/packages/techdocs-common/src/stages/publish/index.ts index 9a21217083..494efe5ac1 100644 --- a/packages/techdocs-common/src/stages/publish/index.ts +++ b/packages/techdocs-common/src/stages/publish/index.ts @@ -14,5 +14,4 @@ * limitations under the License. */ export { Publisher } from './publish'; -export type { PublisherBase } from './publish'; -export type { PublisherType } from './types'; +export type { PublisherBase, PublisherType } from './types'; diff --git a/packages/techdocs-common/src/stages/publish/local.test.ts b/packages/techdocs-common/src/stages/publish/local.test.ts index 793c2374d9..a790284c3f 100644 --- a/packages/techdocs-common/src/stages/publish/local.test.ts +++ b/packages/techdocs-common/src/stages/publish/local.test.ts @@ -21,6 +21,7 @@ import { getVoidLogger, PluginEndpointDiscovery, } from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; import { LocalPublish } from './local'; const createMockEntity = (annotations = {}) => { @@ -45,7 +46,18 @@ describe('local publisher', () => { getExternalBaseUrl: jest.fn(), }; - const publisher = new LocalPublish(logger, testDiscovery); + const mockConfig = ConfigReader.fromConfigs([ + { + context: '', + data: { + techdocs: { + requestUrl: 'http://localhost:7000', + }, + }, + }, + ]); + + const publisher = new LocalPublish(mockConfig, logger, testDiscovery); const mockEntity = createMockEntity(); diff --git a/packages/techdocs-common/src/stages/publish/local.ts b/packages/techdocs-common/src/stages/publish/local.ts index 712bf8266d..c1be8641ef 100644 --- a/packages/techdocs-common/src/stages/publish/local.ts +++ b/packages/techdocs-common/src/stages/publish/local.ts @@ -13,50 +13,47 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import express from 'express'; +import * as expressServeStaticCore from 'express-serve-static-core'; // Type from express library +import fetch from 'cross-fetch'; import fs from 'fs-extra'; import { Logger } from 'winston'; -import { Entity } from '@backstage/catalog-model'; +import { EntityName } from '@backstage/catalog-model'; import { resolvePackagePath, PluginEndpointDiscovery, } from '@backstage/backend-common'; +import { Config } from '@backstage/config'; +import { PublisherBase, PublisherBaseParams } from './types'; -export type LocalPublishParams = { - entity: Entity; - directory: string; -}; +/* `remoteUrl` is the URL which serves files from the local publisher's static directory. */ +export type LocalPublishReturn = Promise<{ remoteUrl: string }>; -export type LocalPublishReturn = - | Promise<{ remoteUrl: string }> - | { remoteUrl: string }; +const staticDocsDir = resolvePackagePath( + '@backstage/plugin-techdocs-backend', + 'static/docs', +); /** - * Type for the local publisher which uses local filesystem to store the generated static files. - * - * It uses a directory called "static" at the root of techdocs-backend plugin. + * Local publisher which uses the local filesystem to store the generated static files. It uses a directory + * called "static" at the root of techdocs-backend plugin. */ -export interface LocalPublisher { - /** - * Store the generated files inside a static folder in local filesystem. - * - * @param {LocalPublishParams} opts Object containing the entity from the service - * catalog, and the directory that contains the generated static files from TechDocs. - * @returns {LocalPublishReturn} Either a promise or an object with `remoteUrl` which is the URL - * which serves files from the local publisher's static directory. - */ - publish(opts: LocalPublishParams): LocalPublishReturn; -} - -export class LocalPublish { +export class LocalPublish implements PublisherBase { + private readonly config: Config; private readonly logger: Logger; private readonly discovery: PluginEndpointDiscovery; - constructor(logger: Logger, discovery: PluginEndpointDiscovery) { + constructor( + config: Config, + logger: Logger, + discovery: PluginEndpointDiscovery, + ) { + this.config = config; this.logger = logger; this.discovery = discovery; } - publish({ entity, directory }: LocalPublishParams): LocalPublishReturn { + publish({ entity, directory }: PublisherBaseParams): LocalPublishReturn { const entityNamespace = entity.metadata.namespace ?? 'default'; const publishDir = resolvePackagePath( @@ -94,4 +91,38 @@ export class LocalPublish { }); }); } + + fetchTechDocsMetadata( + entityName: EntityName, + ): Promise<{ techdocsMetadataJson: string }> { + return new Promise((resolve, reject) => { + this.discovery.getBaseUrl('techdocs').then(techdocsApiUrl => { + const storageUrl = new URL( + new URL(this.config.getString('techdocs.storageUrl')).pathname, + techdocsApiUrl, + ).toString(); + + const path = `${entityName.kind}/${entityName.namespace}/${entityName.name}`; + const metadataURL = `${storageUrl}/${path}/techdocs_metadata.json`; + fetch(metadataURL) + .then(response => + response + .json() + .then(techdocsMetadataJson => resolve(techdocsMetadataJson)) + .catch(err => { + reject( + `Unable to parse metadata JSON for ${path}. Error: ${err}`, + ); + }), + ) + .catch(err => { + reject(`Unable to fetch metadata for ${path}. Error ${err}`); + }); + }); + }); + } + + docsRouter(): expressServeStaticCore.Handler { + return express.static(staticDocsDir); + } } diff --git a/packages/techdocs-common/src/stages/publish/publish.ts b/packages/techdocs-common/src/stages/publish/publish.ts index bf01812fa2..ca3cb99498 100644 --- a/packages/techdocs-common/src/stages/publish/publish.ts +++ b/packages/techdocs-common/src/stages/publish/publish.ts @@ -16,96 +16,34 @@ import { Logger } from 'winston'; import { Config } from '@backstage/config'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; -import { Entity } from '@backstage/catalog-model'; -import { PublisherType } from './types'; + +import { PublisherType, PublisherBase } from './types'; import { LocalPublish } from './local'; -export type PublisherBaseParams = { - entity: Entity; - directory: string; -}; - -export type PublisherBaseReturn = - | Promise<{ remoteUrl: string }> - | { remoteUrl: string }; - /** - * Type for the publisher instance registered with backend which manages publishing of the - * generated static files after the prepare and generate steps of TechDocs. - * - * Depending upon the value of techdocs.publisher.type in app config, this instance creates - * and uses a publisher of the particular type i.e. local, google_gcs, aws_s3, etc. It defaults - * to use the local publisher. + * Factory class to create a TechDocs publisher based on defined publisher type in app config. + * Uses `techdocs.publisher.type`. */ -export interface PublisherBase { - /** - * Invoke the app's configured publisher's publish method. - * - * @param {PublisherBaseParams} opts Object containing the entity from the service - * catalog, and the directory that contains the generated static files from TechDocs. - */ - publish(opts: PublisherBaseParams): PublisherBaseReturn; - - /** - * Return true if local filesystem is being used to store generated files for TechDocs. - */ - isLocalPublisher(): boolean; - - /** - * Return true if an external cloud storage (GCS, S3, SFTP server, etc.) is being used to - * store generated files for TechDocs. - */ - isExternalPublisher(): boolean; -} - -export class Publisher implements PublisherBase { - private readonly logger: Logger; - private readonly config: Config; - private readonly discovery: PluginEndpointDiscovery; - private readonly publisherType: PublisherType; - private readonly publisher: any; - - constructor( - logger: Logger, +export class Publisher { + static fromConfig( config: Config, + logger: Logger, discovery: PluginEndpointDiscovery, - ) { - this.logger = logger; - this.config = config; - this.discovery = discovery; + ): PublisherBase { + const publisherType = (config.getOptionalString( + 'techdocs.publisher.type', + ) ?? 'local') as PublisherType; - this.publisherType = - (this.config.getOptionalString( - 'techdocs.publisher.type', - ) as PublisherType) ?? 'local'; - - switch (this.publisherType) { + switch (publisherType) { case 'google_gcs': - this.logger.info( - 'Creating Google Storage Bucket publisher for TechDocs', - ); - this.publisher = new LocalPublish(this.logger, this.discovery); - break; + logger.info('Creating Google Storage Bucket publisher for TechDocs'); + return new LocalPublish(config, logger, discovery); case 'local': - this.logger.info('Creating Local publisher for TechDocs'); - this.publisher = new LocalPublish(this.logger, this.discovery); - break; + logger.info('Creating Local publisher for TechDocs'); + return new LocalPublish(config, logger, discovery); default: - this.logger.info('Creating Local publisher for TechDocs'); - this.publisher = new LocalPublish(this.logger, this.discovery); - break; + logger.info('Creating Local publisher for TechDocs'); + return new LocalPublish(config, logger, discovery); } } - - publish({ entity, directory }: PublisherBaseParams): PublisherBaseReturn { - return this.publisher.publish({ entity, directory }); - } - - isLocalPublisher(): boolean { - return this.publisherType === 'local'; - } - - isExternalPublisher(): boolean { - return this.publisherType !== 'local'; - } } diff --git a/packages/techdocs-common/src/stages/publish/test-component-folder-AP2Wti/mock-file b/packages/techdocs-common/src/stages/publish/test-component-folder-AP2Wti/mock-file deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/techdocs-common/src/stages/publish/types.ts b/packages/techdocs-common/src/stages/publish/types.ts index 2c2348e37a..56f57c11c3 100644 --- a/packages/techdocs-common/src/stages/publish/types.ts +++ b/packages/techdocs-common/src/stages/publish/types.ts @@ -13,10 +13,48 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { Entity, EntityName } from '@backstage/catalog-model'; +// import serveStatic from 'serve-static'; +// import express from 'express'; +import * as express from 'express-serve-static-core'; /** - * Key for all the different types of TechDocs publishers available. + * Key for all the different types of TechDocs publishers that are supported. */ export type PublisherType = 'local' | 'google_gcs'; -export interface ExternalPublisher {} +export type PublisherBaseParams = { + entity: Entity; + /* The Path to the directory where the generated files are stored. */ + directory: string; +}; + +export type PublisherBaseReturn = Promise<{}>; + +/** + * Base class for a TechDocs publisher (e.g. Local, Google GCS Bucket, AWS S3, etc.) + * The publisher handles publishing of the generated static files after the prepare and generate steps of TechDocs. + * It also provides APIs to communicate with the storage service. + */ +export interface PublisherBase { + /** + * Store the generated static files onto a storage service (either local filesystem or external service). + * + * @param options Object containing the entity from the service + * catalog, and the directory that contains the generated static files from TechDocs. + */ + publish(options: PublisherBaseParams): PublisherBaseReturn; + + /** + * Retrieve TechDocs Metadata about a site e.g. name, contributors, last updated, etc. + * This API uses the techdocs_metadata.json file that co-exists along with the generated docs. + */ + fetchTechDocsMetadata( + entityName: EntityName, + ): Promise<{ techdocsMetadataJson: string }>; + + /** + * + */ + docsRouter(): express.Handler; +} diff --git a/plugins/techdocs-backend/src/service/helpers.ts b/plugins/techdocs-backend/src/service/helpers.ts index 876b80781e..51e7ed8ad2 100644 --- a/plugins/techdocs-backend/src/service/helpers.ts +++ b/plugins/techdocs-backend/src/service/helpers.ts @@ -15,7 +15,7 @@ */ import Docker from 'dockerode'; import { Logger } from 'winston'; -import { Entity } from '@backstage/catalog-model'; +import { Entity, EntityName } from '@backstage/catalog-model'; import { PreparerBuilder, PublisherBase, @@ -127,3 +127,18 @@ export class DocsBuilder { return false; } } + +/** + * Using the path of the TechDocs page URL, return a structured EntityName type object with namespace, + * kind and name of the Entity. + * @param {string} path Example: default/Component/documented-component + */ +export const getEntityNameFromUrlPath = (path: string): EntityName => { + const [kind, namespace, name] = path.split('/'); + + return { + kind, + namespace, + name, + }; +}; diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index 0c5708e09f..dd084bde4b 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -26,12 +26,9 @@ import { PublisherBase, getLocationForEntity, } from '@backstage/techdocs-common'; -import { - PluginEndpointDiscovery, - resolvePackagePath, -} from '@backstage/backend-common'; +import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { Entity } from '@backstage/catalog-model'; -import { DocsBuilder } from './helpers'; +import { DocsBuilder, getEntityNameFromUrlPath } from './helpers'; type RouterOptions = { preparers: PreparerBuilder; @@ -44,11 +41,6 @@ type RouterOptions = { dockerClient: Docker; }; -const staticDocsDir = resolvePackagePath( - '@backstage/plugin-techdocs-backend', - 'static/docs', -); - export async function createRouter({ preparers, generators, @@ -61,24 +53,18 @@ export async function createRouter({ const router = Router(); router.get('/metadata/techdocs/*', async (req, res) => { - let storageUrl = config.getString('techdocs.storageUrl'); - if (publisher.isLocalPublisher()) { - storageUrl = new URL( - new URL(storageUrl).pathname, - await discovery.getBaseUrl('techdocs'), - ).toString(); - } + // path is `:namespace/:kind:/:name` const { '0': path } = req.params; + const entityName = getEntityNameFromUrlPath(path); - const metadataURL = `${storageUrl}/${path}/techdocs_metadata.json`; - - try { - const techdocsMetadata = await (await fetch(metadataURL)).json(); - res.send(techdocsMetadata); - } catch (err) { - logger.info(`Unable to get metadata for ${path} with error ${err}`); - throw new Error(`Unable to get metadata for ${path} with error ${err}`); - } + publisher + .fetchTechDocsMetadata(entityName) + .then(techdocsMetadataJson => { + res.send(techdocsMetadataJson); + }) + .catch(reason => { + res.status(500).send(`Unable to get Metadata. Reason: ${reason}`); + }); }); router.get('/metadata/entity/:namespace/:kind/:name', async (req, res) => { @@ -139,9 +125,7 @@ export async function createRouter({ res.redirect(`${storageUrl}${req.path.replace('/docs', '')}`); }); - if (publisher.isLocalPublisher()) { - router.use('/static/docs', express.static(staticDocsDir)); - } + router.use('/static/docs', publisher.docsRouter()); return router; } diff --git a/plugins/techdocs-backend/src/service/standaloneServer.ts b/plugins/techdocs-backend/src/service/standaloneServer.ts index af3136dcda..c1c4dba8d1 100644 --- a/plugins/techdocs-backend/src/service/standaloneServer.ts +++ b/plugins/techdocs-backend/src/service/standaloneServer.ts @@ -64,7 +64,7 @@ export async function startStandaloneServer( const techdocsGenerator = new TechdocsGenerator(logger, config); generators.register('techdocs', techdocsGenerator); - const publisher = new Publisher(logger, config, discovery); + const publisher = Publisher.fromConfig(config, logger, discovery); const dockerClient = new Docker(); diff --git a/yarn.lock b/yarn.lock index 6f27d7028a..e031375fc0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5208,6 +5208,15 @@ "@types/qs" "*" "@types/range-parser" "*" +"@types/express-serve-static-core@^4.17.14": + version "4.17.14" + resolved "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.14.tgz#cabf91debeeb3cb04b798e2cff908864e89b6106" + integrity sha512-uFTLwu94TfUFMToXNgRZikwPuZdOtDgs3syBtAIr/OXorL1kJqUJT9qCLnRZ5KBOWfZQikQ2xKgR2tnDj1OgDA== + dependencies: + "@types/node" "*" + "@types/qs" "*" + "@types/range-parser" "*" + "@types/express-session@^1.17.2": version "1.17.2" resolved "https://registry.npmjs.org/@types/express-session/-/express-session-1.17.2.tgz#ed6a36dd9f267c7fef86004f653bfb9b5cea3c21" From 084670b37afb133d78453473c7e9bb7ed1c46826 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Thu, 3 Dec 2020 22:40:15 +0100 Subject: [PATCH 006/188] Google Cloud Storage working with TechDocs (publish and fetch) Plus 1. Introduce ncessary configs to connect with storage 2. Introduce config to prefer CI build of docs, or techdocs-backend to build and publish them. 3. Write guide how to use Google Cloud Storage with TechDocs 4. Add a TechDocs Configuration reference page in docucmentation --- app-config.yaml | 11 +- docs/features/techdocs/configuration.md | 71 ++++ docs/features/techdocs/using-cloud-storage.md | 80 +++++ microsite/sidebars.json | 2 + mkdocs.yml | 2 + packages/techdocs-common/package.json | 4 +- .../src/stages/publish/googleStorage.ts | 222 +++++++++++++ .../src/stages/publish/helpers.ts | 46 +++ .../src/stages/publish/local.ts | 38 ++- .../src/stages/publish/publish.ts | 3 +- .../src/stages/publish/types.ts | 15 +- .../BuildMetadataStorage.test.ts | 0 .../BuildMetadataStorage.ts | 0 .../src/DocsBuilder/builder.ts | 129 ++++++++ .../src/{storage => DocsBuilder}/index.ts | 1 + plugins/techdocs-backend/src/index.ts | 1 + .../src/service/helpers.test.ts | 26 ++ .../techdocs-backend/src/service/helpers.ts | 120 +------ .../techdocs-backend/src/service/router.ts | 59 +++- yarn.lock | 304 +++++++++++++++++- 20 files changed, 971 insertions(+), 163 deletions(-) create mode 100644 docs/features/techdocs/configuration.md create mode 100644 docs/features/techdocs/using-cloud-storage.md create mode 100644 packages/techdocs-common/src/stages/publish/googleStorage.ts create mode 100644 packages/techdocs-common/src/stages/publish/helpers.ts rename plugins/techdocs-backend/src/{storage => DocsBuilder}/BuildMetadataStorage.test.ts (100%) rename plugins/techdocs-backend/src/{storage => DocsBuilder}/BuildMetadataStorage.ts (100%) create mode 100644 plugins/techdocs-backend/src/DocsBuilder/builder.ts rename plugins/techdocs-backend/src/{storage => DocsBuilder}/index.ts (95%) create mode 100644 plugins/techdocs-backend/src/service/helpers.test.ts diff --git a/app-config.yaml b/app-config.yaml index f381f71ead..bc01e5eab5 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -55,16 +55,15 @@ proxy: organization: name: My Company +# Reference documentation http://backstage.io/docs/features/techdocs/configuration techdocs: - storageUrl: http://localhost:7000/api/techdocs/static/docs requestUrl: http://localhost:7000/api/techdocs + storageUrl: http://localhost:7000/api/techdocs/static/docs + docsBuilder: 'local' # Alternatives - 'ci' generators: - techdocs: 'docker' + techdocs: 'docker' # Alternatives - 'local' publisher: - # Ref: - type: 'local' # other options - google_gcs, aws_s3, azure_storage, etc. - # if type set to google_gcs - # google_token: 'abc' # with write access + type: 'local' # Alternatives - 'google_gcs'. Read documentation for using alternatives. sentry: organization: my-company diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md new file mode 100644 index 0000000000..8b0f47c134 --- /dev/null +++ b/docs/features/techdocs/configuration.md @@ -0,0 +1,71 @@ +--- +id: configuration +title: TechDocs Configuration Options +description: Reference documentation for configuring TechDocs using app-config.yaml +--- + +Using the `app-config.yaml` in the Backstage app, you can configure TechDocs using several options. +This page serves as a reference to all the available configuration options for TechDocs. + + +```yaml +# File: app-config.yaml + +techdocs: + + # TechDocs makes API calls to techdocs-backend using this URL. e.g. get docs of an entity, get metadata, etc. + + requestUrl: http://localhost:7000/api/techdocs + + + # Just another route in techdocs-backend where TechDocs requests the static files from. This URL uses an HTTP middleware + # to serve files from either a local directory or an External storage provider. + + storageUrl: http://localhost:7000/api/techdocs/static/docs + + + # generators.techdocs can have two values: 'docker' or 'local'. This is to determine how to run the generator - whether to + # spin up the techdocs-container docker image or to run mkdocs locally (assuming all the dependencies are taken care of). + # You want to change this to 'local' if you are running Backstage using your own customer Docker setup and want to avoid running + # into Docker in Docker situation. Read more here + # https://backstage.io/docs/features/techdocs/getting-started#disable-docker-in-docker-situation-optional + + generators: + techdocs: 'docker' + + + # techdocs.builder can be either 'local' or 'ci. + # If builder is set to 'local' and you open a TechDocs page, techdocs-backend will try to build the docs, publish to storage + # and show the generated docs afterwords. This is the "Basic" setup of the TechDocs Architecture. + # If builder is set to 'ci', techdocs-backend will only fetch the docs and will NOT try to build and publish. In this case of 'ci', + # we assume that docs are being built in the CI/CD pipeline of the repository. This is the "Recommended" setup of the architecture. + # Read more here https://backstage.io/docs/features/techdocs/architecture + + builder: 'local' + + + # techdocs.publisher is used to configure the Storage option, whether you want to use the local filesystem to store generated docs + # or you want to use External storage providers like Google Cloud Storage, AWS S3, etc. + + publisher: + + # techdocs.publisher.type can be - 'local' or 'google_gcs' (aws_s3, azure_storage, etc. to be available as well). + # When set to 'local', techdocs-backend will create a 'static' directory at its root to store generated documentation files. + # When set to 'google_gcs', techdocs-backend will use a Google Cloud Storage Bucket to store generated documentation files. + + type: 'local' + + + # Required when techdocs.publisher.type is set to 'google_gcs'. Skip otherwise. + + google: + # An API key is required to write to a storage bucket. + pathToKey: '/path/to/google_application_credentials.json', + + # Your GCP Project ID where the Cloud Storage Bucket is hosted. + projectId: 'gcp-project-id' + + # Cloud Storage Bucket Name + bucketName: 'techdocs-storage', + +``` diff --git a/docs/features/techdocs/using-cloud-storage.md b/docs/features/techdocs/using-cloud-storage.md new file mode 100644 index 0000000000..e435644ca2 --- /dev/null +++ b/docs/features/techdocs/using-cloud-storage.md @@ -0,0 +1,80 @@ +--- +id: using-cloud-storage +title: Using Cloud Storage for TechDocs generated files +description: Using Cloud Storage for TechDocs generated files +--- + +In the [TechDocs architecture](./architecture.md) you have the option to choose where you want to store +the Generated static files which TechDocs uses to render documentation. In both the "Basic" and "Recommended" +setup, you can add cloud storage providers like Google GCS, Amazon AWS S3, etc. By default, TechDocs +uses the local filesystem of the `techdocs-backend` plugin in the "Basic" setup. And in the recommended setup, +having one of the cloud storage is a prerequisite. Read more on the TechDocs Architecture documentation page. + +On this page you you can read how to enable them. + +## Configuring Google GCS Bucket with TechDocs + +Follow the [official Google Cloud documentation](https://googleapis.dev/nodejs/storage/latest/index.html#quickstart) for the latest instructions +on the following steps involving GCP. + +**1. Set `techdocs.publisher.type` config in your `app-config.yaml`** + +Set `techdocs.publisher.type` to `'google_gcs'`. + +```yaml +techdocs: + publisher: + type: 'google_gcs' +``` + +**2. GCP (Google Cloud Platform) Project** + +Create or choose a dedicated GCP project. Set `techdocs.publisher.google.projectId` to the project ID. + +```yaml +techdocs: + publisher: + type: 'google_gcs' + google: + projectId: 'gcp-project-id +``` + +**3. Service account API key** + +Create a new Service Account and a key associated with it. In roles of the service account, use "Storage Admin". + +If you want to create a custom role, make sure to include both `get` and `create` permissions for both "Objects" and "Buckets". See https://cloud.google.com/storage/docs/access-control/iam-permissions + +A service account can have many keys. Open your newly created account's page (in IAM & Admin console), and create a new key. Use JSON format for the key. + +A `.json` file will be downloaded. This is the secret key TechDocs will use to make API calls. Make it available in your Backstage server and/or your local development server and set it in the app config `techdocs.publisher.google.pathToKey`. + +```yaml +techdocs: + publisher: + type: 'google_gcs' + google: + projectId: 'gcp-project-id' + pathToKey: '/path/to/google_application_credentials.json' +``` + +**4. GCS Bucket** + +Create a dedicated bucket for TechDocs sites. techdocs-backend will publish documentation to this bucket. +TechDocs will fetch files from here to serve documentation in Backstage. + +Set the name of the bucket to `techdocs.publisher + +```yaml +techdocs: + publisher: + type: 'google_gcs' + google: + projectId: 'gcp-project-id' + pathToKey: '/path/to/google_application_credentials.json' + bucketName: 'name-of-techdocs-storage-bucket' +``` + +**5. That's it!** + +Your Backstage app is now ready to use Google Cloud Storage for TechDocs, to store the static generated documentation files. diff --git a/microsite/sidebars.json b/microsite/sidebars.json index e60d9c520e..91112dd3b3 100644 --- a/microsite/sidebars.json +++ b/microsite/sidebars.json @@ -74,6 +74,8 @@ "features/techdocs/concepts", "features/techdocs/architecture", "features/techdocs/creating-and-publishing", + "features/techdocs/configuration", + "features/techdocs/using-cloud-storage", "features/techdocs/troubleshooting", "features/techdocs/faqs" ] diff --git a/mkdocs.yml b/mkdocs.yml index 1086259710..70123b384d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -54,6 +54,8 @@ nav: - Concepts: 'features/techdocs/concepts.md' - TechDocs Architecture: 'features/techdocs/architecture.md' - Creating and Publishing Documentation: 'features/techdocs/creating-and-publishing.md' + - Configuration: 'features/techdocs/configuration.md' + - Using Cloud Storage: 'features/techdocs/using-cloud-storage.md' - Troubleshooting: 'features/techdocs/troubleshooting.md' - FAQ: 'features/techdocs/FAQ.md' - Plugins: diff --git a/packages/techdocs-common/package.json b/packages/techdocs-common/package.json index cd5e593ad1..66821a0043 100644 --- a/packages/techdocs-common/package.json +++ b/packages/techdocs-common/package.json @@ -39,14 +39,16 @@ "@backstage/backend-common": "^0.3.2", "@backstage/catalog-model": "^0.3.1", "@backstage/config": "^0.1.1", + "@google-cloud/storage": "^5.6.0", "@types/dockerode": "^2.5.34", - "@types/express-serve-static-core": "^4.17.14", + "@types/klaw": "^3.0.1", "cross-fetch": "^3.0.6", "dockerode": "^3.2.1", "express": "^4.17.1", "fs-extra": "^9.0.1", "git-url-parse": "^11.4.0", "js-yaml": "^3.14.0", + "klaw": "^3.0.0", "mock-fs": "^4.13.0", "nodegit": "^0.27.0", "winston": "^3.2.1" diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts new file mode 100644 index 0000000000..7023f7c1eb --- /dev/null +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -0,0 +1,222 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import path from 'path'; +import express from 'express'; +import walk from 'klaw'; +import { Storage, UploadResponse } from '@google-cloud/storage'; +import { Logger } from 'winston'; +import { Entity, EntityName } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { getHeadersForFileExtension, supportedFileType } from './helpers'; +import { PublisherBase, PublisherBaseParams } from './types'; + +export class GoogleGCSPublish implements PublisherBase { + static fromConfig(config: Config, logger: Logger): PublisherBase { + let pathToKey = ''; + let projectId = ''; + let bucketName = ''; + try { + pathToKey = config.getString('techdocs.publisher.google.pathToKey'); + projectId = config.getString('techdocs.publisher.google.projectId'); + bucketName = config.getString('techdocs.publisher.google.bucketName'); + } catch (error) { + throw new Error( + "Since techdocs.publisher.type is set to 'google_gcs' in your app config, " + + 'pathToKey, projectId and bucketName are required in techdocs.publisher.google ' + + 'required to authenticate with Google Cloud Storage.', + ); + } + + const storageClient = new Storage({ + projectId: projectId, + keyFilename: pathToKey, + }); + + // Check if the defined bucket exists. Being able to connect means the configuration is good + // and the storage client will work. + storageClient + .bucket(bucketName) + .getMetadata() + .then(() => { + logger.info( + `Successfully connected to the GCS bucket ${bucketName} in the GCP project ${projectId}.`, + ); + }) + .catch(reason => { + logger.error( + `Could not retrieve metadata about the GCS bucket ${bucketName} in the GCP project ${projectId}. ` + + 'Make sure the GCP project and the bucket exists and the access key located at the path ' + + "techdocs.publisher.google.pathToKey defined in app config has the role 'Storage Object Creator'. " + + 'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage', + ); + throw new Error(`from GCS client library: ${reason.message}`); + }); + + return new GoogleGCSPublish(storageClient, bucketName, logger); + } + + constructor( + private readonly storageClient: Storage, + private readonly bucketName: string, + private readonly logger: Logger, + ) { + this.storageClient = storageClient; + this.bucketName = bucketName; + this.logger = logger; + } + + /** + * Upload all the files from the generated `directory` to the GCS bucket. + * Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html + */ + publish({ entity, directory }: PublisherBaseParams): Promise<{}> { + return new Promise((resolve, reject) => { + // Path of all files to upload, relative to the root of the source directory + // e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png'] + const allFilesToUpload: Array = []; + + // Iterate on all the files in the directory and its sub-directories + walk(directory) + .on('data', (item: walk.Item) => { + // GCS manages creation of parent directories if they do not exist. + // So collecting path of only the files is good enough. + if (item.stats.isFile()) { + // Remove the absolute path prefix of the source directory + const relativeFilePath = item.path.replace(`${directory}/`, ''); + allFilesToUpload.push(relativeFilePath); + } + }) + .on('error', (err: Error, item: walk.Item) => { + const errorMessage = `Unable to read file at ${item.path}. Error ${err.message}`; + this.logger.error(errorMessage); + reject(errorMessage); + }) + .on('end', () => { + // 'end' event happens when all the files have been read. + const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + allFilesToUpload.forEach(filePath => { + const source = path.join(directory, filePath); // Local file absolutely path + const destination = `${entityRootDir}/${filePath}`; // GCS Bucket file relative path + this.storageClient + .bucket(this.bucketName) + .upload(source, { destination }) + .then( + (uploadResp: UploadResponse) => ({ + fileName: destination, + status: uploadResp[0], + }), + (err: Error) => { + const errorMessage = `Unable to upload file ${destination} to GCS. Error ${err.message}`; + this.logger.error(errorMessage); + reject(errorMessage); + }, + ); + }); + + this.logger.info( + `Successfully uploaded all the generated files for Entity ${entityRootDir}. Total number of files: ${allFilesToUpload.length}`, + ); + resolve({}); + }); + }); + } + + fetchTechDocsMetadata(entityName: EntityName): Promise { + return new Promise((resolve, reject) => { + const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; + + const fileStreamChunks: Array = []; + this.storageClient + .bucket(this.bucketName) + .file(`${entityRootDir}/techdocs_metadata.json`) + .createReadStream() + .on('error', err => { + this.logger.error(err.message); + reject(err.message); + }) + .on('data', chunk => { + fileStreamChunks.push(chunk); + }) + .on('end', () => { + const techdocsMetadataJson = Buffer.concat( + fileStreamChunks, + ).toString(); + resolve(techdocsMetadataJson); + }); + }); + } + + /** + * Express route middleware to serve static files on a route in techdocs-backend. + */ + docsRouter(): express.Handler { + return (req, res) => { + // Trim the leading forward slash + // filePath example - /default/Component/documented-component/index.html + const filePath = req.path.replace(/^\//, ''); + + // Files with different extensions (CSS, HTML) need to be served with different headers + const fileExtension = filePath.split('.')[filePath.split('.').length - 1]; + const responseHeaders = getHeadersForFileExtension( + fileExtension as supportedFileType, + ); + + const fileStreamChunks: Array = []; + this.storageClient + .bucket(this.bucketName) + .file(filePath) + .createReadStream() + .on('error', err => { + this.logger.error(err.message); + res.send(err.message); + }) + .on('data', chunk => { + fileStreamChunks.push(chunk); + }) + .on('end', () => { + const fileContent = Buffer.concat(fileStreamChunks).toString(); + // Inject response headers + for (const [headerKey, headerValue] of Object.entries( + responseHeaders, + )) { + res.setHeader(headerKey, headerValue); + } + + res.send(fileContent); + }); + }; + } + + /** + * A helper function which checks if index.html of an Entity's docs site is available. This + * can be used to verify if there are any pre-generated docs available to serve. + */ + async hasDocsBeenGenerated(entity: Entity): Promise { + return new Promise(resolve => { + const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + this.storageClient + .bucket(this.bucketName) + .file(`${entityRootDir}/index.html`) + .createReadStream() + .on('error', () => { + resolve(false); + }) + .on('data', () => { + resolve(true); + }); + }); + } +} diff --git a/packages/techdocs-common/src/stages/publish/helpers.ts b/packages/techdocs-common/src/stages/publish/helpers.ts new file mode 100644 index 0000000000..cca5d85150 --- /dev/null +++ b/packages/techdocs-common/src/stages/publish/helpers.ts @@ -0,0 +1,46 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export type supportedFileType = 'html' | 'css'; + +export type responseHeadersType = { + 'Content-Type': string; +}; + +export const getHeadersForFileExtension = ( + fileType: supportedFileType, +): responseHeadersType => { + const headersCommon = { + 'Content-Type': 'text/plain', + }; + const headersHTML = { + ...headersCommon, + 'Content-Type': 'text/html; charset=UTF-8', + }; + + const headersCSS = { + ...headersCommon, + 'Content-Type': 'text/css; charset=UTF-8', + }; + + switch (fileType) { + case 'html': + return headersHTML; + case 'css': + return headersCSS; + default: + return headersCommon; + } +}; diff --git a/packages/techdocs-common/src/stages/publish/local.ts b/packages/techdocs-common/src/stages/publish/local.ts index c1be8641ef..0a75ab6914 100644 --- a/packages/techdocs-common/src/stages/publish/local.ts +++ b/packages/techdocs-common/src/stages/publish/local.ts @@ -13,12 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import express from 'express'; -import * as expressServeStaticCore from 'express-serve-static-core'; // Type from express library import fetch from 'cross-fetch'; +import express from 'express'; import fs from 'fs-extra'; import { Logger } from 'winston'; -import { EntityName } from '@backstage/catalog-model'; +import { Entity, EntityName } from '@backstage/catalog-model'; import { resolvePackagePath, PluginEndpointDiscovery, @@ -92,9 +91,7 @@ export class LocalPublish implements PublisherBase { }); } - fetchTechDocsMetadata( - entityName: EntityName, - ): Promise<{ techdocsMetadataJson: string }> { + fetchTechDocsMetadata(entityName: EntityName): Promise { return new Promise((resolve, reject) => { this.discovery.getBaseUrl('techdocs').then(techdocsApiUrl => { const storageUrl = new URL( @@ -102,8 +99,8 @@ export class LocalPublish implements PublisherBase { techdocsApiUrl, ).toString(); - const path = `${entityName.kind}/${entityName.namespace}/${entityName.name}`; - const metadataURL = `${storageUrl}/${path}/techdocs_metadata.json`; + const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; + const metadataURL = `${storageUrl}/${entityRootDir}/techdocs_metadata.json`; fetch(metadataURL) .then(response => response @@ -111,18 +108,37 @@ export class LocalPublish implements PublisherBase { .then(techdocsMetadataJson => resolve(techdocsMetadataJson)) .catch(err => { reject( - `Unable to parse metadata JSON for ${path}. Error: ${err}`, + `Unable to parse metadata JSON for ${entityRootDir}. Error: ${err}`, ); }), ) .catch(err => { - reject(`Unable to fetch metadata for ${path}. Error ${err}`); + reject( + `Unable to fetch metadata for ${entityRootDir}. Error ${err}`, + ); }); }); }); } - docsRouter(): expressServeStaticCore.Handler { + docsRouter(): express.Handler { return express.static(staticDocsDir); } + + async hasDocsBeenGenerated(entity: Entity): Promise { + return new Promise(resolve => { + this.discovery.getBaseUrl('techdocs').then(techdocsApiUrl => { + const storageUrl = new URL( + new URL(this.config.getString('techdocs.storageUrl')).pathname, + techdocsApiUrl, + ).toString(); + + const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + const indexHtmlUrl = `${storageUrl}/${entityRootDir}/index.html`; + fetch(indexHtmlUrl) + .then(() => resolve(true)) + .catch(() => resolve(false)); + }); + }); + } } diff --git a/packages/techdocs-common/src/stages/publish/publish.ts b/packages/techdocs-common/src/stages/publish/publish.ts index ca3cb99498..c91ebe8bb3 100644 --- a/packages/techdocs-common/src/stages/publish/publish.ts +++ b/packages/techdocs-common/src/stages/publish/publish.ts @@ -19,6 +19,7 @@ import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { PublisherType, PublisherBase } from './types'; import { LocalPublish } from './local'; +import { GoogleGCSPublish } from './googleStorage'; /** * Factory class to create a TechDocs publisher based on defined publisher type in app config. @@ -37,7 +38,7 @@ export class Publisher { switch (publisherType) { case 'google_gcs': logger.info('Creating Google Storage Bucket publisher for TechDocs'); - return new LocalPublish(config, logger, discovery); + return GoogleGCSPublish.fromConfig(config, logger); case 'local': logger.info('Creating Local publisher for TechDocs'); return new LocalPublish(config, logger, discovery); diff --git a/packages/techdocs-common/src/stages/publish/types.ts b/packages/techdocs-common/src/stages/publish/types.ts index 56f57c11c3..6b5c9332a9 100644 --- a/packages/techdocs-common/src/stages/publish/types.ts +++ b/packages/techdocs-common/src/stages/publish/types.ts @@ -14,9 +14,7 @@ * limitations under the License. */ import { Entity, EntityName } from '@backstage/catalog-model'; -// import serveStatic from 'serve-static'; -// import express from 'express'; -import * as express from 'express-serve-static-core'; +import express from 'express'; /** * Key for all the different types of TechDocs publishers that are supported. @@ -49,12 +47,15 @@ export interface PublisherBase { * Retrieve TechDocs Metadata about a site e.g. name, contributors, last updated, etc. * This API uses the techdocs_metadata.json file that co-exists along with the generated docs. */ - fetchTechDocsMetadata( - entityName: EntityName, - ): Promise<{ techdocsMetadataJson: string }>; + fetchTechDocsMetadata(entityName: EntityName): Promise; /** - * + * Route middleware to serve static documentation files for an entity. */ docsRouter(): express.Handler; + + /** + * Check if the index.html is present for the Entity at the Storage location. + */ + hasDocsBeenGenerated(entityName: Entity): Promise; } diff --git a/plugins/techdocs-backend/src/storage/BuildMetadataStorage.test.ts b/plugins/techdocs-backend/src/DocsBuilder/BuildMetadataStorage.test.ts similarity index 100% rename from plugins/techdocs-backend/src/storage/BuildMetadataStorage.test.ts rename to plugins/techdocs-backend/src/DocsBuilder/BuildMetadataStorage.test.ts diff --git a/plugins/techdocs-backend/src/storage/BuildMetadataStorage.ts b/plugins/techdocs-backend/src/DocsBuilder/BuildMetadataStorage.ts similarity index 100% rename from plugins/techdocs-backend/src/storage/BuildMetadataStorage.ts rename to plugins/techdocs-backend/src/DocsBuilder/BuildMetadataStorage.ts diff --git a/plugins/techdocs-backend/src/DocsBuilder/builder.ts b/plugins/techdocs-backend/src/DocsBuilder/builder.ts new file mode 100644 index 0000000000..67d0dfb89e --- /dev/null +++ b/plugins/techdocs-backend/src/DocsBuilder/builder.ts @@ -0,0 +1,129 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import Docker from 'dockerode'; +import { Logger } from 'winston'; +import { Entity } from '@backstage/catalog-model'; +import { + PreparerBuilder, + PublisherBase, + GeneratorBuilder, + PreparerBase, + GeneratorBase, + getLocationForEntity, + getLastCommitTimestamp, +} from '@backstage/techdocs-common'; +import { BuildMetadataStorage } from '.'; + +const getEntityId = (entity: Entity) => { + return `${entity.kind}:${entity.metadata.namespace ?? ''}:${ + entity.metadata.name + }`; +}; + +type DocsBuilderArguments = { + preparers: PreparerBuilder; + generators: GeneratorBuilder; + publisher: PublisherBase; + entity: Entity; + logger: Logger; + dockerClient: Docker; +}; + +export class DocsBuilder { + private preparer: PreparerBase; + private generator: GeneratorBase; + private publisher: PublisherBase; + private entity: Entity; + private logger: Logger; + private dockerClient: Docker; + + constructor({ + preparers, + generators, + publisher, + entity, + logger, + dockerClient, + }: DocsBuilderArguments) { + this.preparer = preparers.get(entity); + this.generator = generators.get(entity); + this.publisher = publisher; + this.entity = entity; + this.logger = logger; + this.dockerClient = dockerClient; + } + + public async build() { + this.logger.info(`Running preparer on entity ${getEntityId(this.entity)}`); + const preparedDir = await this.preparer.prepare(this.entity); + + const parsedLocationAnnotation = getLocationForEntity(this.entity); + + this.logger.info(`Running generator on entity ${getEntityId(this.entity)}`); + const { resultDir } = await this.generator.run({ + directory: preparedDir, + dockerClient: this.dockerClient, + parsedLocationAnnotation, + }); + + this.logger.info(`Running publisher on entity ${getEntityId(this.entity)}`); + await this.publisher.publish({ + entity: this.entity, + directory: resultDir, + }); + + if (!this.entity.metadata.uid) { + throw new Error( + 'Trying to build documentation for entity not in service catalog', + ); + } + + new BuildMetadataStorage(this.entity.metadata.uid).storeBuildTimestamp(); + } + + public async docsUpToDate() { + if (!this.entity.metadata.uid) { + throw new Error( + 'Trying to build documentation for entity not in service catalog', + ); + } + + const buildMetadataStorage = new BuildMetadataStorage( + this.entity.metadata.uid, + ); + const { type, target } = getLocationForEntity(this.entity); + + // Unless docs are stored locally + const nonAgeCheckTypes = ['dir', 'file', 'url']; + if (!nonAgeCheckTypes.includes(type)) { + const lastCommit = await getLastCommitTimestamp(target, this.logger); + const storageTimeStamp = buildMetadataStorage.getTimestamp(); + + // Check if documentation source is newer than what we have + if (storageTimeStamp && storageTimeStamp >= lastCommit) { + this.logger.debug( + `Docs for entity ${getEntityId(this.entity)} is up to date.`, + ); + return true; + } + } + + this.logger.debug( + `Docs for entity ${getEntityId(this.entity)} was outdated.`, + ); + return false; + } +} diff --git a/plugins/techdocs-backend/src/storage/index.ts b/plugins/techdocs-backend/src/DocsBuilder/index.ts similarity index 95% rename from plugins/techdocs-backend/src/storage/index.ts rename to plugins/techdocs-backend/src/DocsBuilder/index.ts index 3d37f69679..1380e24e7c 100644 --- a/plugins/techdocs-backend/src/storage/index.ts +++ b/plugins/techdocs-backend/src/DocsBuilder/index.ts @@ -14,3 +14,4 @@ * limitations under the License. */ export * from './BuildMetadataStorage'; +export * from './builder'; diff --git a/plugins/techdocs-backend/src/index.ts b/plugins/techdocs-backend/src/index.ts index 5c57882788..50a98990cc 100644 --- a/plugins/techdocs-backend/src/index.ts +++ b/plugins/techdocs-backend/src/index.ts @@ -15,4 +15,5 @@ */ export * from './service/router'; +// TODO: Do named exports here e.g. publishers, generators. export * from '@backstage/techdocs-common'; diff --git a/plugins/techdocs-backend/src/service/helpers.test.ts b/plugins/techdocs-backend/src/service/helpers.test.ts new file mode 100644 index 0000000000..507373107d --- /dev/null +++ b/plugins/techdocs-backend/src/service/helpers.test.ts @@ -0,0 +1,26 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { getEntityNameFromUrlPath } from './helpers'; + +describe('getEntityNameFromUrlPath', () => { + it('should parse correctly', () => { + const path = 'default/Component/documented-component'; + const parsedEntity = getEntityNameFromUrlPath(path); + expect(parsedEntity).toHaveProperty('namespace', 'default'); + expect(parsedEntity).toHaveProperty('kind', 'Component'); + expect(parsedEntity).toHaveProperty('name', 'documented-component'); + }); +}); diff --git a/plugins/techdocs-backend/src/service/helpers.ts b/plugins/techdocs-backend/src/service/helpers.ts index 51e7ed8ad2..fc80ab717d 100644 --- a/plugins/techdocs-backend/src/service/helpers.ts +++ b/plugins/techdocs-backend/src/service/helpers.ts @@ -13,132 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import Docker from 'dockerode'; -import { Logger } from 'winston'; -import { Entity, EntityName } from '@backstage/catalog-model'; -import { - PreparerBuilder, - PublisherBase, - GeneratorBuilder, - PreparerBase, - GeneratorBase, - getLocationForEntity, - getLastCommitTimestamp, -} from '@backstage/techdocs-common'; -import { BuildMetadataStorage } from '../storage'; - -const getEntityId = (entity: Entity) => { - return `${entity.kind}:${entity.metadata.namespace ?? ''}:${ - entity.metadata.name - }`; -}; - -type DocsBuilderArguments = { - preparers: PreparerBuilder; - generators: GeneratorBuilder; - publisher: PublisherBase; - entity: Entity; - logger: Logger; - dockerClient: Docker; -}; - -export class DocsBuilder { - private preparer: PreparerBase; - private generator: GeneratorBase; - private publisher: PublisherBase; - private entity: Entity; - private logger: Logger; - private dockerClient: Docker; - - constructor({ - preparers, - generators, - publisher, - entity, - logger, - dockerClient, - }: DocsBuilderArguments) { - this.preparer = preparers.get(entity); - this.generator = generators.get(entity); - this.publisher = publisher; - this.entity = entity; - this.logger = logger; - this.dockerClient = dockerClient; - } - - public async build() { - this.logger.info(`Running preparer on entity ${getEntityId(this.entity)}`); - const preparedDir = await this.preparer.prepare(this.entity); - - const parsedLocationAnnotation = getLocationForEntity(this.entity); - - this.logger.info(`Running generator on entity ${getEntityId(this.entity)}`); - const { resultDir } = await this.generator.run({ - directory: preparedDir, - dockerClient: this.dockerClient, - parsedLocationAnnotation, - }); - - this.logger.info(`Running publisher on entity ${getEntityId(this.entity)}`); - await this.publisher.publish({ - entity: this.entity, - directory: resultDir, - }); - - if (!this.entity.metadata.uid) { - throw new Error( - 'Trying to build documentation for entity not in service catalog', - ); - } - - new BuildMetadataStorage(this.entity.metadata.uid).storeBuildTimestamp(); - } - - public async docsUpToDate() { - if (!this.entity.metadata.uid) { - throw new Error( - 'Trying to build documentation for entity not in service catalog', - ); - } - - const buildMetadataStorage = new BuildMetadataStorage( - this.entity.metadata.uid, - ); - const { type, target } = getLocationForEntity(this.entity); - - // Unless docs are stored locally - const nonAgeCheckTypes = ['dir', 'file', 'url']; - if (!nonAgeCheckTypes.includes(type)) { - const lastCommit = await getLastCommitTimestamp(target, this.logger); - const storageTimeStamp = buildMetadataStorage.getTimestamp(); - - // Check if documentation source is newer than what we have - if (storageTimeStamp && storageTimeStamp >= lastCommit) { - this.logger.debug( - `Docs for entity ${getEntityId(this.entity)} is up to date.`, - ); - return true; - } - } - - this.logger.debug( - `Docs for entity ${getEntityId(this.entity)} was outdated.`, - ); - return false; - } -} - +import { EntityName } from '@backstage/catalog-model'; /** * Using the path of the TechDocs page URL, return a structured EntityName type object with namespace, * kind and name of the Entity. * @param {string} path Example: default/Component/documented-component */ export const getEntityNameFromUrlPath = (path: string): EntityName => { - const [kind, namespace, name] = path.split('/'); + const [namespace, kind, name] = path.split('/'); return { - kind, namespace, + kind, name, }; }; diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index dd084bde4b..7f6d7c69bc 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -28,7 +28,8 @@ import { } from '@backstage/techdocs-common'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; import { Entity } from '@backstage/catalog-model'; -import { DocsBuilder, getEntityNameFromUrlPath } from './helpers'; +import { getEntityNameFromUrlPath } from './helpers'; +import { DocsBuilder } from '../DocsBuilder'; type RouterOptions = { preparers: PreparerBuilder; @@ -92,9 +93,8 @@ export async function createRouter({ }); router.get('/docs/:namespace/:kind/:name/*', async (req, res) => { - const storageUrl = config.getString('techdocs.storageUrl'); - const { kind, namespace, name } = req.params; + const storageUrl = config.getString('techdocs.storageUrl'); const catalogUrl = await discovery.getBaseUrl('catalog'); const triple = [kind, namespace, name].map(encodeURIComponent).join('/'); @@ -109,22 +109,55 @@ export async function createRouter({ const entity: Entity = await catalogRes.json(); - const docsBuilder = new DocsBuilder({ - preparers, - generators, - publisher, - dockerClient, - logger, - entity, - }); + let publisherType = ''; + try { + publisherType = config.getString('techdocs.publisher.type'); + } catch (err) { + throw new Error( + 'Unable to get techdocs.publisher.type in your app config. Set it to either ' + + "'local', 'google_gcs' or other support storage providers. Read more here " + + 'https://backstage.io/docs/features/techdocs/architecture', + ); + } - if (!(await docsBuilder.docsUpToDate())) { - await docsBuilder.build(); + // techdocs-backend will only try to build documentation for an entity if techdocs.docsBuilder is set to 'local' + // If set to 'ci', it will only try to fetch and assume that that CI/CD pipeline of the repository hosting the + // entity's documentation is responsible for building and publishing documentation to the storage provider. + const shouldBuildDocs = + config.getString('techdocs.docsBuilder') === 'local'; + + if (shouldBuildDocs) { + const docsBuilder = new DocsBuilder({ + preparers, + generators, + publisher, + dockerClient, + logger, + entity, + }); + if (publisherType === 'local') { + if (!(await docsBuilder.docsUpToDate())) { + await docsBuilder.build(); + } + } else if (publisherType === 'google_gcs') { + if (!(await publisher.hasDocsBeenGenerated(entity))) { + logger.info( + 'Did not find generated docs files for the entity. Building docs.', + ); + await docsBuilder.build(); + } else { + logger.info( + 'Found pre-generated docs for this entity. Serving them.', + ); + // TODO: When to re-trigger build for cache invalidation? + } + } } res.redirect(`${storageUrl}${req.path.replace('/docs', '')}`); }); + // Route middleware which serves files from the storage set in the publisher. router.use('/static/docs', publisher.docsRouter()); return router; diff --git a/yarn.lock b/yarn.lock index e031375fc0..9e43ab4deb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1828,6 +1828,65 @@ query-string "^6.13.3" xcase "^2.0.1" +"@google-cloud/common@^3.5.0": + version "3.5.0" + resolved "https://registry.npmjs.org/@google-cloud/common/-/common-3.5.0.tgz#0959e769e8075a06eb0823cc567eef00fd0c2d02" + integrity sha512-10d7ZAvKhq47L271AqvHEd8KzJqGU45TY+rwM2Z3JHuB070FeTi7oJJd7elfrnKaEvaktw3hH2wKnRWxk/3oWQ== + dependencies: + "@google-cloud/projectify" "^2.0.0" + "@google-cloud/promisify" "^2.0.0" + arrify "^2.0.1" + duplexify "^4.1.1" + ent "^2.2.0" + extend "^3.0.2" + google-auth-library "^6.1.1" + retry-request "^4.1.1" + teeny-request "^7.0.0" + +"@google-cloud/paginator@^3.0.0": + version "3.0.5" + resolved "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-3.0.5.tgz#9d6b96c421a89bd560c1bc2c197c7611ef21db6c" + integrity sha512-N4Uk4BT1YuskfRhKXBs0n9Lg2YTROZc6IMpkO/8DIHODtm5s3xY8K5vVBo23v/2XulY3azwITQlYWgT4GdLsUw== + dependencies: + arrify "^2.0.0" + extend "^3.0.2" + +"@google-cloud/projectify@^2.0.0": + version "2.0.1" + resolved "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-2.0.1.tgz#13350ee609346435c795bbfe133a08dfeab78d65" + integrity sha512-ZDG38U/Yy6Zr21LaR3BTiiLtpJl6RkPS/JwoRT453G+6Q1DhlV0waNf8Lfu+YVYGIIxgKnLayJRfYlFJfiI8iQ== + +"@google-cloud/promisify@^2.0.0": + version "2.0.3" + resolved "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-2.0.3.tgz#f934b5cdc939e3c7039ff62b9caaf59a9d89e3a8" + integrity sha512-d4VSA86eL/AFTe5xtyZX+ePUjE8dIFu2T8zmdeNBSa5/kNgXPCx/o/wbFNHAGLJdGnk1vddRuMESD9HbOC8irw== + +"@google-cloud/storage@^5.6.0": + version "5.6.0" + resolved "https://registry.npmjs.org/@google-cloud/storage/-/storage-5.6.0.tgz#bc6925c7970c375212a4da21c123298fc9665dec" + integrity sha512-nLcym8IuCzy1O7tNTXNFuMHfX900sTM3kSTqbKe7oFSoKUiaIM+FHuuuDimMMlieY6StA1xYNPRFFHz57Nv8YQ== + dependencies: + "@google-cloud/common" "^3.5.0" + "@google-cloud/paginator" "^3.0.0" + "@google-cloud/promisify" "^2.0.0" + arrify "^2.0.0" + compressible "^2.0.12" + date-and-time "^0.14.0" + duplexify "^4.0.0" + extend "^3.0.2" + gaxios "^4.0.0" + gcs-resumable-upload "^3.1.0" + get-stream "^6.0.0" + hash-stream-validation "^0.2.2" + mime "^2.2.0" + mime-types "^2.0.8" + onetime "^5.1.0" + p-limit "^3.0.1" + pumpify "^2.0.0" + snakeize "^0.1.0" + stream-events "^1.0.1" + xdg-basedir "^4.0.0" + "@graphql-codegen/cli@^1.17.7": version "1.17.10" resolved "https://registry.npmjs.org/@graphql-codegen/cli/-/cli-1.17.10.tgz#efebf9b887fdb94dd26dbf3eb1950e832efcda0e" @@ -5492,6 +5551,13 @@ dependencies: "@types/node" "*" +"@types/klaw@^3.0.1": + version "3.0.1" + resolved "https://registry.npmjs.org/@types/klaw/-/klaw-3.0.1.tgz#29f90021c0234976aa4eb97efced9cb6db9fa8b3" + integrity sha512-acnF3n9mYOr1aFJKFyvfNX0am9EtPUsYPq22QUCGdJE+MVt6UyAN1jwo+PmOPqXD4K7ZS9MtxDEp/un0lxFccA== + dependencies: + "@types/node" "*" + "@types/koa-compose@*": version "3.2.5" resolved "https://registry.npmjs.org/@types/koa-compose/-/koa-compose-3.2.5.tgz#85eb2e80ac50be95f37ccf8c407c09bbe3468e9d" @@ -6521,6 +6587,13 @@ abbrev@1: resolved "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q== +abort-controller@^3.0.0: + version "3.0.0" + resolved "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392" + integrity sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg== + dependencies: + event-target-shim "^5.0.0" + abstract-logging@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/abstract-logging/-/abstract-logging-2.0.0.tgz#08a85814946c98ef06f4256ad470aba1886d4490" @@ -7208,6 +7281,11 @@ arrify@^1.0.1: resolved "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz#898508da2226f380df904728456849c1501a4b0d" integrity sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0= +arrify@^2.0.0, arrify@^2.0.1: + version "2.0.1" + resolved "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz#c9655e9331e0abcd588d2a7cad7e9956f66701fa" + integrity sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug== + asap@^2.0.0, asap@~2.0.3, asap@~2.0.6: version "2.0.6" resolved "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz#e50347611d7e690943208bbdafebcbc2fb866d46" @@ -7807,7 +7885,7 @@ base64-js@^1.0.2, base64-js@^1.2.0: resolved "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz#58ece8cb75dd07e71ed08c736abc5fac4dbf8df1" integrity sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g== -base64-js@^1.3.1: +base64-js@^1.3.0, base64-js@^1.3.1: version "1.5.1" resolved "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== @@ -7893,6 +7971,11 @@ big.js@^5.2.2: resolved "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz#65f0af382f578bcdc742bd9c281e9cb2d7768328" integrity sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ== +bignumber.js@^9.0.0: + version "9.0.1" + resolved "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.1.tgz#8d7ba124c882bfd8e43260c67475518d0689e4e5" + integrity sha512-IdZR9mh6ahOBv/hYGiXyVuyCetmGJhtYkqLBpTStdhEGjegpPlUawydyaF3pbIOFynJTpllEs+NP+CS9jKFLjA== + binary-extensions@^1.0.0: version "1.13.1" resolved "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.13.1.tgz#598afe54755b2868a5330d2aff9d4ebb53209b65" @@ -9134,7 +9217,7 @@ compress-commons@^4.0.0: normalize-path "^3.0.0" readable-stream "^3.6.0" -compressible@~2.0.16: +compressible@^2.0.12, compressible@~2.0.16: version "2.0.18" resolved "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz#af53cca6b070d4c3c0750fbd77286a6d7cc46fba" integrity sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg== @@ -9228,7 +9311,7 @@ config-chain@^1.1.11: ini "^1.3.4" proto-list "~1.2.1" -configstore@^5.0.1: +configstore@^5.0.0, configstore@^5.0.1: version "5.0.1" resolved "https://registry.npmjs.org/configstore/-/configstore-5.0.1.tgz#d365021b5df4b98cdd187d6a3b0e3f6a7cc5ed96" integrity sha512-aMKprgk5YhBNyH25hj8wGt2+D52Sw1DRRIzqBwLp2Ya9mFmY8KPvvtvmna8SxVR9JMZ4kzMD68N22vlaRpkeFA== @@ -10193,6 +10276,11 @@ dataloader@2.0.0: resolved "https://registry.npmjs.org/dataloader/-/dataloader-2.0.0.tgz#41eaf123db115987e21ca93c005cd7753c55fe6f" integrity sha512-YzhyDAwA4TaQIhM5go+vCLmU0UikghC/t9DTQYZR2M/UvZ1MdOhPezSDZcjj9uqQJOMqjLcpWtyW2iNINdlatQ== +date-and-time@^0.14.0: + version "0.14.1" + resolved "https://registry.npmjs.org/date-and-time/-/date-and-time-0.14.1.tgz#969634697b78956fb66b8be6fb0f39fbd631f2f6" + integrity sha512-M4RggEH5OF2ZuCOxgOU67R6Z9ohjKbxGvAQz48vj53wLmL0bAgumkBvycR32f30pK+Og9pIR+RFDyChbaE4oLA== + date-fns@^1.27.2: version "1.30.1" resolved "https://registry.npmjs.org/date-fns/-/date-fns-1.30.1.tgz#2e71bf0b119153dbb4cc4e88d9ea5acfb50dc05c" @@ -10874,6 +10962,16 @@ duplexify@^3.4.2, duplexify@^3.6.0: readable-stream "^2.0.0" stream-shift "^1.0.0" +duplexify@^4.0.0, duplexify@^4.1.1: + version "4.1.1" + resolved "https://registry.npmjs.org/duplexify/-/duplexify-4.1.1.tgz#7027dc374f157b122a8ae08c2d3ea4d2d953aa61" + integrity sha512-DY3xVEmVHTv1wSzKNbwoU6nVjzI369Y6sPoqfYr0/xlx3IdX2n94xIszTcjPO8W8ZIv0Wb0PXNcjuZyT4wiICA== + dependencies: + end-of-stream "^1.4.1" + inherits "^2.0.3" + readable-stream "^3.1.1" + stream-shift "^1.0.0" + ecc-jsbn@~0.1.1: version "0.1.2" resolved "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9" @@ -10882,7 +10980,7 @@ ecc-jsbn@~0.1.1: jsbn "~0.1.0" safer-buffer "^2.1.0" -ecdsa-sig-formatter@1.0.11: +ecdsa-sig-formatter@1.0.11, ecdsa-sig-formatter@^1.0.11: version "1.0.11" resolved "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz#ae0f0fa2d85045ef14a817daa3ce9acd0489e5bf" integrity sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ== @@ -11026,6 +11124,11 @@ enquirer@^2.3.0, enquirer@^2.3.5: dependencies: ansi-colors "^4.1.1" +ent@^2.2.0: + version "2.2.0" + resolved "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz#e964219325a21d05f44466a2f686ed6ce5f5dd1d" + integrity sha1-6WQhkyWiHQX0RGai9obtbOX13R0= + entities@^1.1.1, entities@^1.1.2: version "1.1.2" resolved "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz#bdfa735299664dfafd34529ed4f8522a275fea56" @@ -11574,6 +11677,11 @@ event-stream@=3.3.4: stream-combiner "~0.0.4" through "~2.3.1" +event-target-shim@^5.0.0: + version "5.0.1" + resolved "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789" + integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ== + eventemitter2@^6.4.2: version "6.4.3" resolved "https://registry.npmjs.org/eventemitter2/-/eventemitter2-6.4.3.tgz#35c563619b13f3681e7eb05cbdaf50f56ba58820" @@ -11809,7 +11917,7 @@ extend-shallow@^3.0.0, extend-shallow@^3.0.2: assign-symbols "^1.0.0" is-extendable "^1.0.1" -extend@3.0.2, extend@^3.0.0, extend@~3.0.2: +extend@3.0.2, extend@^3.0.0, extend@^3.0.2, extend@~3.0.2: version "3.0.2" resolved "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== @@ -11936,6 +12044,11 @@ fast-shallow-equal@^1.0.0: resolved "https://registry.npmjs.org/fast-shallow-equal/-/fast-shallow-equal-1.0.0.tgz#d4dcaf6472440dcefa6f88b98e3251e27f25628b" integrity sha512-HPtaa38cPgWvaCFmRNhlc6NG7pv6NUHqjPgVAkWGoB9mQMwYB27/K0CvOM5Czy+qpT3e8XJ6Q4aPAnzpNpzNaw== +fast-text-encoding@^1.0.0: + version "1.0.3" + resolved "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.3.tgz#ec02ac8e01ab8a319af182dae2681213cfe9ce53" + integrity sha512-dtm4QZH9nZtcDt8qJiOH9fcQd1NAgi+K1O2DbE6GG1PPCK/BWfOH3idCTRQ4ImXRUOyopDEgDEnVEE7Y/2Wrig== + fastest-stable-stringify@^1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/fastest-stable-stringify/-/fastest-stable-stringify-1.0.1.tgz#9122d406d4c9d98bea644a6b6853d5874b87b028" @@ -12532,6 +12645,49 @@ gauge@~2.7.3: strip-ansi "^3.0.1" wide-align "^1.1.0" +gaxios@^3.0.0: + version "3.2.0" + resolved "https://registry.npmjs.org/gaxios/-/gaxios-3.2.0.tgz#11b6f0e8fb08d94a10d4d58b044ad3bec6dd486a" + integrity sha512-+6WPeVzPvOshftpxJwRi2Ozez80tn/hdtOUag7+gajDHRJvAblKxTFSSMPtr2hmnLy7p0mvYz0rMXLBl8pSO7Q== + dependencies: + abort-controller "^3.0.0" + extend "^3.0.2" + https-proxy-agent "^5.0.0" + is-stream "^2.0.0" + node-fetch "^2.3.0" + +gaxios@^4.0.0: + version "4.0.1" + resolved "https://registry.npmjs.org/gaxios/-/gaxios-4.0.1.tgz#bc7b205a89d883452822cc75e138620c35e3291e" + integrity sha512-jOin8xRZ/UytQeBpSXFqIzqU7Fi5TqgPNLlUsSB8kjJ76+FiGBfImF8KJu++c6J4jOldfJUtt0YmkRj2ZpSHTQ== + dependencies: + abort-controller "^3.0.0" + extend "^3.0.2" + https-proxy-agent "^5.0.0" + is-stream "^2.0.0" + node-fetch "^2.3.0" + +gcp-metadata@^4.2.0: + version "4.2.1" + resolved "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-4.2.1.tgz#31849fbcf9025ef34c2297c32a89a1e7e9f2cd62" + integrity sha512-tSk+REe5iq/N+K+SK1XjZJUrFPuDqGZVzCy2vocIHIGmPlTGsa8owXMJwGkrXr73NO0AzhPW4MF2DEHz7P2AVw== + dependencies: + gaxios "^4.0.0" + json-bigint "^1.0.0" + +gcs-resumable-upload@^3.1.0: + version "3.1.1" + resolved "https://registry.npmjs.org/gcs-resumable-upload/-/gcs-resumable-upload-3.1.1.tgz#67c766a0555d6a352f9651b7603337207167d0de" + integrity sha512-RS1osvAicj9+MjCc6jAcVL1Pt3tg7NK2C2gXM5nqD1Gs0klF2kj5nnAFSBy97JrtslMIQzpb7iSuxaG8rFWd2A== + dependencies: + abort-controller "^3.0.0" + configstore "^5.0.0" + extend "^3.0.2" + gaxios "^3.0.0" + google-auth-library "^6.0.0" + pumpify "^2.0.0" + stream-events "^1.0.4" + generic-names@^2.0.1: version "2.0.1" resolved "https://registry.npmjs.org/generic-names/-/generic-names-2.0.1.tgz#f8a378ead2ccaa7a34f0317b05554832ae41b872" @@ -12636,6 +12792,11 @@ get-stream@^5.0.0, get-stream@^5.1.0: dependencies: pump "^3.0.0" +get-stream@^6.0.0: + version "6.0.0" + resolved "https://registry.npmjs.org/get-stream/-/get-stream-6.0.0.tgz#3e0012cb6827319da2706e601a1583e8629a6718" + integrity sha512-A1B3Bh1UmL0bidM/YX2NsCOTnGJePL9rO/M+Mw3m9f2gUpfokS0hi5Eah0WSUEWZdZhIZtMjkIYS7mDfOqNHbg== + get-value@^2.0.3, get-value@^2.0.6: version "2.0.6" resolved "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz#dc15ca1c672387ca76bd37ac0a395ba2042a2c28" @@ -12930,6 +13091,28 @@ good-listener@^1.2.2: dependencies: delegate "^3.1.2" +google-auth-library@^6.0.0, google-auth-library@^6.1.1: + version "6.1.3" + resolved "https://registry.npmjs.org/google-auth-library/-/google-auth-library-6.1.3.tgz#39d868140b70d0c4b32c6f6d8f4ccc1400d84dca" + integrity sha512-m9mwvY3GWbr7ZYEbl61isWmk+fvTmOt0YNUfPOUY2VH8K5pZlAIWJjxEi0PqR3OjMretyiQLI6GURMrPSwHQ2g== + dependencies: + arrify "^2.0.0" + base64-js "^1.3.0" + ecdsa-sig-formatter "^1.0.11" + fast-text-encoding "^1.0.0" + gaxios "^4.0.0" + gcp-metadata "^4.2.0" + gtoken "^5.0.4" + jws "^4.0.0" + lru-cache "^6.0.0" + +google-p12-pem@^3.0.3: + version "3.0.3" + resolved "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-3.0.3.tgz#673ac3a75d3903a87f05878f3c75e06fc151669e" + integrity sha512-wS0ek4ZtFx/ACKYF3JhyGe5kzH7pgiQ7J5otlumqR9psmWMYc+U9cErKlCYVYHoUaidXHdZ2xbo34kB+S+24hA== + dependencies: + node-forge "^0.10.0" + got@^10.7.0: version "10.7.0" resolved "https://registry.npmjs.org/got/-/got-10.7.0.tgz#62889dbcd6cca32cd6a154cc2d0c6895121d091f" @@ -13195,6 +13378,16 @@ growly@^1.3.0: resolved "https://registry.npmjs.org/growly/-/growly-1.3.0.tgz#f10748cbe76af964b7c96c93c6bcc28af120c081" integrity sha1-8QdIy+dq+WS3yWyTxrzCivEgwIE= +gtoken@^5.0.4: + version "5.1.0" + resolved "https://registry.npmjs.org/gtoken/-/gtoken-5.1.0.tgz#4ba8d2fc9a8459098f76e7e8fd7beaa39fda9fe4" + integrity sha512-4d8N6Lk8TEAHl9vVoRVMh9BNOKWVgl2DdNtr3428O75r3QFrF/a5MMu851VmK0AA8+iSvbwRv69k5XnMLURGhg== + dependencies: + gaxios "^4.0.0" + google-p12-pem "^3.0.3" + jws "^4.0.0" + mime "^2.2.0" + gud@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/gud/-/gud-1.0.0.tgz#a489581b17e6a70beca9abe3ae57de7a499852c0" @@ -13326,6 +13519,11 @@ hash-base@^3.0.0: inherits "^2.0.1" safe-buffer "^5.0.1" +hash-stream-validation@^0.2.2: + version "0.2.4" + resolved "https://registry.npmjs.org/hash-stream-validation/-/hash-stream-validation-0.2.4.tgz#ee68b41bf822f7f44db1142ec28ba9ee7ccb7512" + integrity sha512-Gjzu0Xn7IagXVkSu9cSFuK1fqzwtLwFhNhVL8IFJijRNMgUttFbBSIAzKuSIrsFMO1+g1RlsoN49zPIbwPDMGQ== + hash.js@^1.0.0, hash.js@^1.0.3: version "1.1.7" resolved "https://registry.npmjs.org/hash.js/-/hash.js-1.1.7.tgz#0babca538e8d4ee4a0f8988d68866537a003cf42" @@ -13618,7 +13816,7 @@ http-proxy-agent@^2.1.0: agent-base "4" debug "3.1.0" -http-proxy-agent@^4.0.1: +http-proxy-agent@^4.0.0, http-proxy-agent@^4.0.1: version "4.0.1" resolved "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz#8a8c8ef7f5932ccf953c296ca8291b95aa74aa3a" integrity sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg== @@ -15287,6 +15485,13 @@ jsesc@~0.5.0: resolved "https://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz#e7dee66e35d6fc16f710fe91d5cf69f70f08911d" integrity sha1-597mbjXW/Bb3EP6R1c9p9w8IkR0= +json-bigint@^1.0.0: + version "1.0.0" + resolved "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz#ae547823ac0cad8398667f8cd9ef4730f5b01ff1" + integrity sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ== + dependencies: + bignumber.js "^9.0.0" + json-buffer@3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.0.tgz#5b1f397afc75d677bde8bcfc0e47e1f9a3d9a898" @@ -15587,6 +15792,15 @@ jwa@^1.4.1: ecdsa-sig-formatter "1.0.11" safe-buffer "^5.0.1" +jwa@^2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz#a7e9c3f29dae94027ebcaf49975c9345593410fc" + integrity sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA== + dependencies: + buffer-equal-constant-time "1.0.1" + ecdsa-sig-formatter "1.0.11" + safe-buffer "^5.0.1" + jws@^3.2.2: version "3.2.2" resolved "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz#001099f3639468c9414000e99995fa52fb478304" @@ -15595,6 +15809,14 @@ jws@^3.2.2: jwa "^1.4.1" safe-buffer "^5.0.1" +jws@^4.0.0: + version "4.0.0" + resolved "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz#2d4e8cf6a318ffaa12615e9dec7e86e6c97310f4" + integrity sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg== + dependencies: + jwa "^2.0.0" + safe-buffer "^5.0.1" + jwt-decode@2.2.0: version "2.2.0" resolved "https://registry.npmjs.org/jwt-decode/-/jwt-decode-2.2.0.tgz#7d86bd56679f58ce6a84704a657dd392bba81a79" @@ -15657,6 +15879,13 @@ klaw@^1.0.0: optionalDependencies: graceful-fs "^4.1.9" +klaw@^3.0.0: + version "3.0.0" + resolved "https://registry.npmjs.org/klaw/-/klaw-3.0.0.tgz#b11bec9cf2492f06756d6e809ab73a2910259146" + integrity sha512-0Fo5oir+O9jnXu5EefYbVK+mHMBeEVEy2cmctR1O1NECcCkPRreJKrS6Qt/j3KC2C148Dfo9i3pCmCMsdqGr0g== + dependencies: + graceful-fs "^4.1.9" + kleur@^3.0.3: version "3.0.3" resolved "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz#a79c9ecc86ee1ce3fa6206d1216c501f147fc07e" @@ -16862,7 +17091,7 @@ mime-db@1.44.0, "mime-db@>= 1.43.0 < 2": resolved "https://registry.npmjs.org/mime-db/-/mime-db-1.44.0.tgz#fa11c5eb0aca1334b4233cb4d52f10c5a6272f92" integrity sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg== -mime-types@^2.1.12, mime-types@^2.1.26, mime-types@~2.1.17, mime-types@~2.1.19, mime-types@~2.1.24: +mime-types@^2.0.8, mime-types@^2.1.12, mime-types@^2.1.26, mime-types@~2.1.17, mime-types@~2.1.19, mime-types@~2.1.24: version "2.1.27" resolved "https://registry.npmjs.org/mime-types/-/mime-types-2.1.27.tgz#47949f98e279ea53119f5722e0f34e529bec009f" integrity sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w== @@ -16874,6 +17103,11 @@ mime@1.6.0, mime@^1.4.1: resolved "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== +mime@^2.2.0: + version "2.4.6" + resolved "https://registry.npmjs.org/mime/-/mime-2.4.6.tgz#e5b407c90db442f2beb5b162373d07b69affa4d1" + integrity sha512-RZKhC3EmpBchfTGBVb8fb+RL2cWyw/32lshnsETttkBAyAUXSGHxbEJWWRXc751DrIxG1q04b8QwMbAwkRPpUA== + mime@^2.3.1, mime@^2.4.4: version "2.4.4" resolved "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz#bd7b91135fc6b01cde3e9bae33d659b63d8857e5" @@ -18177,6 +18411,13 @@ p-limit@^2.0.0, p-limit@^2.2.0: dependencies: p-try "^2.0.0" +p-limit@^3.0.1: + version "3.1.0" + resolved "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz#e1daccbe78d0d1388ca18c64fea38e3e57e3706b" + integrity sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ== + dependencies: + yocto-queue "^0.1.0" + p-locate@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz#20a0103b222a70c8fd39cc2e580680f3dde5ec43" @@ -19707,6 +19948,15 @@ pumpify@^1.3.3: inherits "^2.0.3" pump "^2.0.0" +pumpify@^2.0.0: + version "2.0.1" + resolved "https://registry.npmjs.org/pumpify/-/pumpify-2.0.1.tgz#abfc7b5a621307c728b551decbbefb51f0e4aa1e" + integrity sha512-m7KOje7jZxrmutanlkS1daj1dS6z6BgslzOXmcSEpIlCxM3VJH7lG5QLeck/6hgF6F4crFf01UtQmNsJfweTAw== + dependencies: + duplexify "^4.1.1" + inherits "^2.0.3" + pump "^3.0.0" + punycode@1.3.2: version "1.3.2" resolved "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d" @@ -21183,6 +21433,13 @@ ret@~0.1.10: resolved "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz#b8a4825d5bdb1fc3f6f53c2bc33f81388681c7bc" integrity sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg== +retry-request@^4.1.1: + version "4.1.3" + resolved "https://registry.npmjs.org/retry-request/-/retry-request-4.1.3.tgz#d5f74daf261372cff58d08b0a1979b4d7cab0fde" + integrity sha512-QnRZUpuPNgX0+D1xVxul6DbJ9slvo4Rm6iV/dn63e048MvGbUZiKySVt6Tenp04JqmchxjiLltGerOJys7kJYQ== + dependencies: + debug "^4.1.1" + retry@0.12.0, retry@^0.12.0: version "0.12.0" resolved "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz#1b42a6266a21f07421d1b0b54b7dc167b01c013b" @@ -21847,6 +22104,11 @@ smartwrap@^1.2.3: wcwidth "^1.0.1" yargs "^15.1.0" +snakeize@^0.1.0: + version "0.1.0" + resolved "https://registry.npmjs.org/snakeize/-/snakeize-0.1.0.tgz#10c088d8b58eb076b3229bb5a04e232ce126422d" + integrity sha1-EMCI2LWOsHazIpu1oE4jLOEmQi0= + snapdragon-node@^2.0.1: version "2.1.1" resolved "https://registry.npmjs.org/snapdragon-node/-/snapdragon-node-2.1.1.tgz#6c175f86ff14bdb0724563e8f3c1b021a286853b" @@ -22298,6 +22560,13 @@ stream-each@^1.1.0: end-of-stream "^1.1.0" stream-shift "^1.0.0" +stream-events@^1.0.1, stream-events@^1.0.4, stream-events@^1.0.5: + version "1.0.5" + resolved "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz#bbc898ec4df33a4902d892333d47da9bf1c406d5" + integrity sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg== + dependencies: + stubs "^3.0.0" + stream-http@^2.7.2: version "2.8.3" resolved "https://registry.npmjs.org/stream-http/-/stream-http-2.8.3.tgz#b2d242469288a5a27ec4fe8933acf623de6514fc" @@ -22577,6 +22846,11 @@ strong-log-transformer@^2.0.0: minimist "^1.2.0" through "^2.3.4" +stubs@^3.0.0: + version "3.0.0" + resolved "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz#e8d2ba1fa9c90570303c030b6900f7d5f89abe5b" + integrity sha1-6NK6H6nJBXAwPAMLaQD31fiavls= + style-inject@^0.3.0: version "0.3.0" resolved "https://registry.npmjs.org/style-inject/-/style-inject-0.3.0.tgz#d21c477affec91811cc82355832a700d22bf8dd3" @@ -22953,6 +23227,17 @@ tdigest@^0.1.1: dependencies: bintrees "1.0.1" +teeny-request@^7.0.0: + version "7.0.1" + resolved "https://registry.npmjs.org/teeny-request/-/teeny-request-7.0.1.tgz#bdd41fdffea5f8fbc0d29392cb47bec4f66b2b4c" + integrity sha512-sasJmQ37klOlplL4Ia/786M5YlOcoLGQyq2TE4WHSRupbAuDaQW0PfVxV4MtdBtRJ4ngzS+1qim8zP6Zp35qCw== + dependencies: + http-proxy-agent "^4.0.0" + https-proxy-agent "^5.0.0" + node-fetch "^2.6.1" + stream-events "^1.0.5" + uuid "^8.0.0" + telejson@^5.0.2: version "5.0.2" resolved "https://registry.npmjs.org/telejson/-/telejson-5.0.2.tgz#ed1e64be250cc1c757a53c19e1740b49832b3d51" @@ -25045,6 +25330,11 @@ yn@^4.0.0: resolved "https://registry.npmjs.org/yn/-/yn-4.0.0.tgz#611480051ea43b510da1dfdbe177ed159f00a979" integrity sha512-huWiiCS4TxKc4SfgmTwW1K7JmXPPAmuXWYy4j9qjQo4+27Kni8mGhAAi1cloRWmBe2EqcLgt3IGqQoRL/MtPgg== +yocto-queue@^0.1.0: + version "0.1.0" + resolved "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b" + integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== + yup@^0.29.3: version "0.29.3" resolved "https://registry.npmjs.org/yup/-/yup-0.29.3.tgz#69a30fd3f1c19f5d9e31b1cf1c2b851ce8045fea" From 1051e5f3424974302cc4b3c9364282d7f1213427 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Thu, 3 Dec 2020 22:54:30 +0100 Subject: [PATCH 007/188] Rebuild yarn.lock after new release --- docs/features/techdocs/configuration.md | 9 +- docs/features/techdocs/using-cloud-storage.md | 45 +++-- yarn.lock | 175 ++++++++++++++++-- 3 files changed, 198 insertions(+), 31 deletions(-) diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index 8b0f47c134..5c1b9ab5c0 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -1,12 +1,13 @@ --- id: configuration title: TechDocs Configuration Options -description: Reference documentation for configuring TechDocs using app-config.yaml +description: + Reference documentation for configuring TechDocs using app-config.yaml --- -Using the `app-config.yaml` in the Backstage app, you can configure TechDocs using several options. -This page serves as a reference to all the available configuration options for TechDocs. - +Using the `app-config.yaml` in the Backstage app, you can configure TechDocs +using several options. This page serves as a reference to all the available +configuration options for TechDocs. ```yaml # File: app-config.yaml diff --git a/docs/features/techdocs/using-cloud-storage.md b/docs/features/techdocs/using-cloud-storage.md index e435644ca2..07b2982e18 100644 --- a/docs/features/techdocs/using-cloud-storage.md +++ b/docs/features/techdocs/using-cloud-storage.md @@ -4,18 +4,21 @@ title: Using Cloud Storage for TechDocs generated files description: Using Cloud Storage for TechDocs generated files --- -In the [TechDocs architecture](./architecture.md) you have the option to choose where you want to store -the Generated static files which TechDocs uses to render documentation. In both the "Basic" and "Recommended" -setup, you can add cloud storage providers like Google GCS, Amazon AWS S3, etc. By default, TechDocs -uses the local filesystem of the `techdocs-backend` plugin in the "Basic" setup. And in the recommended setup, -having one of the cloud storage is a prerequisite. Read more on the TechDocs Architecture documentation page. +In the [TechDocs architecture](./architecture.md) you have the option to choose +where you want to store the Generated static files which TechDocs uses to render +documentation. In both the "Basic" and "Recommended" setup, you can add cloud +storage providers like Google GCS, Amazon AWS S3, etc. By default, TechDocs uses +the local filesystem of the `techdocs-backend` plugin in the "Basic" setup. And +in the recommended setup, having one of the cloud storage is a prerequisite. +Read more on the TechDocs Architecture documentation page. -On this page you you can read how to enable them. +On this page you can read how to enable them. ## Configuring Google GCS Bucket with TechDocs -Follow the [official Google Cloud documentation](https://googleapis.dev/nodejs/storage/latest/index.html#quickstart) for the latest instructions -on the following steps involving GCP. +Follow the +[official Google Cloud documentation](https://googleapis.dev/nodejs/storage/latest/index.html#quickstart) +for the latest instructions on the following steps involving GCP. **1. Set `techdocs.publisher.type` config in your `app-config.yaml`** @@ -29,7 +32,8 @@ techdocs: **2. GCP (Google Cloud Platform) Project** -Create or choose a dedicated GCP project. Set `techdocs.publisher.google.projectId` to the project ID. +Create or choose a dedicated GCP project. Set +`techdocs.publisher.google.projectId` to the project ID. ```yaml techdocs: @@ -41,13 +45,20 @@ techdocs: **3. Service account API key** -Create a new Service Account and a key associated with it. In roles of the service account, use "Storage Admin". +Create a new Service Account and a key associated with it. In roles of the +service account, use "Storage Admin". -If you want to create a custom role, make sure to include both `get` and `create` permissions for both "Objects" and "Buckets". See https://cloud.google.com/storage/docs/access-control/iam-permissions +If you want to create a custom role, make sure to include both `get` and +`create` permissions for both "Objects" and "Buckets". See +https://cloud.google.com/storage/docs/access-control/iam-permissions -A service account can have many keys. Open your newly created account's page (in IAM & Admin console), and create a new key. Use JSON format for the key. +A service account can have many keys. Open your newly created account's page (in +IAM & Admin console), and create a new key. Use JSON format for the key. -A `.json` file will be downloaded. This is the secret key TechDocs will use to make API calls. Make it available in your Backstage server and/or your local development server and set it in the app config `techdocs.publisher.google.pathToKey`. +A `.json` file will be downloaded. This is the secret +key TechDocs will use to make API calls. Make it available in your Backstage +server and/or your local development server and set it in the app config +`techdocs.publisher.google.pathToKey`. ```yaml techdocs: @@ -60,8 +71,9 @@ techdocs: **4. GCS Bucket** -Create a dedicated bucket for TechDocs sites. techdocs-backend will publish documentation to this bucket. -TechDocs will fetch files from here to serve documentation in Backstage. +Create a dedicated bucket for TechDocs sites. techdocs-backend will publish +documentation to this bucket. TechDocs will fetch files from here to serve +documentation in Backstage. Set the name of the bucket to `techdocs.publisher @@ -77,4 +89,5 @@ techdocs: **5. That's it!** -Your Backstage app is now ready to use Google Cloud Storage for TechDocs, to store the static generated documentation files. +Your Backstage app is now ready to use Google Cloud Storage for TechDocs, to +store the static generated documentation files. diff --git a/yarn.lock b/yarn.lock index ef90303927..f8662a26bc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1316,7 +1316,7 @@ to-fast-properties "^2.0.0" "@backstage/catalog-model@^0.2.0": - version "0.3.1" + version "0.4.0" dependencies: "@backstage/config" "^0.1.1" "@types/json-schema" "^7.0.5" @@ -1326,6 +1326,132 @@ uuid "^8.0.0" yup "^0.29.3" +"@backstage/catalog-model@^0.3.0": + version "0.4.0" + dependencies: + "@backstage/config" "^0.1.1" + "@types/json-schema" "^7.0.5" + "@types/yup" "^0.29.8" + json-schema "^0.2.5" + lodash "^4.17.15" + uuid "^8.0.0" + yup "^0.29.3" + +"@backstage/catalog-model@^0.3.1": + version "0.3.1" + resolved "https://registry.npmjs.org/@backstage/catalog-model/-/catalog-model-0.3.1.tgz#45d08e2f333c9c566b2bf2629fd707fe989bb404" + integrity sha512-9XhV7c4rmVW+Yzj2PiwTQ7DsegWGB3C4ELsDRExuEVZONdqNcC02cyJtrt3fT5F31ZS3tHkB9bMUymFOBLqUSA== + dependencies: + "@backstage/config" "^0.1.1" + "@types/json-schema" "^7.0.5" + "@types/yup" "^0.29.8" + json-schema "^0.2.5" + lodash "^4.17.15" + uuid "^8.0.0" + yup "^0.29.3" + +"@backstage/cli@^0.3.1": + version "0.3.2" + resolved "https://registry.npmjs.org/@backstage/cli/-/cli-0.3.2.tgz#e1972beb30cd7cd7078ed405d810357449a8c9c5" + integrity sha512-JxVOD0RJivHX38dmRZuvk2atePVXcsv3jw5SC/l4S/dmGdnDKIbp21A5DNNzmHqxCJcnArtO54nFuss2pNgc1Q== + dependencies: + "@backstage/cli-common" "^0.1.1" + "@backstage/config" "^0.1.1" + "@backstage/config-loader" "^0.3.0" + "@hot-loader/react-dom" "^16.13.0" + "@lerna/package-graph" "^3.18.5" + "@lerna/project" "^3.18.0" + "@rollup/plugin-commonjs" "^16.0.0" + "@rollup/plugin-json" "^4.0.2" + "@rollup/plugin-node-resolve" "^9.0.0" + "@rollup/plugin-yaml" "^2.1.1" + "@spotify/eslint-config-base" "^9.0.0" + "@spotify/eslint-config-react" "^9.0.0" + "@spotify/eslint-config-typescript" "^9.0.0" + "@sucrase/webpack-loader" "^2.0.0" + "@svgr/plugin-jsx" "5.4.x" + "@svgr/plugin-svgo" "5.4.x" + "@svgr/rollup" "5.4.x" + "@svgr/webpack" "5.4.x" + "@types/start-server-webpack-plugin" "^2.2.0" + "@types/webpack-env" "^1.15.2" + "@types/webpack-node-externals" "^2.5.0" + "@typescript-eslint/eslint-plugin" "^v3.10.1" + "@typescript-eslint/parser" "^v3.10.1" + "@yarnpkg/lockfile" "^1.1.0" + bfj "^7.0.2" + chalk "^4.0.0" + chokidar "^3.3.1" + commander "^6.1.0" + css-loader "^3.5.3" + dashify "^2.0.0" + diff "^4.0.2" + esbuild "^0.7.7" + eslint "^7.1.0" + eslint-config-prettier "^6.0.0" + eslint-formatter-friendly "^7.0.0" + eslint-plugin-import "^2.20.2" + eslint-plugin-jest "^24.1.0" + eslint-plugin-jsx-a11y "^6.2.1" + eslint-plugin-monorepo "^0.2.1" + eslint-plugin-react "^7.12.4" + eslint-plugin-react-hooks "^4.0.0" + fork-ts-checker-webpack-plugin "^4.0.5" + fs-extra "^9.0.0" + handlebars "^4.7.3" + html-webpack-plugin "^4.3.0" + inquirer "^7.0.4" + jest "^26.0.1" + jest-css-modules "^2.1.0" + jest-esm-transformer "^1.0.0" + lodash "^4.17.19" + mini-css-extract-plugin "^0.9.0" + ora "^4.0.3" + raw-loader "^4.0.1" + react "^16.0.0" + react-dev-utils "^10.2.1" + react-hot-loader "^4.12.21" + recursive-readdir "^2.2.2" + replace-in-file "^6.0.0" + rollup "2.33.x" + rollup-plugin-dts "1.4.13" + rollup-plugin-esbuild "2.3.x" + rollup-plugin-peer-deps-external "^2.2.2" + rollup-plugin-postcss "^3.1.1" + rollup-plugin-typescript2 "^0.27.3" + rollup-pluginutils "^2.8.2" + semver "^7.3.2" + start-server-webpack-plugin "^2.2.5" + style-loader "^1.2.1" + sucrase "^3.16.0" + tar "^6.0.1" + terser-webpack-plugin "^1.4.3" + ts-jest "^26.4.3" + ts-loader "^7.0.4" + typescript "^4.0.3" + url-loader "^4.1.0" + webpack "^4.41.6" + webpack-dev-server "^3.11.0" + webpack-node-externals "^2.5.0" + yaml "^1.10.0" + yml-loader "^2.1.0" + yn "^4.0.0" + +"@backstage/config-loader@^0.3.0": + version "0.3.0" + resolved "https://registry.npmjs.org/@backstage/config-loader/-/config-loader-0.3.0.tgz#6e648b415cd2dda1b49e9b6da7f8554fd162efb6" + integrity sha512-XFcoDJ/+1i4v8A1rnZ43LouuxVIbQqhTcwCK+J5kY54N7Qi+ialnCVSpj3t9U3wI51JfztG55uIL5fEsCAu27g== + dependencies: + "@backstage/cli-common" "^0.1.1" + "@backstage/config" "^0.1.1" + ajv "^6.12.5" + fs-extra "^9.0.0" + json-schema "^0.2.5" + json-schema-merge-allof "^0.7.0" + typescript-json-schema "^0.43.0" + yaml "^1.9.2" + yup "^0.29.3" + "@bcoe/v8-coverage@^0.2.3": version "0.2.3" resolved "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" @@ -5267,15 +5393,6 @@ "@types/qs" "*" "@types/range-parser" "*" -"@types/express-serve-static-core@^4.17.14": - version "4.17.14" - resolved "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.14.tgz#cabf91debeeb3cb04b798e2cff908864e89b6106" - integrity sha512-uFTLwu94TfUFMToXNgRZikwPuZdOtDgs3syBtAIr/OXorL1kJqUJT9qCLnRZ5KBOWfZQikQ2xKgR2tnDj1OgDA== - dependencies: - "@types/node" "*" - "@types/qs" "*" - "@types/range-parser" "*" - "@types/express-session@^1.17.2": version "1.17.2" resolved "https://registry.npmjs.org/@types/express-session/-/express-session-1.17.2.tgz#ed6a36dd9f267c7fef86004f653bfb9b5cea3c21" @@ -11319,6 +11436,11 @@ es6-weak-map@^2.0.2: es6-iterator "^2.0.3" es6-symbol "^3.1.1" +esbuild@^0.7.7: + version "0.7.22" + resolved "https://registry.npmjs.org/esbuild/-/esbuild-0.7.22.tgz#9149b903f8128b7c45a754046c24199d76bbe08e" + integrity sha512-B43SYg8LGWYTCv9Gs0RnuLNwjzpuWOoCaZHTWEDEf5AfrnuDMerPVMdCEu7xOdhFvQ+UqfP2MGU9lxEy0JzccA== + esbuild@^0.8.16: version "0.8.16" resolved "https://registry.npmjs.org/esbuild/-/esbuild-0.8.16.tgz#8ae34b15d938e8b8b5ac2459414fe3e7fd7dd6b2" @@ -12928,7 +13050,7 @@ glob-to-regexp@^0.3.0: resolved "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz#8c5a1494d2066c570cc3bfe4496175acc4d502ab" integrity sha1-jFoUlNIGbFcMw7/kSWF1rMTVAqs= -glob@7.1.6, glob@^7.0.0, glob@^7.0.3, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6: +glob@7.1.6, glob@^7.0.0, glob@^7.0.3, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6, glob@~7.1.6: version "7.1.6" resolved "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz#141f33b81a7c2492e125594307480c46679278a6" integrity sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA== @@ -15811,6 +15933,14 @@ jws@^3.2.2: jwa "^1.4.1" safe-buffer "^5.0.1" +jws@^4.0.0: + version "4.0.0" + resolved "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz#2d4e8cf6a318ffaa12615e9dec7e86e6c97310f4" + integrity sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg== + dependencies: + jwa "^2.0.0" + safe-buffer "^5.0.1" + jwt-decode@*, jwt-decode@^3.1.0: version "3.1.2" resolved "https://registry.npmjs.org/jwt-decode/-/jwt-decode-3.1.2.tgz#3fb319f3675a2df0c2895c8f5e9fa4b67b04ed59" @@ -21512,6 +21642,13 @@ rollup-plugin-dts@1.4.13: optionalDependencies: "@babel/code-frame" "^7.10.4" +rollup-plugin-esbuild@2.3.x: + version "2.3.0" + resolved "https://registry.npmjs.org/rollup-plugin-esbuild/-/rollup-plugin-esbuild-2.3.0.tgz#7c107d4508af80d30966a148b306cb7d5b36b258" + integrity sha512-lNbWDRaClwXauaIybuaiOBRWgxcp3GkZu0UUix9yH/OHFgVohKmzlU0NtNbvfbxAcfEooyuBRAv6YCaJl1Mbpw== + dependencies: + "@rollup/pluginutils" "^3.1.0" + rollup-plugin-esbuild@2.6.x: version "2.6.0" resolved "https://registry.npmjs.org/rollup-plugin-esbuild/-/rollup-plugin-esbuild-2.6.0.tgz#80336399b113a179ccb2af5bdf7c03f061f37146" @@ -23901,6 +24038,17 @@ typedarray@^0.0.6: resolved "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c= +typescript-json-schema@^0.43.0: + version "0.43.0" + resolved "https://registry.npmjs.org/typescript-json-schema/-/typescript-json-schema-0.43.0.tgz#8bd9c832f1f15f006ff933907ce192222fdfd92f" + integrity sha512-4c9IMlIlHYJiQtzL1gh2nIPJEjBgJjDUs50gsnnc+GFyDSK1oFM3uQIBSVosiuA/4t6LSAXDS9vTdqbQC6EcgA== + dependencies: + "@types/json-schema" "^7.0.5" + glob "~7.1.6" + json-stable-stringify "^1.0.1" + typescript "~4.0.2" + yargs "^15.4.1" + typescript-json-schema@^0.45.0: version "0.45.0" resolved "https://registry.npmjs.org/typescript-json-schema/-/typescript-json-schema-0.45.0.tgz#2f244e99518e589a442ee5f9c1d2c85a1ec7dc84" @@ -23917,6 +24065,11 @@ typescript@^4.0.3, typescript@^4.1.2: resolved "https://registry.npmjs.org/typescript/-/typescript-4.1.2.tgz#6369ef22516fe5e10304aae5a5c4862db55380e9" integrity sha512-thGloWsGH3SOxv1SoY7QojKi0tc+8FnOmiarEGMbd/lar7QOEd3hvlx3Fp5y6FlDUGl9L+pd4n2e+oToGMmhRQ== +typescript@~4.0.2: + version "4.0.5" + resolved "https://registry.npmjs.org/typescript/-/typescript-4.0.5.tgz#ae9dddfd1069f1cb5beb3ef3b2170dd7c1332389" + integrity sha512-ywmr/VrTVCmNTJ6iV2LwIrfG1P+lv6luD8sUJs+2eI9NLGigaN+nUQc13iHqisq7bra9lnmUSYqbJvegraBOPQ== + ua-parser-js@^0.7.18: version "0.7.21" resolved "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.21.tgz#853cf9ce93f642f67174273cc34565ae6f308777" From 2cbad5c19df83e3a773f4d814ee66536ddb9cde6 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Fri, 4 Dec 2020 19:47:27 +0100 Subject: [PATCH 008/188] techdocs-common: Bump @backstage/cli version --- packages/techdocs-common/package.json | 2 +- yarn.lock | 132 +------------------------- 2 files changed, 2 insertions(+), 132 deletions(-) diff --git a/packages/techdocs-common/package.json b/packages/techdocs-common/package.json index 66821a0043..4f9747f64f 100644 --- a/packages/techdocs-common/package.json +++ b/packages/techdocs-common/package.json @@ -54,6 +54,6 @@ "winston": "^3.2.1" }, "devDependencies": { - "@backstage/cli": "^0.3.1" + "@backstage/cli": "^0.4.0" } } diff --git a/yarn.lock b/yarn.lock index f8662a26bc..36fa5d6f96 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1350,108 +1350,6 @@ uuid "^8.0.0" yup "^0.29.3" -"@backstage/cli@^0.3.1": - version "0.3.2" - resolved "https://registry.npmjs.org/@backstage/cli/-/cli-0.3.2.tgz#e1972beb30cd7cd7078ed405d810357449a8c9c5" - integrity sha512-JxVOD0RJivHX38dmRZuvk2atePVXcsv3jw5SC/l4S/dmGdnDKIbp21A5DNNzmHqxCJcnArtO54nFuss2pNgc1Q== - dependencies: - "@backstage/cli-common" "^0.1.1" - "@backstage/config" "^0.1.1" - "@backstage/config-loader" "^0.3.0" - "@hot-loader/react-dom" "^16.13.0" - "@lerna/package-graph" "^3.18.5" - "@lerna/project" "^3.18.0" - "@rollup/plugin-commonjs" "^16.0.0" - "@rollup/plugin-json" "^4.0.2" - "@rollup/plugin-node-resolve" "^9.0.0" - "@rollup/plugin-yaml" "^2.1.1" - "@spotify/eslint-config-base" "^9.0.0" - "@spotify/eslint-config-react" "^9.0.0" - "@spotify/eslint-config-typescript" "^9.0.0" - "@sucrase/webpack-loader" "^2.0.0" - "@svgr/plugin-jsx" "5.4.x" - "@svgr/plugin-svgo" "5.4.x" - "@svgr/rollup" "5.4.x" - "@svgr/webpack" "5.4.x" - "@types/start-server-webpack-plugin" "^2.2.0" - "@types/webpack-env" "^1.15.2" - "@types/webpack-node-externals" "^2.5.0" - "@typescript-eslint/eslint-plugin" "^v3.10.1" - "@typescript-eslint/parser" "^v3.10.1" - "@yarnpkg/lockfile" "^1.1.0" - bfj "^7.0.2" - chalk "^4.0.0" - chokidar "^3.3.1" - commander "^6.1.0" - css-loader "^3.5.3" - dashify "^2.0.0" - diff "^4.0.2" - esbuild "^0.7.7" - eslint "^7.1.0" - eslint-config-prettier "^6.0.0" - eslint-formatter-friendly "^7.0.0" - eslint-plugin-import "^2.20.2" - eslint-plugin-jest "^24.1.0" - eslint-plugin-jsx-a11y "^6.2.1" - eslint-plugin-monorepo "^0.2.1" - eslint-plugin-react "^7.12.4" - eslint-plugin-react-hooks "^4.0.0" - fork-ts-checker-webpack-plugin "^4.0.5" - fs-extra "^9.0.0" - handlebars "^4.7.3" - html-webpack-plugin "^4.3.0" - inquirer "^7.0.4" - jest "^26.0.1" - jest-css-modules "^2.1.0" - jest-esm-transformer "^1.0.0" - lodash "^4.17.19" - mini-css-extract-plugin "^0.9.0" - ora "^4.0.3" - raw-loader "^4.0.1" - react "^16.0.0" - react-dev-utils "^10.2.1" - react-hot-loader "^4.12.21" - recursive-readdir "^2.2.2" - replace-in-file "^6.0.0" - rollup "2.33.x" - rollup-plugin-dts "1.4.13" - rollup-plugin-esbuild "2.3.x" - rollup-plugin-peer-deps-external "^2.2.2" - rollup-plugin-postcss "^3.1.1" - rollup-plugin-typescript2 "^0.27.3" - rollup-pluginutils "^2.8.2" - semver "^7.3.2" - start-server-webpack-plugin "^2.2.5" - style-loader "^1.2.1" - sucrase "^3.16.0" - tar "^6.0.1" - terser-webpack-plugin "^1.4.3" - ts-jest "^26.4.3" - ts-loader "^7.0.4" - typescript "^4.0.3" - url-loader "^4.1.0" - webpack "^4.41.6" - webpack-dev-server "^3.11.0" - webpack-node-externals "^2.5.0" - yaml "^1.10.0" - yml-loader "^2.1.0" - yn "^4.0.0" - -"@backstage/config-loader@^0.3.0": - version "0.3.0" - resolved "https://registry.npmjs.org/@backstage/config-loader/-/config-loader-0.3.0.tgz#6e648b415cd2dda1b49e9b6da7f8554fd162efb6" - integrity sha512-XFcoDJ/+1i4v8A1rnZ43LouuxVIbQqhTcwCK+J5kY54N7Qi+ialnCVSpj3t9U3wI51JfztG55uIL5fEsCAu27g== - dependencies: - "@backstage/cli-common" "^0.1.1" - "@backstage/config" "^0.1.1" - ajv "^6.12.5" - fs-extra "^9.0.0" - json-schema "^0.2.5" - json-schema-merge-allof "^0.7.0" - typescript-json-schema "^0.43.0" - yaml "^1.9.2" - yup "^0.29.3" - "@bcoe/v8-coverage@^0.2.3": version "0.2.3" resolved "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" @@ -11436,11 +11334,6 @@ es6-weak-map@^2.0.2: es6-iterator "^2.0.3" es6-symbol "^3.1.1" -esbuild@^0.7.7: - version "0.7.22" - resolved "https://registry.npmjs.org/esbuild/-/esbuild-0.7.22.tgz#9149b903f8128b7c45a754046c24199d76bbe08e" - integrity sha512-B43SYg8LGWYTCv9Gs0RnuLNwjzpuWOoCaZHTWEDEf5AfrnuDMerPVMdCEu7xOdhFvQ+UqfP2MGU9lxEy0JzccA== - esbuild@^0.8.16: version "0.8.16" resolved "https://registry.npmjs.org/esbuild/-/esbuild-0.8.16.tgz#8ae34b15d938e8b8b5ac2459414fe3e7fd7dd6b2" @@ -13050,7 +12943,7 @@ glob-to-regexp@^0.3.0: resolved "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz#8c5a1494d2066c570cc3bfe4496175acc4d502ab" integrity sha1-jFoUlNIGbFcMw7/kSWF1rMTVAqs= -glob@7.1.6, glob@^7.0.0, glob@^7.0.3, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6, glob@~7.1.6: +glob@7.1.6, glob@^7.0.0, glob@^7.0.3, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, glob@^7.1.6: version "7.1.6" resolved "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz#141f33b81a7c2492e125594307480c46679278a6" integrity sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA== @@ -21642,13 +21535,6 @@ rollup-plugin-dts@1.4.13: optionalDependencies: "@babel/code-frame" "^7.10.4" -rollup-plugin-esbuild@2.3.x: - version "2.3.0" - resolved "https://registry.npmjs.org/rollup-plugin-esbuild/-/rollup-plugin-esbuild-2.3.0.tgz#7c107d4508af80d30966a148b306cb7d5b36b258" - integrity sha512-lNbWDRaClwXauaIybuaiOBRWgxcp3GkZu0UUix9yH/OHFgVohKmzlU0NtNbvfbxAcfEooyuBRAv6YCaJl1Mbpw== - dependencies: - "@rollup/pluginutils" "^3.1.0" - rollup-plugin-esbuild@2.6.x: version "2.6.0" resolved "https://registry.npmjs.org/rollup-plugin-esbuild/-/rollup-plugin-esbuild-2.6.0.tgz#80336399b113a179ccb2af5bdf7c03f061f37146" @@ -24038,17 +23924,6 @@ typedarray@^0.0.6: resolved "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c= -typescript-json-schema@^0.43.0: - version "0.43.0" - resolved "https://registry.npmjs.org/typescript-json-schema/-/typescript-json-schema-0.43.0.tgz#8bd9c832f1f15f006ff933907ce192222fdfd92f" - integrity sha512-4c9IMlIlHYJiQtzL1gh2nIPJEjBgJjDUs50gsnnc+GFyDSK1oFM3uQIBSVosiuA/4t6LSAXDS9vTdqbQC6EcgA== - dependencies: - "@types/json-schema" "^7.0.5" - glob "~7.1.6" - json-stable-stringify "^1.0.1" - typescript "~4.0.2" - yargs "^15.4.1" - typescript-json-schema@^0.45.0: version "0.45.0" resolved "https://registry.npmjs.org/typescript-json-schema/-/typescript-json-schema-0.45.0.tgz#2f244e99518e589a442ee5f9c1d2c85a1ec7dc84" @@ -24065,11 +23940,6 @@ typescript@^4.0.3, typescript@^4.1.2: resolved "https://registry.npmjs.org/typescript/-/typescript-4.1.2.tgz#6369ef22516fe5e10304aae5a5c4862db55380e9" integrity sha512-thGloWsGH3SOxv1SoY7QojKi0tc+8FnOmiarEGMbd/lar7QOEd3hvlx3Fp5y6FlDUGl9L+pd4n2e+oToGMmhRQ== -typescript@~4.0.2: - version "4.0.5" - resolved "https://registry.npmjs.org/typescript/-/typescript-4.0.5.tgz#ae9dddfd1069f1cb5beb3ef3b2170dd7c1332389" - integrity sha512-ywmr/VrTVCmNTJ6iV2LwIrfG1P+lv6luD8sUJs+2eI9NLGigaN+nUQc13iHqisq7bra9lnmUSYqbJvegraBOPQ== - ua-parser-js@^0.7.18: version "0.7.21" resolved "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.21.tgz#853cf9ce93f642f67174273cc34565ae6f308777" From d073ba4f16b298941400de40c7fa1f2b6e8b55c3 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Fri, 4 Dec 2020 19:59:15 +0100 Subject: [PATCH 009/188] techdocs-common: Bump @types/dockerode version --- packages/techdocs-common/package.json | 2 +- yarn.lock | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/packages/techdocs-common/package.json b/packages/techdocs-common/package.json index 4f9747f64f..ac7036e509 100644 --- a/packages/techdocs-common/package.json +++ b/packages/techdocs-common/package.json @@ -40,7 +40,7 @@ "@backstage/catalog-model": "^0.3.1", "@backstage/config": "^0.1.1", "@google-cloud/storage": "^5.6.0", - "@types/dockerode": "^2.5.34", + "@types/dockerode": "^3.2.1", "@types/klaw": "^3.0.1", "cross-fetch": "^3.0.6", "dockerode": "^3.2.1", diff --git a/yarn.lock b/yarn.lock index 1e6be595f2..b6f1463f6e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5240,13 +5240,6 @@ resolved "https://registry.npmjs.org/@types/diff/-/diff-4.0.2.tgz#2e9bb89f9acc3ab0108f0f3dc4dbdcf2fff8a99c" integrity sha512-mIenTfsIe586/yzsyfql69KRnA75S8SVXQbTLpDejRrjH0QSJcpu3AUOi/Vjnt9IOsXKxPhJfGpQUNMueIU1fQ== -"@types/dockerode@^2.5.34": - version "2.5.34" - resolved "https://registry.npmjs.org/@types/dockerode/-/dockerode-2.5.34.tgz#9adb884f7cc6c012a6eb4b2ad794cc5d01439959" - integrity sha512-LcbLGcvcBwBAvjH9UrUI+4qotY+A5WCer5r43DR5XHv2ZIEByNXFdPLo1XxR+v/BjkGjlggW8qUiXuVEhqfkpA== - dependencies: - "@types/node" "*" - "@types/dockerode@^3.2.1": version "3.2.1" resolved "https://registry.npmjs.org/@types/dockerode/-/dockerode-3.2.1.tgz#f713a8f6f1017c227845ab33239383da721207b9" From 50a0911364b43fc7472ff2a57094aefe245fc577 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Fri, 4 Dec 2020 20:14:23 +0100 Subject: [PATCH 010/188] techdocs-common: Export @types/express from dependencies --- packages/techdocs-common/package.json | 7 ++++--- yarn.lock | 13 ------------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/packages/techdocs-common/package.json b/packages/techdocs-common/package.json index ac7036e509..7cd53c69c1 100644 --- a/packages/techdocs-common/package.json +++ b/packages/techdocs-common/package.json @@ -36,12 +36,12 @@ "url": "https://github.com/backstage/backstage/issues" }, "dependencies": { - "@backstage/backend-common": "^0.3.2", - "@backstage/catalog-model": "^0.3.1", + "@backstage/backend-common": "^0.3.3", + "@backstage/catalog-model": "^0.4.0", "@backstage/config": "^0.1.1", "@google-cloud/storage": "^5.6.0", "@types/dockerode": "^3.2.1", - "@types/klaw": "^3.0.1", + "@types/express": "^4.17.6", "cross-fetch": "^3.0.6", "dockerode": "^3.2.1", "express": "^4.17.1", @@ -54,6 +54,7 @@ "winston": "^3.2.1" }, "devDependencies": { + "@types/klaw": "^3.0.1", "@backstage/cli": "^0.4.0" } } diff --git a/yarn.lock b/yarn.lock index b6f1463f6e..3e94d8d9b0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1337,19 +1337,6 @@ uuid "^8.0.0" yup "^0.29.3" -"@backstage/catalog-model@^0.3.1": - version "0.3.1" - resolved "https://registry.npmjs.org/@backstage/catalog-model/-/catalog-model-0.3.1.tgz#45d08e2f333c9c566b2bf2629fd707fe989bb404" - integrity sha512-9XhV7c4rmVW+Yzj2PiwTQ7DsegWGB3C4ELsDRExuEVZONdqNcC02cyJtrt3fT5F31ZS3tHkB9bMUymFOBLqUSA== - dependencies: - "@backstage/config" "^0.1.1" - "@types/json-schema" "^7.0.5" - "@types/yup" "^0.29.8" - json-schema "^0.2.5" - lodash "^4.17.15" - uuid "^8.0.0" - yup "^0.29.3" - "@bcoe/v8-coverage@^0.2.3": version "0.2.3" resolved "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" From 3a50df5ee3ac4f655424c8f01274ade633936940 Mon Sep 17 00:00:00 2001 From: Ryan Vazquez Date: Fri, 4 Dec 2020 15:31:07 -0500 Subject: [PATCH 011/188] convert duration + billing date to intervals --- plugins/cost-insights/src/api/CostInsightsApi.ts | 11 ++--------- .../components/ProductInsights/ProductInsights.tsx | 8 +++----- plugins/cost-insights/src/index.ts | 1 - 3 files changed, 5 insertions(+), 15 deletions(-) diff --git a/plugins/cost-insights/src/api/CostInsightsApi.ts b/plugins/cost-insights/src/api/CostInsightsApi.ts index 9baad5ffab..28e72bccc6 100644 --- a/plugins/cost-insights/src/api/CostInsightsApi.ts +++ b/plugins/cost-insights/src/api/CostInsightsApi.ts @@ -18,7 +18,6 @@ import { createApiRef } from '@backstage/core'; import { Alert, Cost, - Duration, Entity, Group, Project, @@ -38,15 +37,9 @@ export type ProductInsightsOptions = { group: string; /** - * A time duration, such as P1M. See the Duration type for a detailed explanation - * of how the durations are interpreted in Cost Insights. + * An ISO 8601 repeating interval string, such as R2/P3M/2020-09-01 */ - duration: Duration; - - /** - * The most current date for which billing data is complete, in YYYY-MM-DD format. - */ - lastCompleteBillingDate: string; + intervals: string; /** * (optional) The project id from getGroupProjects or query parameters diff --git a/plugins/cost-insights/src/components/ProductInsights/ProductInsights.tsx b/plugins/cost-insights/src/components/ProductInsights/ProductInsights.tsx index 52abe3fd8b..0ef35bf04c 100644 --- a/plugins/cost-insights/src/components/ProductInsights/ProductInsights.tsx +++ b/plugins/cost-insights/src/components/ProductInsights/ProductInsights.tsx @@ -21,7 +21,7 @@ import { useApi } from '@backstage/core'; import { costInsightsApiRef } from '../../api'; import { ProductInsightsCardList } from '../ProductInsightsCard/ProductInsightsCardList'; import { Duration, Entity, Maybe, Product } from '../../types'; -import { DEFAULT_DURATION } from '../../utils/duration'; +import { intervalsOf, DEFAULT_DURATION } from '../../utils/duration'; import { DefaultLoadingAction, initialStatesOf, @@ -75,8 +75,7 @@ export const ProductInsights = ({ group: group, project: project, product: product.kind, - duration: duration, - lastCompleteBillingDate: lastCompleteBillingDate, + intervals: intervalsOf(duration, lastCompleteBillingDate), }); }, [client, group, project, lastCompleteBillingDate], @@ -97,8 +96,7 @@ export const ProductInsights = ({ group: group, project: project, product: product.kind, - duration: DEFAULT_DURATION, - lastCompleteBillingDate: lastCompleteBillingDate, + intervals: intervalsOf(DEFAULT_DURATION, lastCompleteBillingDate), }), ), ).then(settledResponseOf); diff --git a/plugins/cost-insights/src/index.ts b/plugins/cost-insights/src/index.ts index 28bdb1a8b3..7cd0456ad1 100644 --- a/plugins/cost-insights/src/index.ts +++ b/plugins/cost-insights/src/index.ts @@ -21,5 +21,4 @@ export * from './components'; export { useCurrency } from './hooks'; export * from './types'; export * from './utils/tests'; -export * from './utils/duration'; export * from './utils/alerts'; From cd26fb65778693662a9298ba0047d29fbfe41fe9 Mon Sep 17 00:00:00 2001 From: Ryan Vazquez Date: Fri, 4 Dec 2020 15:31:38 -0500 Subject: [PATCH 012/188] supress recharts react warnings in test --- .../ProjectGrowthAlertCard/ProjectGrowthAlertCard.test.tsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/cost-insights/src/components/ProjectGrowthAlertCard/ProjectGrowthAlertCard.test.tsx b/plugins/cost-insights/src/components/ProjectGrowthAlertCard/ProjectGrowthAlertCard.test.tsx index db3f2f9bf6..0e24f042bc 100644 --- a/plugins/cost-insights/src/components/ProjectGrowthAlertCard/ProjectGrowthAlertCard.test.tsx +++ b/plugins/cost-insights/src/components/ProjectGrowthAlertCard/ProjectGrowthAlertCard.test.tsx @@ -27,6 +27,9 @@ import { AlertCost } from '../../types'; import { defaultCurrencies } from '../../utils/currency'; import { findAlways } from '../../utils/assert'; +// suppress recharts componentDidUpdate deprecation warnings +jest.spyOn(console, 'warn').mockImplementation(() => {}); + const engineers = findAlways(defaultCurrencies, c => c.kind === null); const MockProject = 'test-project-1'; From d6e8099ed58b2558b707edda62bb37b2d0f03e10 Mon Sep 17 00:00:00 2001 From: Ryan Vazquez Date: Fri, 4 Dec 2020 15:35:59 -0500 Subject: [PATCH 013/188] changeset --- .changeset/cost-insights-wet-walls-jog.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/cost-insights-wet-walls-jog.md diff --git a/.changeset/cost-insights-wet-walls-jog.md b/.changeset/cost-insights-wet-walls-jog.md new file mode 100644 index 0000000000..8de3037f19 --- /dev/null +++ b/.changeset/cost-insights-wet-walls-jog.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-cost-insights': minor +--- + +convert duration + last completed billing date to intervals From 07e7c201e8b93ca2bc9150026d481847495bf997 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Fri, 4 Dec 2020 21:46:55 +0100 Subject: [PATCH 014/188] techdocs-backend: Wait for some time before published files show up in storage --- .../techdocs-backend/src/service/router.ts | 38 ++++++++++++++----- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index 7f6d7c69bc..6b0fd55ba7 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -123,10 +123,7 @@ export async function createRouter({ // techdocs-backend will only try to build documentation for an entity if techdocs.docsBuilder is set to 'local' // If set to 'ci', it will only try to fetch and assume that that CI/CD pipeline of the repository hosting the // entity's documentation is responsible for building and publishing documentation to the storage provider. - const shouldBuildDocs = - config.getString('techdocs.docsBuilder') === 'local'; - - if (shouldBuildDocs) { + if (config.getString('techdocs.docsBuilder') === 'local') { const docsBuilder = new DocsBuilder({ preparers, generators, @@ -140,17 +137,38 @@ export async function createRouter({ await docsBuilder.build(); } } else if (publisherType === 'google_gcs') { + // This block should be valid for all external storage implementations. So no need to duplicate in future, + // add the publisher type in the list here. if (!(await publisher.hasDocsBeenGenerated(entity))) { logger.info( - 'Did not find generated docs files for the entity. Building docs.', + 'No pre-generated documentation files found for the entity in the storage. Building docs...', ); await docsBuilder.build(); - } else { - logger.info( - 'Found pre-generated docs for this entity. Serving them.', - ); - // TODO: When to re-trigger build for cache invalidation? + // With a maximum of ~5 seconds wait, check if the files got published and if docs will be fetched + // on the user's page. If not, respond with a message asking them to check back later. + // The delay here is to make sure GCS registers newly uploaded files which is usually <1 second + for (let attempt = 0; attempt < 5; attempt++) { + if (await publisher.hasDocsBeenGenerated(entity)) { + break; + } + await new Promise(r => setTimeout(r, 1000)); + logger.error( + 'Published files are taking longer to show up in storage. Something went wrong.', + ); + res + .status(408) + .send( + 'Sorry! It is taking longer for the generated docs to show up up in storage. Check back later.', + ); + return; + } } + + logger.info('Found pre-generated docs for this entity. Serving them.'); + // TODO: re-trigger build for cache invalidation. + // Compare the date modified of the requested file on storage and compare it against + // the last modified or last commit timestamp in the repository. + // Without this, docs will not be re-built once they have been generated. } } From 11340fb2f66826a5a2df22a618947fad5c0fd566 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Fri, 4 Dec 2020 22:09:38 +0100 Subject: [PATCH 015/188] techdocs: Remove spinners MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spinners cause unnecessary UX distraction. Case 1 (when docs are built and are to be served): Spinners appear for a split second before the name of site shows up. This unnecessarily distracts eyes because spinners increase the size of the Header. A dot (.) would do fine. Definitely more can be done. Case 2 (when docs are being generated): There is already a linear progress bar (which is recommended in Storybook). Spinners are not good. 🤷 --- plugins/techdocs/src/reader/components/TechDocsPageHeader.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/techdocs/src/reader/components/TechDocsPageHeader.tsx b/plugins/techdocs/src/reader/components/TechDocsPageHeader.tsx index d7a5a3c48f..ac9d21d319 100644 --- a/plugins/techdocs/src/reader/components/TechDocsPageHeader.tsx +++ b/plugins/techdocs/src/reader/components/TechDocsPageHeader.tsx @@ -16,7 +16,6 @@ import React from 'react'; import { AsyncState } from 'react-use/lib/useAsync'; -import { CircularProgress } from '@material-ui/core'; import CodeIcon from '@material-ui/icons/Code'; import { EntityName } from '@backstage/catalog-model'; import { Header, HeaderLabel, Link } from '@backstage/core'; @@ -86,7 +85,7 @@ export const TechDocsPageHeader = ({ return (
} + title={siteName ? siteName : '.'} pageTitleOverride={siteName || name} subtitle={ siteDescription && siteDescription !== 'None' ? siteDescription : '' From c91fd25a532a830195540359719830580563ef62 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Fri, 4 Dec 2020 22:37:14 +0100 Subject: [PATCH 016/188] techdocs-backend: Fix logic to determine if docs have been published --- plugins/techdocs-backend/src/service/router.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index 6b0fd55ba7..76778f291f 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -147,11 +147,15 @@ export async function createRouter({ // With a maximum of ~5 seconds wait, check if the files got published and if docs will be fetched // on the user's page. If not, respond with a message asking them to check back later. // The delay here is to make sure GCS registers newly uploaded files which is usually <1 second + let foundDocs = false; for (let attempt = 0; attempt < 5; attempt++) { if (await publisher.hasDocsBeenGenerated(entity)) { + foundDocs = true; break; } await new Promise(r => setTimeout(r, 1000)); + } + if (!foundDocs) { logger.error( 'Published files are taking longer to show up in storage. Something went wrong.', ); @@ -162,13 +166,15 @@ export async function createRouter({ ); return; } + } else { + logger.info( + 'Found pre-generated docs for this entity. Serving them.', + ); + // TODO: re-trigger build for cache invalidation. + // Compare the date modified of the requested file on storage and compare it against + // the last modified or last commit timestamp in the repository. + // Without this, docs will not be re-built once they have been generated. } - - logger.info('Found pre-generated docs for this entity. Serving them.'); - // TODO: re-trigger build for cache invalidation. - // Compare the date modified of the requested file on storage and compare it against - // the last modified or last commit timestamp in the repository. - // Without this, docs will not be re-built once they have been generated. } } From a212e431425ed93d4214c7c0c9608b5d163f21ce Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Sat, 5 Dec 2020 10:48:54 +0100 Subject: [PATCH 017/188] techdocs: Add JSON Schema config Show better error if techdocs.builder is set to 'ci' and if no docs are found. Return 404 from googleStorage client when a file is not found. --- app-config.yaml | 2 +- .../src/stages/publish/googleStorage.ts | 4 +- .../techdocs-backend/src/service/router.ts | 4 +- plugins/techdocs/package.json | 49 ++++++++++++++++++- .../reader/components/TechDocsNotFound.tsx | 16 +++++- 5 files changed, 66 insertions(+), 9 deletions(-) diff --git a/app-config.yaml b/app-config.yaml index 65581b7d23..41e573ec68 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -65,7 +65,7 @@ organization: techdocs: requestUrl: http://localhost:7000/api/techdocs storageUrl: http://localhost:7000/api/techdocs/static/docs - docsBuilder: 'local' # Alternatives - 'ci' + builder: 'local' # Alternatives - 'ci' generators: techdocs: 'docker' # Alternatives - 'local' publisher: diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index 7023f7c1eb..f21928a824 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -180,8 +180,8 @@ export class GoogleGCSPublish implements PublisherBase { .file(filePath) .createReadStream() .on('error', err => { - this.logger.error(err.message); - res.send(err.message); + this.logger.warn(err.message); + res.status(404).send(err.message); }) .on('data', chunk => { fileStreamChunks.push(chunk); diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index 76778f291f..ea6ae51c97 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -120,10 +120,10 @@ export async function createRouter({ ); } - // techdocs-backend will only try to build documentation for an entity if techdocs.docsBuilder is set to 'local' + // techdocs-backend will only try to build documentation for an entity if techdocs.builder is set to 'local' // If set to 'ci', it will only try to fetch and assume that that CI/CD pipeline of the repository hosting the // entity's documentation is responsible for building and publishing documentation to the storage provider. - if (config.getString('techdocs.docsBuilder') === 'local') { + if (config.getString('techdocs.builder') === 'local') { const docsBuilder = new DocsBuilder({ preparers, generators, diff --git a/plugins/techdocs/package.json b/plugins/techdocs/package.json index 3ec6ed6457..9c6e010f63 100644 --- a/plugins/techdocs/package.json +++ b/plugins/techdocs/package.json @@ -68,11 +68,56 @@ "visibility": "frontend" }, "storageUrl": { - "type": "string" + "type": "string", + "visibility": "backend" + }, + "generators": { + "type": "object", + "properties": { + "techdocs": { + "type": "string", + "visibility": "backend" + } + } + }, + "builder": { + "type": "string", + "visibility": "frontend" + }, + "publisher": { + "type": "object", + "properties": { + "type": { + "type": "string", + "visibility": "backend" + }, + "google": { + "type": "object", + "properties": { + "pathToKey": { + "type": "string", + "visibility": "secret" + }, + "projectId": { + "type": "string", + "visibility": "secret" + }, + "bucketName": { + "type": "string", + "visibility": "secret" + } + } + } + }, + "required": [ + "type" + ] } }, "required": [ - "requestUrl" + "requestUrl", + "storageUrl", + "builder" ] } } diff --git a/plugins/techdocs/src/reader/components/TechDocsNotFound.tsx b/plugins/techdocs/src/reader/components/TechDocsNotFound.tsx index 744242e343..d3b3bda8bc 100644 --- a/plugins/techdocs/src/reader/components/TechDocsNotFound.tsx +++ b/plugins/techdocs/src/reader/components/TechDocsNotFound.tsx @@ -15,18 +15,30 @@ */ import React from 'react'; -import { ErrorPage } from '@backstage/core'; +import { ErrorPage, useApi, configApiRef } from '@backstage/core'; type Props = { errorMessage?: string; }; export const TechDocsNotFound = ({ errorMessage }: Props) => { + const techdocsBuilder = useApi(configApiRef).getOptionalString( + 'techdocs.builder', + ); + + let additionalInfo = ''; + if (techdocsBuilder === 'ci') { + additionalInfo = + "Note that you have set techdocs.builder to 'ci' in your config, which means this Backstage app will not " + + "build docs if they are not found. Make sure the project's CI/CD pipeline builds and publishes docs. Or " + + "change techdocs.builder to 'local' to build docs from this Backstage instance."; + } + return ( ); }; From ff3dd9c02b7796daae1ca5974c001f59c3422f45 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Sat, 5 Dec 2020 11:13:27 +0100 Subject: [PATCH 018/188] create-app: Add new techdocs default configs --- .../create-app/templates/default-app/app-config.yaml.hbs | 5 ++++- packages/techdocs-common/src/stages/publish/googleStorage.ts | 2 +- plugins/techdocs-backend/src/service/router.ts | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/create-app/templates/default-app/app-config.yaml.hbs b/packages/create-app/templates/default-app/app-config.yaml.hbs index 595812dec3..7b70a2192c 100644 --- a/packages/create-app/templates/default-app/app-config.yaml.hbs +++ b/packages/create-app/templates/default-app/app-config.yaml.hbs @@ -57,10 +57,13 @@ proxy: changeOrigin: true techdocs: - storageUrl: http://localhost:7000/api/techdocs/static/docs requestUrl: http://localhost:7000/api/techdocs + storageUrl: http://localhost:7000/api/techdocs/static/docs + builder: 'local' generators: techdocs: 'docker' + publisher: + type: 'local' lighthouse: baseUrl: http://localhost:3003 diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index f21928a824..5df4f2d886 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -108,7 +108,7 @@ export class GoogleGCSPublish implements PublisherBase { // 'end' event happens when all the files have been read. const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; allFilesToUpload.forEach(filePath => { - const source = path.join(directory, filePath); // Local file absolutely path + const source = path.join(directory, filePath); // Local file absolute path const destination = `${entityRootDir}/${filePath}`; // GCS Bucket file relative path this.storageClient .bucket(this.bucketName) diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index ea6ae51c97..97a950d8f2 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -162,7 +162,7 @@ export async function createRouter({ res .status(408) .send( - 'Sorry! It is taking longer for the generated docs to show up up in storage. Check back later.', + 'Sorry! It is taking longer for the generated docs to show up in storage. Check back later.', ); return; } From 17cab77fd2f2acf4236e1bafd78b817455fcbf92 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Sat, 5 Dec 2020 17:42:21 +0100 Subject: [PATCH 019/188] techdocs-common: Add tests for google publisher and some more --- .../__mocks__/@google-cloud/storage.ts | 53 ++++++++ packages/techdocs-common/__mocks__/klaw.ts | 26 ++++ packages/techdocs-common/package.json | 9 +- packages/techdocs-common/src/helpers.test.ts | 121 +++++++++++++++--- .../src/stages/publish/googleStorage.test.ts | 88 +++++++++++++ .../src/stages/publish/publish.test.ts | 97 ++++++++++++++ 6 files changed, 371 insertions(+), 23 deletions(-) create mode 100644 packages/techdocs-common/__mocks__/@google-cloud/storage.ts create mode 100644 packages/techdocs-common/__mocks__/klaw.ts create mode 100644 packages/techdocs-common/src/stages/publish/googleStorage.test.ts create mode 100644 packages/techdocs-common/src/stages/publish/publish.test.ts diff --git a/packages/techdocs-common/__mocks__/@google-cloud/storage.ts b/packages/techdocs-common/__mocks__/@google-cloud/storage.ts new file mode 100644 index 0000000000..e95cee11d0 --- /dev/null +++ b/packages/techdocs-common/__mocks__/@google-cloud/storage.ts @@ -0,0 +1,53 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +type storageOptions = { + projectId?: string; + keyFilename?: string; +}; + +class Bucket { + private readonly bucketName; + + constructor(bucketName: string) { + this.bucketName = bucketName; + } + + getMetadata() { + return new Promise(resolve => { + resolve(''); + }); + } + + upload(source: string, { destination }) { + return new Promise(resolve => { + resolve({ source, destination }); + }); + } +} + +export class Storage { + private readonly projectId; + private readonly keyFilename; + + constructor(options: storageOptions) { + this.projectId = options.projectId; + this.keyFilename = options.keyFilename; + } + + bucket(bucketName) { + return new Bucket(bucketName); + } +} diff --git a/packages/techdocs-common/__mocks__/klaw.ts b/packages/techdocs-common/__mocks__/klaw.ts new file mode 100644 index 0000000000..ce2ce90a5f --- /dev/null +++ b/packages/techdocs-common/__mocks__/klaw.ts @@ -0,0 +1,26 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { EventEmitter } from 'events'; + +const walk = () => { + const emitter = new EventEmitter(); + setTimeout(() => { + emitter.emit('end'); + }, 10); + return emitter; +}; + +export default walk; diff --git a/packages/techdocs-common/package.json b/packages/techdocs-common/package.json index 7cd53c69c1..eb917edc2b 100644 --- a/packages/techdocs-common/package.json +++ b/packages/techdocs-common/package.json @@ -54,7 +54,12 @@ "winston": "^3.2.1" }, "devDependencies": { - "@types/klaw": "^3.0.1", - "@backstage/cli": "^0.4.0" + "@backstage/cli": "^0.4.0", + "@types/klaw": "^3.0.1" + }, + "jest": { + "roots": [ + ".." + ] } } diff --git a/packages/techdocs-common/src/helpers.test.ts b/packages/techdocs-common/src/helpers.test.ts index 10518df74f..88c4d8829e 100644 --- a/packages/techdocs-common/src/helpers.test.ts +++ b/packages/techdocs-common/src/helpers.test.ts @@ -15,10 +15,108 @@ */ import { Readable } from 'stream'; -import { getDocFilesFromRepository } from './helpers'; +import { + getDocFilesFromRepository, + getLocationForEntity, + parseReferenceAnnotation, +} from './helpers'; import { UrlReader, ReadTreeResponse } from '@backstage/backend-common'; import { Entity } from '@backstage/catalog-model'; +const entityBase: Entity = { + metadata: { + namespace: 'default', + name: 'mytestcomponent', + description: 'A component for testing', + }, + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + spec: { + type: 'documentation', + lifecycle: 'experimental', + owner: 'testuser', + }, +}; + +const metadataBase = { + namespace: 'default', + name: 'mytestcomponent', + description: 'A component for testing', +}; + +const goodAnnotation = { + annotations: { + 'backstage.io/techdocs-ref': + 'url:https://github.com/backstage/backstage/blob/master/subfolder/', + }, +}; + +const mockEntityWithAnnotation: Entity = { + ...entityBase, + ...{ + metadata: { + ...metadataBase, + ...goodAnnotation, + }, + }, +}; + +const badAnnotation = { + annotations: { + 'backstage.io/techdocs-ref': 'bad-annotation', + }, +}; + +const mockEntityWithBadAnnotation: Entity = { + ...entityBase, + ...{ + metadata: { + ...metadataBase, + ...badAnnotation, + }, + }, +}; + +describe('parseReferenceAnnotation', () => { + it('should parse annotation', () => { + const parsedLocationAnnotation = parseReferenceAnnotation( + 'backstage.io/techdocs-ref', + mockEntityWithAnnotation, + ); + expect(parsedLocationAnnotation.type).toBe('url'); + expect(parsedLocationAnnotation.target).toBe( + 'https://github.com/backstage/backstage/blob/master/subfolder/', + ); + }); + + it('should throw error without annotation', () => { + expect(() => { + parseReferenceAnnotation('backstage.io/techdocs-ref', entityBase); + }).toThrow(/No location annotation/); + }); + + it('should throw error with bad annotation', () => { + expect(() => { + parseReferenceAnnotation( + 'backstage.io/techdocs-ref', + mockEntityWithBadAnnotation, + ); + }).toThrow(/Failure to parse/); + }); +}); + +describe('getLocationForEntity', () => { + it('should get location for entity', () => { + const parsedLocationAnnotation = getLocationForEntity( + mockEntityWithAnnotation, + ); + expect(parsedLocationAnnotation.type).toBe('url'); + expect(parsedLocationAnnotation.target).toBe( + 'https://github.com/backstage/backstage/blob/master/subfolder/', + ); + }); +}); + describe('getDocFilesFromRepository', () => { it('should read a remote directory using UrlReader.readTree', async () => { class MockUrlReader implements UrlReader { @@ -41,28 +139,9 @@ describe('getDocFilesFromRepository', () => { } } - const mockEntity: Entity = { - metadata: { - namespace: 'default', - annotations: { - 'backstage.io/techdocs-ref': - 'url:https://github.com/backstage/backstage/blob/master/subfolder/', - }, - name: 'mytestcomponent', - description: 'A component for testing', - }, - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - spec: { - type: 'documentation', - lifecycle: 'experimental', - owner: 'testuser', - }, - }; - const output = await getDocFilesFromRepository( new MockUrlReader(), - mockEntity, + mockEntityWithAnnotation, ); expect(output).toBe('/tmp/testfolder'); diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts new file mode 100644 index 0000000000..e618f7edc7 --- /dev/null +++ b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts @@ -0,0 +1,88 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import mockFs from 'mock-fs'; +import * as winston from 'winston'; +import { ConfigReader } from '@backstage/config'; +import { GoogleGCSPublish } from './googleStorage'; +import { PublisherBase } from './types'; + +const createMockEntity = (annotations = {}) => { + return { + apiVersion: 'version', + kind: 'TestKind', + metadata: { + name: 'test-component-name', + annotations: { + ...annotations, + }, + }, + }; +}; + +const logger = winston.createLogger(); +jest.spyOn(logger, 'info').mockReturnValue(logger); + +let publisher: PublisherBase; + +beforeEach(() => { + mockFs({ + '/path/to/google-application-credentials.json': '{}', + }); + + const mockConfig = ConfigReader.fromConfigs([ + { + context: '', + data: { + techdocs: { + requestUrl: 'http://localhost:7000', + publisher: { + type: 'google_gcs', + google: { + pathToKey: '/path/to/google-application-credentials.json', + projectId: 'gcp-project-id', + bucketName: 'bucketName', + }, + }, + }, + }, + }, + ]); + + publisher = GoogleGCSPublish.fromConfig(mockConfig, logger); +}); + +afterEach(() => { + mockFs.restore(); +}); + +describe('GoogleGCSPublish', () => { + it('should publish a directory', () => { + mockFs({ + '/path/to/generatedDirectory': { + 'index.html': '', + '404.html': '', + assets: { + 'main.css': '', + }, + }, + }); + + const entity = createMockEntity(); + return expect( + publisher.publish({ entity, directory: '/path/to/generatedDirectory' }), + ).resolves.toStrictEqual({}); + }); +}); diff --git a/packages/techdocs-common/src/stages/publish/publish.test.ts b/packages/techdocs-common/src/stages/publish/publish.test.ts new file mode 100644 index 0000000000..002719997b --- /dev/null +++ b/packages/techdocs-common/src/stages/publish/publish.test.ts @@ -0,0 +1,97 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import mockFs from 'mock-fs'; +import { + getVoidLogger, + PluginEndpointDiscovery, +} from '@backstage/backend-common'; +import { ConfigReader } from '@backstage/config'; +import { Publisher } from './publish'; +import { LocalPublish } from './local'; +import { GoogleGCSPublish } from './googleStorage'; + +const logger = getVoidLogger(); +const testDiscovery: jest.Mocked = { + getBaseUrl: jest.fn().mockResolvedValueOnce('http://localhost:7000'), + getExternalBaseUrl: jest.fn(), +}; + +describe('Publisher', () => { + it('should create local publisher by default', () => { + const mockConfig = ConfigReader.fromConfigs([ + { + context: '', + data: { + techdocs: { + requestUrl: 'http://localhost:7000', + }, + }, + }, + ]); + + const publisher = Publisher.fromConfig(mockConfig, logger, testDiscovery); + expect(publisher).toBeInstanceOf(LocalPublish); + }); + + it('should create local publisher from config', () => { + const mockConfig = ConfigReader.fromConfigs([ + { + context: '', + data: { + techdocs: { + requestUrl: 'http://localhost:7000', + publisher: { + type: 'local', + }, + }, + }, + }, + ]); + + const publisher = Publisher.fromConfig(mockConfig, logger, testDiscovery); + expect(publisher).toBeInstanceOf(LocalPublish); + }); + + it('should create google gcs publisher from config', () => { + mockFs({ + '/path/to/google-application-credentials.json': '{}', + }); + + const mockConfig = ConfigReader.fromConfigs([ + { + context: '', + data: { + techdocs: { + requestUrl: 'http://localhost:7000', + publisher: { + type: 'google_gcs', + google: { + pathToKey: '/path/to/google-application-credentials.json', + projectId: 'gcp-project-id', + bucketName: 'bucketName', + }, + }, + }, + }, + }, + ]); + + const publisher = Publisher.fromConfig(mockConfig, logger, testDiscovery); + expect(publisher).toBeInstanceOf(GoogleGCSPublish); + + mockFs.restore(); + }); +}); From 372160ede54bf3179c6b2e890e2ba6c420e1e60c Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Sat, 5 Dec 2020 18:07:50 +0100 Subject: [PATCH 020/188] Update README of all three techdocs packages/plugins --- packages/techdocs-common/README.md | 42 ++++++++++++++++++++++++++- plugins/techdocs-backend/README.md | 27 ++++------------- plugins/techdocs-backend/src/index.ts | 1 - plugins/techdocs/README.md | 4 +-- 4 files changed, 47 insertions(+), 27 deletions(-) diff --git a/packages/techdocs-common/README.md b/packages/techdocs-common/README.md index acf267bbfb..26e6a8db54 100644 --- a/packages/techdocs-common/README.md +++ b/packages/techdocs-common/README.md @@ -2,6 +2,46 @@ Common functionalities for TechDocs, to be shared between techdocs plugins and techdocs-cli +This package is used by `techdocs-backend` to serve docs from different types of publishers (Google GCS, Local, etc.). +It is also used to build docs and publish them to storage, by both `techdocs-backend` and `techdocs-cli`. ## Usage -TODO: List supported APIs +Create a preparer instance from the [preparers available](/packages/techdocs-common/src/stages/prepare) at which takes an Entity instance. +Run the [docs generator](/packages/techdocs-common/src/stages/generate) on the prepared directory. +Publish the generated directory files to a [storage](/packages/techdocs-common/src/stages/publish) of your choice. + +Example: +```js +async () => { + const preparedDir = await preparer.prepare(entity); + + const parsedLocationAnnotation = getLocationForEntity(entity); + const { resultDir } = await generator.run({ + directory: preparedDir, + dockerClient: dockerClient, + parsedLocationAnnotation, + }); + + await publisher.publish({ + entity: entity, + directory: resultDir, + }); +} +``` + +## Features + +Currently the build process is split up in these three stages. + +- Preparers +- Generators +- Publishers + +Preparers read your entity data and creates a working directory with your documentation source code. For example if you have set your `backstage.io/techdocs-ref` to `github:https://github.com/backstage/backstage.git` it will clone that repository to a temp folder and pass that on to the generator. + +Generators takes the prepared source and runs the `techdocs-container` on it. It then passes on the output folder of that build to the publisher. + +Publishers gets a folder path from the generator and publish it to your storage solution. Read documentation to know more about configuring storage solutions. +http://backstage.io/docs/features/techdocs/configuration + +Any of these can be extended. We want to extend our support to most of the storage providers (Publishers) and source code host providers (Preparers). diff --git a/plugins/techdocs-backend/README.md b/plugins/techdocs-backend/README.md index 625f0910b5..c2a1cb99aa 100644 --- a/plugins/techdocs-backend/README.md +++ b/plugins/techdocs-backend/README.md @@ -18,30 +18,13 @@ yarn start ## What techdocs-backend does -This plugin is the backend part of the techdocs plugin. It provides building and serving of your docs without having to use another service and hosting provider. To use it set your techdocs storageUrl in your `app-config.yml` to `http://localhost:7000/api/techdocs/static/docs`. +This plugin is the backend part of the techdocs plugin. It provides serving and building of documentation for any entity. +To configure various storage providers and building options, see http://backstage.io/docs/features/techdocs/configuration -```yaml -techdocs: - storageUrl: http://localhost:7000/api/techdocs/static/docs -``` - -## Extending techdocs-backend - -Currently the build process of techdocs-backend is split up in these three stages. - -- Preparers -- Generators -- Publishers - -Preparers read your entity data and creates a working directory with your documentation source code. For example if you have set your `backstage.io/techdocs-ref` to `github:https://github.com/backstage/backstage.git` it will clone that repository to a temp folder and pass that on to the generator. - -Generators takes the prepared source and runs the `techdocs-container` on it. It then passes on the output folder of that build to the publisher. - -Publishers gets a folder path from the generator and publish it to your storage solution. Currently the only built in storage solution is a folder called `static/docs` inside the techdocs-backend plugin. - -Any of these can be extended. If we want to publish to a external static file server using rsync for example that can be done by creating a rsync publisher. _(Keep in mind that if you want techdocs-backend to initiate a build this would also require techdocs-backend to act as a proxy, which is not yet implemented.)_ +The techdocs-backend re-exports the [techdocs-common](https://github.com/backstage/backstage/tree/master/packages/techdocs-common) package which has the features to prepare, generate and publish docs. +The Publishers are also used to fetch the static documentation files and render them in TechDocs. ## Links - [Frontend part of the plugin](https://github.com/backstage/backstage/tree/master/plugins/techdocs) -- [The Backstage homepage](https://backstage.io) +- [Backstage homepage](https://backstage.io) diff --git a/plugins/techdocs-backend/src/index.ts b/plugins/techdocs-backend/src/index.ts index 50a98990cc..5c57882788 100644 --- a/plugins/techdocs-backend/src/index.ts +++ b/plugins/techdocs-backend/src/index.ts @@ -15,5 +15,4 @@ */ export * from './service/router'; -// TODO: Do named exports here e.g. publishers, generators. export * from '@backstage/techdocs-common'; diff --git a/plugins/techdocs/README.md b/plugins/techdocs/README.md index 2e9365f6c6..c5e8fa6887 100644 --- a/plugins/techdocs/README.md +++ b/plugins/techdocs/README.md @@ -6,9 +6,7 @@ Set up Backstage and TechDocs by follow our guide on [Getting Started](../../doc ## Configuration -### Custom Storage URL - -TechDocs will try to read your documentation from the URL you have specified in the `techdocs storageUrl` in `app-config.yml`. +http://backstage.io/docs/features/techdocs/configuration ### TechDocs Storage Api From dae4f39838c9fa35bd8a6ef2fac91d9f13cc527a Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Sat, 5 Dec 2020 18:36:50 +0100 Subject: [PATCH 021/188] techdocs: Add changeset for all three packages --- .changeset/friendly-carpets-repeat.md | 45 +++++++++++++++++++++++++++ .github/styles/vocab.txt | 1 + packages/techdocs-common/README.md | 4 ++- 3 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 .changeset/friendly-carpets-repeat.md diff --git a/.changeset/friendly-carpets-repeat.md b/.changeset/friendly-carpets-repeat.md new file mode 100644 index 0000000000..d42a28a8a6 --- /dev/null +++ b/.changeset/friendly-carpets-repeat.md @@ -0,0 +1,45 @@ +--- +'@backstage/techdocs-common': minor +'@backstage/plugin-techdocs': minor +'@backstage/plugin-techdocs-backend': minor +--- + +_Breaking changes_ + +1. Added option to use Google Cloud Storage as a choice to store the static generated files for TechDocs. + It can be configured using `techdocs.publisher.type` option in `app-config.yaml`. + Step-by-step guide to configure GCS is available here https://backstage.io/docs/features/techdocs/using-cloud-storage + Set `techdocs.publisher.type` to `'local'` if you want to continue using local filesystem to store TechDocs files. + +2. `techdocs.builder` is now required and can be set to `'local'` or `'ci'`. (Set it to `'local'` for now, since CI/CD build + workflow for TechDocs will be available soon (in few weeks)). + If builder is set to 'local' and you open a TechDocs page, `techdocs-backend` will try to build the docs, publish to storage and + show the generated docs afterwords. + If builder is set to `'ci'`, `techdocs-backend` will only fetch the docs and will NOT try to build and publish. In this case of 'ci', + we assume that docs are being built in the CI/CD pipeline of the repository. + TechDocs will not assume a default value for `techdocs.builder`. It is better to explicitly define it in the `app-config.yaml`. + +3. When configuring TechDocs in your backend, there is a difference in how a new publisher is created. + +``` +--- const publisher = new LocalPublish(logger, discovery); ++++ const publisher = Publisher.fromConfig(config, logger, discovery); +``` + +Based on the config `techdocs.publisher.type`, the publisher could be either Local publisher or Google Cloud Storage publisher. + +4. `techdocs.storageUrl` is now a required config. Should be `http://localhost:7000/api/techdocs/static/docs` in most setups. + +5. Parts of `@backstage/plugin-techdocs-backend` have been moved to a new package `@backstage/techdocs-common` to build docs. Also to publish docs + to-and-fro between TechDocs and a storage (either local or external). However, a Backstage app does NOT need to import the `techdocs-common` package - + app should only import `@backstage/plugin-techdocs` and `@backstage/plugin-techdocs-backend`. + +_Patch changes_ + +1. See all of TechDocs config options and its documentation https://backstage.io/docs/features/techdocs/configuration + +2. Logic about serving static files and metadata retrieval have been abstracted away from the router in `techdocs-backend` to the instance of publisher. + +3. Removed Material UI Spinner from TechDocs header. Spinners cause unnecessary UX distraction. + Case 1 (when docs are built and are to be served): Spinners appear for a split second before the name of site shows up. This unnecessarily distracts eyes because spinners increase the size of the Header. A dot (.) would do fine. Definitely more can be done. + Case 2 (when docs are being generated): There is already a linear progress bar (which is recommended in Storybook). diff --git a/.github/styles/vocab.txt b/.github/styles/vocab.txt index 7f889392ac..fa0a0d2087 100644 --- a/.github/styles/vocab.txt +++ b/.github/styles/vocab.txt @@ -23,6 +23,7 @@ changesets Changesets chanwit Chanwit +ci cisphobia cissexist classname diff --git a/packages/techdocs-common/README.md b/packages/techdocs-common/README.md index 26e6a8db54..2940df2524 100644 --- a/packages/techdocs-common/README.md +++ b/packages/techdocs-common/README.md @@ -4,6 +4,7 @@ Common functionalities for TechDocs, to be shared between techdocs plugins and t This package is used by `techdocs-backend` to serve docs from different types of publishers (Google GCS, Local, etc.). It is also used to build docs and publish them to storage, by both `techdocs-backend` and `techdocs-cli`. + ## Usage Create a preparer instance from the [preparers available](/packages/techdocs-common/src/stages/prepare) at which takes an Entity instance. @@ -11,6 +12,7 @@ Run the [docs generator](/packages/techdocs-common/src/stages/generate) on the p Publish the generated directory files to a [storage](/packages/techdocs-common/src/stages/publish) of your choice. Example: + ```js async () => { const preparedDir = await preparer.prepare(entity); @@ -26,7 +28,7 @@ async () => { entity: entity, directory: resultDir, }); -} +}; ``` ## Features From ac3560b42d1306f16b8fc4e1fdd9c9f9c83a59f9 Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Thu, 26 Nov 2020 18:16:53 +0100 Subject: [PATCH 022/188] Remove implementsApis This was deprecated in #3449. This should not be merged sooner then Dec 14th, 2020! --- .changeset/thirty-fans-hammer.md | 7 +++++++ .../software-catalog/descriptor-format.md | 15 ------------- .../components/petstore-component.yaml | 2 +- .../src/kinds/ComponentEntityV1alpha1.test.ts | 21 ------------------- .../src/kinds/ComponentEntityV1alpha1.ts | 7 ------- .../BuiltinKindsEntityProcessor.test.ts | 19 +---------------- .../processors/BuiltinKindsEntityProcessor.ts | 6 ------ 7 files changed, 9 insertions(+), 68 deletions(-) create mode 100644 .changeset/thirty-fans-hammer.md diff --git a/.changeset/thirty-fans-hammer.md b/.changeset/thirty-fans-hammer.md new file mode 100644 index 0000000000..45fb398f8c --- /dev/null +++ b/.changeset/thirty-fans-hammer.md @@ -0,0 +1,7 @@ +--- +'@backstage/catalog-model': minor +'@backstage/plugin-catalog-backend': minor +--- + +Remove `implementsApis` from `Component` entities. Deprecation happened in [#3449](https://github.com/backstage/backstage/pull/3449). +Use `providesApis` instead. diff --git a/docs/features/software-catalog/descriptor-format.md b/docs/features/software-catalog/descriptor-format.md index 19c5c74f68..3702f972d3 100644 --- a/docs/features/software-catalog/descriptor-format.md +++ b/docs/features/software-catalog/descriptor-format.md @@ -444,21 +444,6 @@ Apart from being a string, the software catalog leaves the format of this field open to implementers to choose. Most commonly, it is set to the ID or email of a group of people in an organizational structure. -### `spec.implementsApis` [optional] - -**NOTE**: This field was marked for deprecation on Nov 25nd, 2020. It will be -removed entirely from the model on Dec 14th, 2020 in the repository and will not -be present in released packages following the next release after that. Please -update your code to not consume this field before the removal date. - -Links APIs that are implemented by the component, e.g. `artist-api`. This field -is optional. - -The software catalog expects a list of one or more strings that references the -names of other entities of the `kind` `API`. - -This field has the same behavior as `spec.providesApis`. - ### `spec.providesApis` [optional] Links APIs that are provided by the component, e.g. `artist-api`. This field is diff --git a/packages/catalog-model/examples/components/petstore-component.yaml b/packages/catalog-model/examples/components/petstore-component.yaml index e6dd56c274..acbb2f82b0 100644 --- a/packages/catalog-model/examples/components/petstore-component.yaml +++ b/packages/catalog-model/examples/components/petstore-component.yaml @@ -7,7 +7,7 @@ spec: type: service lifecycle: experimental owner: team-c - implementsApis: + providesApis: - petstore - streetlights - hello-world diff --git a/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.test.ts b/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.test.ts index 8eec63daac..030c99c151 100644 --- a/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.test.ts +++ b/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.test.ts @@ -33,7 +33,6 @@ describe('ComponentV1alpha1Validator', () => { type: 'service', lifecycle: 'production', owner: 'me', - implementsApis: ['api-0'], providesApis: ['api-0'], consumesApis: ['api-0'], }, @@ -104,26 +103,6 @@ describe('ComponentV1alpha1Validator', () => { await expect(validator.check(entity)).rejects.toThrow(/owner/); }); - it('accepts missing implementsApis', async () => { - delete (entity as any).spec.implementsApis; - await expect(validator.check(entity)).resolves.toBe(true); - }); - - it('rejects empty implementsApis', async () => { - (entity as any).spec.implementsApis = ['']; - await expect(validator.check(entity)).rejects.toThrow(/implementsApis/); - }); - - it('rejects undefined implementsApis', async () => { - (entity as any).spec.implementsApis = [undefined]; - await expect(validator.check(entity)).rejects.toThrow(/implementsApis/); - }); - - it('accepts no implementsApis', async () => { - (entity as any).spec.implementsApis = []; - await expect(validator.check(entity)).resolves.toBe(true); - }); - it('accepts missing providesApis', async () => { delete (entity as any).spec.providesApis; await expect(validator.check(entity)).resolves.toBe(true); diff --git a/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.ts b/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.ts index 680674f16e..e511dcb24a 100644 --- a/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.ts +++ b/packages/catalog-model/src/kinds/ComponentEntityV1alpha1.ts @@ -29,7 +29,6 @@ const schema = yup.object>({ type: yup.string().required().min(1), lifecycle: yup.string().required().min(1), owner: yup.string().required().min(1), - implementsApis: yup.array(yup.string().required()).notRequired(), providesApis: yup.array(yup.string().required()).notRequired(), consumesApis: yup.array(yup.string().required()).notRequired(), }) @@ -43,12 +42,6 @@ export interface ComponentEntityV1alpha1 extends Entity { type: string; lifecycle: string; owner: string; - /** - * @deprecated This field will disappear on Dec 14th, 2020. Please remove - * any consuming code. The new field providesApis provides the - * same functionality like before. - */ - implementsApis?: string[]; providesApis?: string[]; consumesApis?: string[]; }; diff --git a/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.test.ts index e456dfd7a2..ca5328230e 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.test.ts @@ -67,7 +67,6 @@ describe('BuiltinKindsEntityProcessor', () => { type: 'service', owner: 'o', lifecycle: 'l', - implementsApis: ['a'], providesApis: ['b'], consumesApis: ['c'], }, @@ -75,7 +74,7 @@ describe('BuiltinKindsEntityProcessor', () => { await processor.postProcessEntity(entity, location, emit); - expect(emit).toBeCalledTimes(8); + expect(emit).toBeCalledTimes(6); expect(emit).toBeCalledWith({ type: 'relation', relation: { @@ -92,22 +91,6 @@ describe('BuiltinKindsEntityProcessor', () => { target: { kind: 'Group', namespace: 'default', name: 'o' }, }, }); - expect(emit).toBeCalledWith({ - type: 'relation', - relation: { - source: { kind: 'API', namespace: 'default', name: 'a' }, - type: 'apiProvidedBy', - target: { kind: 'Component', namespace: 'default', name: 'n' }, - }, - }); - expect(emit).toBeCalledWith({ - type: 'relation', - relation: { - source: { kind: 'Component', namespace: 'default', name: 'n' }, - type: 'providesApi', - target: { kind: 'API', namespace: 'default', name: 'a' }, - }, - }); expect(emit).toBeCalledWith({ type: 'relation', relation: { diff --git a/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts index 62b496dd65..c6c7435216 100644 --- a/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/BuiltinKindsEntityProcessor.ts @@ -134,12 +134,6 @@ export class BuiltinKindsEntityProcessor implements CatalogProcessor { RELATION_OWNED_BY, RELATION_OWNER_OF, ); - doEmit( - component.spec.implementsApis, - { defaultKind: 'API', defaultNamespace: selfRef.namespace }, - RELATION_PROVIDES_API, - RELATION_API_PROVIDED_BY, - ); doEmit( component.spec.providesApis, { defaultKind: 'API', defaultNamespace: selfRef.namespace }, From 7babbfaf3cc4acbbf478bd99fee5b8d1083da979 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96stberg?= Date: Tue, 8 Dec 2020 12:06:58 +0100 Subject: [PATCH 023/188] Add support for GitHub Enterprise from integration-config --- plugins/catalog-import/src/api/CatalogImportApi.ts | 2 ++ plugins/catalog-import/src/api/CatalogImportClient.ts | 6 +++++- plugins/catalog-import/src/util/useGithubRepos.ts | 9 ++++++++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-import/src/api/CatalogImportApi.ts b/plugins/catalog-import/src/api/CatalogImportApi.ts index 0ce569171f..414eacf128 100644 --- a/plugins/catalog-import/src/api/CatalogImportApi.ts +++ b/plugins/catalog-import/src/api/CatalogImportApi.ts @@ -16,6 +16,7 @@ import { createApiRef } from '@backstage/core'; import { PartialEntity } from '../util/types'; +import { GitHubIntegrationConfig } from '@backstage/integration' export const catalogImportApiRef = createApiRef({ id: 'plugin.catalog-import.service', @@ -27,6 +28,7 @@ export interface CatalogImportApi { owner: string; repo: string; fileContent: string; + githubIntegrationConfig: GitHubIntegrationConfig; }): Promise<{ link: string; location: string }>; createRepositoryLocation(options: { location: string }): Promise; generateEntityDefinitions(options: { diff --git a/plugins/catalog-import/src/api/CatalogImportClient.ts b/plugins/catalog-import/src/api/CatalogImportClient.ts index aaa110329e..57c6a8490a 100644 --- a/plugins/catalog-import/src/api/CatalogImportClient.ts +++ b/plugins/catalog-import/src/api/CatalogImportClient.ts @@ -19,6 +19,7 @@ import { DiscoveryApi, OAuthApi } from '@backstage/core'; import { CatalogImportApi } from './CatalogImportApi'; import { AnalyzeLocationResponse } from '@backstage/plugin-catalog-backend'; import { PartialEntity } from '../util/types'; +import { GitHubIntegrationConfig } from '@backstage/integration' export class CatalogImportClient implements CatalogImportApi { private readonly discoveryApi: DiscoveryApi; @@ -91,15 +92,18 @@ export class CatalogImportClient implements CatalogImportApi { owner, repo, fileContent, + githubIntegrationConfig, }: { owner: string; repo: string; fileContent: string; + githubIntegrationConfig: GitHubIntegrationConfig; }): Promise<{ link: string; location: string }> { const token = await this.githubAuthApi.getAccessToken(['repo']); const octo = new Octokit({ auth: token, + baseUrl: githubIntegrationConfig.apiBaseUrl, }); const branchName = 'backstage-integration'; @@ -179,7 +183,7 @@ export class CatalogImportClient implements CatalogImportApi { return { link: pullRequestResponse.data.html_url, - location: `https://github.com/${owner}/${repo}/blob/${repoData.data.default_branch}/${fileName}`, + location: `https://${githubIntegrationConfig.host}/${owner}/${repo}/blob/${repoData.data.default_branch}/${fileName}`, }; } } diff --git a/plugins/catalog-import/src/util/useGithubRepos.ts b/plugins/catalog-import/src/util/useGithubRepos.ts index a18edc2777..435d260f1b 100644 --- a/plugins/catalog-import/src/util/useGithubRepos.ts +++ b/plugins/catalog-import/src/util/useGithubRepos.ts @@ -15,15 +15,21 @@ */ import * as YAML from 'yaml'; -import { useApi } from '@backstage/core'; +import { useApi, configApiRef } from '@backstage/core'; import { catalogImportApiRef } from '../api/CatalogImportApi'; import { ConfigSpec } from '../components/ImportComponentPage'; +import { readGitHubIntegrationConfigs } from '@backstage/integration'; export function useGithubRepos() { const api = useApi(catalogImportApiRef); + const config = useApi(configApiRef); const submitPrToRepo = async (selectedRepo: ConfigSpec) => { const [ownerName, repoName] = selectedRepo.location.split('/').slice(-2); + const configs = readGitHubIntegrationConfigs( + config.getOptionalConfigArray('integrations.github') ?? [] + ) + const githubIntegrationConfig = configs[0] const submitPRResponse = await api .submitPrToRepo({ owner: ownerName, @@ -31,6 +37,7 @@ export function useGithubRepos() { fileContent: selectedRepo.config .map(entity => `---\n${YAML.stringify(entity)}`) .join('\n'), + githubIntegrationConfig }) .catch(e => { throw new Error(`Failed to submit PR to repo:\n${e.message}`); From ac454782f2f38ddb466208a6a4f3db1c25059f49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96stberg?= Date: Tue, 8 Dec 2020 12:24:18 +0100 Subject: [PATCH 024/188] Add dependency on module --- plugins/catalog-import/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/catalog-import/package.json b/plugins/catalog-import/package.json index 74cb3635be..86995e1e7a 100644 --- a/plugins/catalog-import/package.json +++ b/plugins/catalog-import/package.json @@ -25,6 +25,7 @@ "@backstage/core": "^0.3.2", "@backstage/plugin-catalog": "^0.2.5", "@backstage/plugin-catalog-backend": "^0.3.0", + "@backstage/integration": "^0.1.2", "@backstage/theme": "^0.2.1", "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", From a3cb631ca5a1d8b509328224b743145714c871fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96stberg?= Date: Tue, 8 Dec 2020 12:34:31 +0100 Subject: [PATCH 025/188] Ran prettier ... --- plugins/catalog-import/src/api/CatalogImportApi.ts | 2 +- plugins/catalog-import/src/api/CatalogImportClient.ts | 2 +- plugins/catalog-import/src/util/useGithubRepos.ts | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/catalog-import/src/api/CatalogImportApi.ts b/plugins/catalog-import/src/api/CatalogImportApi.ts index 414eacf128..99526a3eaa 100644 --- a/plugins/catalog-import/src/api/CatalogImportApi.ts +++ b/plugins/catalog-import/src/api/CatalogImportApi.ts @@ -16,7 +16,7 @@ import { createApiRef } from '@backstage/core'; import { PartialEntity } from '../util/types'; -import { GitHubIntegrationConfig } from '@backstage/integration' +import { GitHubIntegrationConfig } from '@backstage/integration'; export const catalogImportApiRef = createApiRef({ id: 'plugin.catalog-import.service', diff --git a/plugins/catalog-import/src/api/CatalogImportClient.ts b/plugins/catalog-import/src/api/CatalogImportClient.ts index 57c6a8490a..4ac015ffbf 100644 --- a/plugins/catalog-import/src/api/CatalogImportClient.ts +++ b/plugins/catalog-import/src/api/CatalogImportClient.ts @@ -19,7 +19,7 @@ import { DiscoveryApi, OAuthApi } from '@backstage/core'; import { CatalogImportApi } from './CatalogImportApi'; import { AnalyzeLocationResponse } from '@backstage/plugin-catalog-backend'; import { PartialEntity } from '../util/types'; -import { GitHubIntegrationConfig } from '@backstage/integration' +import { GitHubIntegrationConfig } from '@backstage/integration'; export class CatalogImportClient implements CatalogImportApi { private readonly discoveryApi: DiscoveryApi; diff --git a/plugins/catalog-import/src/util/useGithubRepos.ts b/plugins/catalog-import/src/util/useGithubRepos.ts index 435d260f1b..af1ef43c73 100644 --- a/plugins/catalog-import/src/util/useGithubRepos.ts +++ b/plugins/catalog-import/src/util/useGithubRepos.ts @@ -27,9 +27,9 @@ export function useGithubRepos() { const submitPrToRepo = async (selectedRepo: ConfigSpec) => { const [ownerName, repoName] = selectedRepo.location.split('/').slice(-2); const configs = readGitHubIntegrationConfigs( - config.getOptionalConfigArray('integrations.github') ?? [] - ) - const githubIntegrationConfig = configs[0] + config.getOptionalConfigArray('integrations.github') ?? [], + ); + const githubIntegrationConfig = configs[0]; const submitPRResponse = await api .submitPrToRepo({ owner: ownerName, @@ -37,7 +37,7 @@ export function useGithubRepos() { fileContent: selectedRepo.config .map(entity => `---\n${YAML.stringify(entity)}`) .join('\n'), - githubIntegrationConfig + githubIntegrationConfig, }) .catch(e => { throw new Error(`Failed to submit PR to repo:\n${e.message}`); From 2783ec0184fdcf2044c0525db659b1777456cd67 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Tue, 8 Dec 2020 15:19:21 +0100 Subject: [PATCH 026/188] create-app: Add changeset for techdocs-backend changes around GCS publisher --- .changeset/six-mugs-camp.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .changeset/six-mugs-camp.md diff --git a/.changeset/six-mugs-camp.md b/.changeset/six-mugs-camp.md new file mode 100644 index 0000000000..376a406b1d --- /dev/null +++ b/.changeset/six-mugs-camp.md @@ -0,0 +1,20 @@ +--- +'@backstage/create-app': patch +--- + +In the techdocs-backend plugin (`packages/backend/src/plugins/techdocs.ts`), create a publisher using + +``` + const publisher = Publisher.fromConfig(config, logger, discovery); +``` + +instead of + +``` + const publisher = new LocalPublish(logger, discovery); +``` + +An instance of `publisher` can either be a local filesystem publisher or a Google Cloud Storage publisher. + +Read more about the configs here https://backstage.io/docs/features/techdocs/configuration +(You will also have to update `techdocs.storage.type` to `local` or `google_gcs`. And `techdocs.builder` to either `local` or `ci`.) From a3b99fab844d95afee38fcf64ddf219659ab79e0 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Tue, 8 Dec 2020 16:07:34 +0100 Subject: [PATCH 027/188] techdocs-common: Use request/response in type names --- .../techdocs-common/src/stages/publish/googleStorage.ts | 4 ++-- packages/techdocs-common/src/stages/publish/local.ts | 4 ++-- packages/techdocs-common/src/stages/publish/types.ts | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index 5df4f2d886..3449a7e6e3 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -21,7 +21,7 @@ import { Logger } from 'winston'; import { Entity, EntityName } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { getHeadersForFileExtension, supportedFileType } from './helpers'; -import { PublisherBase, PublisherBaseParams } from './types'; +import { PublisherBase, PublishRequest } from './types'; export class GoogleGCSPublish implements PublisherBase { static fromConfig(config: Config, logger: Logger): PublisherBase { @@ -82,7 +82,7 @@ export class GoogleGCSPublish implements PublisherBase { * Upload all the files from the generated `directory` to the GCS bucket. * Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html */ - publish({ entity, directory }: PublisherBaseParams): Promise<{}> { + publish({ entity, directory }: PublishRequest): Promise<{}> { return new Promise((resolve, reject) => { // Path of all files to upload, relative to the root of the source directory // e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png'] diff --git a/packages/techdocs-common/src/stages/publish/local.ts b/packages/techdocs-common/src/stages/publish/local.ts index 0a75ab6914..7bb670f2d4 100644 --- a/packages/techdocs-common/src/stages/publish/local.ts +++ b/packages/techdocs-common/src/stages/publish/local.ts @@ -23,7 +23,7 @@ import { PluginEndpointDiscovery, } from '@backstage/backend-common'; import { Config } from '@backstage/config'; -import { PublisherBase, PublisherBaseParams } from './types'; +import { PublisherBase, PublishRequest } from './types'; /* `remoteUrl` is the URL which serves files from the local publisher's static directory. */ export type LocalPublishReturn = Promise<{ remoteUrl: string }>; @@ -52,7 +52,7 @@ export class LocalPublish implements PublisherBase { this.discovery = discovery; } - publish({ entity, directory }: PublisherBaseParams): LocalPublishReturn { + publish({ entity, directory }: PublishRequest): LocalPublishReturn { const entityNamespace = entity.metadata.namespace ?? 'default'; const publishDir = resolvePackagePath( diff --git a/packages/techdocs-common/src/stages/publish/types.ts b/packages/techdocs-common/src/stages/publish/types.ts index 6b5c9332a9..fae61241b2 100644 --- a/packages/techdocs-common/src/stages/publish/types.ts +++ b/packages/techdocs-common/src/stages/publish/types.ts @@ -21,13 +21,13 @@ import express from 'express'; */ export type PublisherType = 'local' | 'google_gcs'; -export type PublisherBaseParams = { +export type PublishRequest = { entity: Entity; /* The Path to the directory where the generated files are stored. */ directory: string; }; -export type PublisherBaseReturn = Promise<{}>; +export type PublishResponse = {}; /** * Base class for a TechDocs publisher (e.g. Local, Google GCS Bucket, AWS S3, etc.) @@ -38,10 +38,10 @@ export interface PublisherBase { /** * Store the generated static files onto a storage service (either local filesystem or external service). * - * @param options Object containing the entity from the service + * @param request Object containing the entity from the service * catalog, and the directory that contains the generated static files from TechDocs. */ - publish(options: PublisherBaseParams): PublisherBaseReturn; + publish(request: PublishRequest): Promise; /** * Retrieve TechDocs Metadata about a site e.g. name, contributors, last updated, etc. From d0c62e7dc7acede47ea900fd6efb760a5899de97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96stberg?= Date: Wed, 9 Dec 2020 11:05:29 +0100 Subject: [PATCH 028/188] Add todo and find config by hostname --- .../catalog-import/src/util/useGithubRepos.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/plugins/catalog-import/src/util/useGithubRepos.ts b/plugins/catalog-import/src/util/useGithubRepos.ts index af1ef43c73..7f49bb3363 100644 --- a/plugins/catalog-import/src/util/useGithubRepos.ts +++ b/plugins/catalog-import/src/util/useGithubRepos.ts @@ -18,6 +18,9 @@ import * as YAML from 'yaml'; import { useApi, configApiRef } from '@backstage/core'; import { catalogImportApiRef } from '../api/CatalogImportApi'; import { ConfigSpec } from '../components/ImportComponentPage'; + +//TODO: O5ten, refactor into a core API instead of direct usage like this +//https://github.com/backstage/backstage/pull/3613#issuecomment-7408929430 import { readGitHubIntegrationConfigs } from '@backstage/integration'; export function useGithubRepos() { @@ -25,11 +28,14 @@ export function useGithubRepos() { const config = useApi(configApiRef); const submitPrToRepo = async (selectedRepo: ConfigSpec) => { - const [ownerName, repoName] = selectedRepo.location.split('/').slice(-2); + const [hostname, ownerName, repoName] = selectedRepo.location.split('/').slice(-3); const configs = readGitHubIntegrationConfigs( - config.getOptionalConfigArray('integrations.github') ?? [], - ); - const githubIntegrationConfig = configs[0]; + config.getOptionalConfigArray('integrations.github') ?? [] + ) + const githubIntegrationConfig = configs.find(v => v.host === hostname); + if(!githubIntegrationConfig) { + throw new Error(`Unable to locate github-integration for repo-location: ${selectedRepo.location}`); + } const submitPRResponse = await api .submitPrToRepo({ owner: ownerName, @@ -37,7 +43,7 @@ export function useGithubRepos() { fileContent: selectedRepo.config .map(entity => `---\n${YAML.stringify(entity)}`) .join('\n'), - githubIntegrationConfig, + githubIntegrationConfig }) .catch(e => { throw new Error(`Failed to submit PR to repo:\n${e.message}`); From 7dda6180b0728d582b0b34be3cd952d01c959018 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96stberg?= Date: Wed, 9 Dec 2020 11:11:17 +0100 Subject: [PATCH 029/188] Ran prettier --- .../catalog-import/src/util/useGithubRepos.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/plugins/catalog-import/src/util/useGithubRepos.ts b/plugins/catalog-import/src/util/useGithubRepos.ts index 7f49bb3363..af65799987 100644 --- a/plugins/catalog-import/src/util/useGithubRepos.ts +++ b/plugins/catalog-import/src/util/useGithubRepos.ts @@ -28,13 +28,17 @@ export function useGithubRepos() { const config = useApi(configApiRef); const submitPrToRepo = async (selectedRepo: ConfigSpec) => { - const [hostname, ownerName, repoName] = selectedRepo.location.split('/').slice(-3); + const [hostname, ownerName, repoName] = selectedRepo.location + .split('/') + .slice(-3); const configs = readGitHubIntegrationConfigs( - config.getOptionalConfigArray('integrations.github') ?? [] - ) + config.getOptionalConfigArray('integrations.github') ?? [], + ); const githubIntegrationConfig = configs.find(v => v.host === hostname); - if(!githubIntegrationConfig) { - throw new Error(`Unable to locate github-integration for repo-location: ${selectedRepo.location}`); + if (!githubIntegrationConfig) { + throw new Error( + `Unable to locate github-integration for repo-location: ${selectedRepo.location}`, + ); } const submitPRResponse = await api .submitPrToRepo({ @@ -43,7 +47,7 @@ export function useGithubRepos() { fileContent: selectedRepo.config .map(entity => `---\n${YAML.stringify(entity)}`) .join('\n'), - githubIntegrationConfig + githubIntegrationConfig, }) .catch(e => { throw new Error(`Failed to submit PR to repo:\n${e.message}`); From 5066852ef299c8bc1f03303ce397b669392c9a60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96stberg?= Date: Wed, 9 Dec 2020 11:45:52 +0100 Subject: [PATCH 030/188] Update useGithubRepos.ts --- plugins/catalog-import/src/util/useGithubRepos.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-import/src/util/useGithubRepos.ts b/plugins/catalog-import/src/util/useGithubRepos.ts index af65799987..740f0581b1 100644 --- a/plugins/catalog-import/src/util/useGithubRepos.ts +++ b/plugins/catalog-import/src/util/useGithubRepos.ts @@ -19,8 +19,8 @@ import { useApi, configApiRef } from '@backstage/core'; import { catalogImportApiRef } from '../api/CatalogImportApi'; import { ConfigSpec } from '../components/ImportComponentPage'; -//TODO: O5ten, refactor into a core API instead of direct usage like this -//https://github.com/backstage/backstage/pull/3613#issuecomment-7408929430 +// TODO: (O5ten) Refactor into a core API instead of direct usage like this +// https://github.com/backstage/backstage/pull/3613#issuecomment-7408929430 import { readGitHubIntegrationConfigs } from '@backstage/integration'; export function useGithubRepos() { From 9bd130a73a724991009e8d5acfd5dace44c69c64 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Wed, 9 Dec 2020 12:31:40 +0100 Subject: [PATCH 031/188] techdocs: Use 'local' and 'external' for alternatives for techdocs.builder config Co-authored-by: freben --- .changeset/friendly-carpets-repeat.md | 4 ++-- .changeset/six-mugs-camp.md | 2 +- app-config.yaml | 2 +- docs/features/techdocs/configuration.md | 10 +++++----- plugins/techdocs-backend/src/service/router.ts | 4 ++-- .../src/reader/components/TechDocsNotFound.tsx | 9 +++++---- 6 files changed, 16 insertions(+), 15 deletions(-) diff --git a/.changeset/friendly-carpets-repeat.md b/.changeset/friendly-carpets-repeat.md index d42a28a8a6..e2ee348cf4 100644 --- a/.changeset/friendly-carpets-repeat.md +++ b/.changeset/friendly-carpets-repeat.md @@ -11,11 +11,11 @@ _Breaking changes_ Step-by-step guide to configure GCS is available here https://backstage.io/docs/features/techdocs/using-cloud-storage Set `techdocs.publisher.type` to `'local'` if you want to continue using local filesystem to store TechDocs files. -2. `techdocs.builder` is now required and can be set to `'local'` or `'ci'`. (Set it to `'local'` for now, since CI/CD build +2. `techdocs.builder` is now required and can be set to `'local'` or `'external'`. (Set it to `'local'` for now, since CI/CD build workflow for TechDocs will be available soon (in few weeks)). If builder is set to 'local' and you open a TechDocs page, `techdocs-backend` will try to build the docs, publish to storage and show the generated docs afterwords. - If builder is set to `'ci'`, `techdocs-backend` will only fetch the docs and will NOT try to build and publish. In this case of 'ci', + If builder is set to `'external'`, `techdocs-backend` will only fetch the docs and will NOT try to build and publish. In this case of `'external'`, we assume that docs are being built in the CI/CD pipeline of the repository. TechDocs will not assume a default value for `techdocs.builder`. It is better to explicitly define it in the `app-config.yaml`. diff --git a/.changeset/six-mugs-camp.md b/.changeset/six-mugs-camp.md index 376a406b1d..a27a25a15b 100644 --- a/.changeset/six-mugs-camp.md +++ b/.changeset/six-mugs-camp.md @@ -17,4 +17,4 @@ instead of An instance of `publisher` can either be a local filesystem publisher or a Google Cloud Storage publisher. Read more about the configs here https://backstage.io/docs/features/techdocs/configuration -(You will also have to update `techdocs.storage.type` to `local` or `google_gcs`. And `techdocs.builder` to either `local` or `ci`.) +(You will also have to update `techdocs.storage.type` to `local` or `google_gcs`. And `techdocs.builder` to either `local` or `external`.) diff --git a/app-config.yaml b/app-config.yaml index f8529f834b..81538d8f61 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -72,7 +72,7 @@ organization: techdocs: requestUrl: http://localhost:7000/api/techdocs storageUrl: http://localhost:7000/api/techdocs/static/docs - builder: 'local' # Alternatives - 'ci' + builder: 'local' # Alternatives - 'external' generators: techdocs: 'docker' # Alternatives - 'local' publisher: diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index 5c1b9ab5c0..73e0cc0f66 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -27,7 +27,7 @@ techdocs: # generators.techdocs can have two values: 'docker' or 'local'. This is to determine how to run the generator - whether to # spin up the techdocs-container docker image or to run mkdocs locally (assuming all the dependencies are taken care of). - # You want to change this to 'local' if you are running Backstage using your own customer Docker setup and want to avoid running + # You want to change this to 'local' if you are running Backstage using your own custom Docker setup and want to avoid running # into Docker in Docker situation. Read more here # https://backstage.io/docs/features/techdocs/getting-started#disable-docker-in-docker-situation-optional @@ -35,12 +35,12 @@ techdocs: techdocs: 'docker' - # techdocs.builder can be either 'local' or 'ci. + # techdocs.builder can be either 'local' or 'external. # If builder is set to 'local' and you open a TechDocs page, techdocs-backend will try to build the docs, publish to storage # and show the generated docs afterwords. This is the "Basic" setup of the TechDocs Architecture. - # If builder is set to 'ci', techdocs-backend will only fetch the docs and will NOT try to build and publish. In this case of 'ci', - # we assume that docs are being built in the CI/CD pipeline of the repository. This is the "Recommended" setup of the architecture. - # Read more here https://backstage.io/docs/features/techdocs/architecture + # If builder is set to 'external', techdocs-backend will only fetch the docs and will NOT try to build and publish. In this case of 'external', + # we assume that docs are being built by an external process (e.g. in the CI/CD pipeline of the repository). This is the "Recommended" setup of + # the architecture. Read more here https://backstage.io/docs/features/techdocs/architecture builder: 'local' diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index 97a950d8f2..a13f25790a 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -121,8 +121,8 @@ export async function createRouter({ } // techdocs-backend will only try to build documentation for an entity if techdocs.builder is set to 'local' - // If set to 'ci', it will only try to fetch and assume that that CI/CD pipeline of the repository hosting the - // entity's documentation is responsible for building and publishing documentation to the storage provider. + // If set to 'external', it will only try to fetch and assume that an external process (e.g. CI/CD pipeline + // of the repository) is responsible for building and publishing documentation to the storage provider. if (config.getString('techdocs.builder') === 'local') { const docsBuilder = new DocsBuilder({ preparers, diff --git a/plugins/techdocs/src/reader/components/TechDocsNotFound.tsx b/plugins/techdocs/src/reader/components/TechDocsNotFound.tsx index d3b3bda8bc..e19cb69631 100644 --- a/plugins/techdocs/src/reader/components/TechDocsNotFound.tsx +++ b/plugins/techdocs/src/reader/components/TechDocsNotFound.tsx @@ -27,11 +27,12 @@ export const TechDocsNotFound = ({ errorMessage }: Props) => { ); let additionalInfo = ''; - if (techdocsBuilder === 'ci') { + if (techdocsBuilder !== 'local') { additionalInfo = - "Note that you have set techdocs.builder to 'ci' in your config, which means this Backstage app will not " + - "build docs if they are not found. Make sure the project's CI/CD pipeline builds and publishes docs. Or " + - "change techdocs.builder to 'local' to build docs from this Backstage instance."; + "Note that techdocs.builder is not set to 'local' in your config, which means this Backstage app will not " + + "build docs if they are not found. Make sure the project's docs are generated and published by some external " + + "process (e.g. CI/CD pipeline). Or change techdocs.builder to 'local' to build docs from this Backstage " + + 'instance.'; } return ( From 1d0d1a3a451580d828ceb115154f9e3dd2a6966c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96stberg?= Date: Wed, 9 Dec 2020 12:32:09 +0100 Subject: [PATCH 032/188] Fix missing baseUrl in graphql client for GHE --- .../src/ingestion/processors/GithubOrgReaderProcessor.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts index 9aecb8b86e..a1e8b5dad5 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts @@ -64,6 +64,7 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { const client = !provider.token ? graphql : graphql.defaults({ + baseUrl: provider.apiBaseUrl, headers: { authorization: `token ${provider.token}`, }, From 704f7c7be0b4b0f785fecb43efb6d8e0ed54ceb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96stberg?= Date: Wed, 9 Dec 2020 12:35:22 +0100 Subject: [PATCH 033/188] Update app-config.yaml --- app-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app-config.yaml b/app-config.yaml index 9d4d702473..0eb00ebbf3 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -133,7 +133,7 @@ catalog: $env: GITHUB_TOKEN #### Example for how to add your GitHub Enterprise instance using the API: # - target: https://ghe.example.net - # apiBaseUrl: https://ghe.example.net/api/v3 + # apiBaseUrl: https://ghe.example.net/api # token: # $env: GHE_TOKEN ldapOrg: From 9d08ef8f4699e7b490809f41b7074d0fea19103d Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Wed, 9 Dec 2020 16:01:29 +0100 Subject: [PATCH 034/188] techdocs: replace 'build' with 'generate' in most places --- .changeset/friendly-carpets-repeat.md | 6 +++--- docs/features/techdocs/architecture.md | 8 ++++---- docs/features/techdocs/configuration.md | 4 ++-- packages/techdocs-common/src/stages/generate/helpers.ts | 2 +- .../techdocs/src/reader/components/TechDocsNotFound.tsx | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.changeset/friendly-carpets-repeat.md b/.changeset/friendly-carpets-repeat.md index e2ee348cf4..7fb0d2f6eb 100644 --- a/.changeset/friendly-carpets-repeat.md +++ b/.changeset/friendly-carpets-repeat.md @@ -13,9 +13,9 @@ _Breaking changes_ 2. `techdocs.builder` is now required and can be set to `'local'` or `'external'`. (Set it to `'local'` for now, since CI/CD build workflow for TechDocs will be available soon (in few weeks)). - If builder is set to 'local' and you open a TechDocs page, `techdocs-backend` will try to build the docs, publish to storage and + If builder is set to 'local' and you open a TechDocs page, `techdocs-backend` will try to generate the docs, publish to storage and show the generated docs afterwords. - If builder is set to `'external'`, `techdocs-backend` will only fetch the docs and will NOT try to build and publish. In this case of `'external'`, + If builder is set to `'external'`, `techdocs-backend` will only fetch the docs and will NOT try to generate and publish. In this case of `'external'`, we assume that docs are being built in the CI/CD pipeline of the repository. TechDocs will not assume a default value for `techdocs.builder`. It is better to explicitly define it in the `app-config.yaml`. @@ -30,7 +30,7 @@ Based on the config `techdocs.publisher.type`, the publisher could be either Loc 4. `techdocs.storageUrl` is now a required config. Should be `http://localhost:7000/api/techdocs/static/docs` in most setups. -5. Parts of `@backstage/plugin-techdocs-backend` have been moved to a new package `@backstage/techdocs-common` to build docs. Also to publish docs +5. Parts of `@backstage/plugin-techdocs-backend` have been moved to a new package `@backstage/techdocs-common` to generate docs. Also to publish docs to-and-fro between TechDocs and a storage (either local or external). However, a Backstage app does NOT need to import the `techdocs-common` package - app should only import `@backstage/plugin-techdocs` and `@backstage/plugin-techdocs-backend`. diff --git a/docs/features/techdocs/architecture.md b/docs/features/techdocs/architecture.md index f502b3836e..a71a23cabb 100644 --- a/docs/features/techdocs/architecture.md +++ b/docs/features/techdocs/architecture.md @@ -50,7 +50,7 @@ built. We assume each entity lives in a repository somewhere (GitHub, GitLab, etc.). We recommend using a CI/CD pipeline with the repository that has a dedicated -step/job to build docs for TechDocs. The generated static files are then stored +step/job to generate docs for TechDocs. The generated static files are then stored in a cloud storage solution of your choice. [Track progress here](https://github.com/backstage/backstage/issues/3096). @@ -60,9 +60,9 @@ your configured storage solution for the necessary files and returns them to TechDocs Reader. We will provide instructions, scripts and/or templates (e.g. GitHub actions) to -build docs in your CI/CD system. +generate docs in your CI/CD system. [Track progress here.](https://github.com/backstage/backstage/issues/3400) You -will be able to use `techdocs-cli` to build docs and publish the generated docs +will be able to use `techdocs-cli` to generate docs and publish the generated docs site files to your cloud storage system. Note about caching: We have noticed internally that some storage providers can @@ -120,7 +120,7 @@ docs site in real-time?** A: Generating the content from Markdown on the fly is not optimal (although that is how the basic out-of-the-box setup is implemented). Storage solutions act as a cache for the generated static content. TechDocs is also currently built on -MkDocs which does not allow us to build docs per-page, so we would have to build +MkDocs which does not allow us to generate docs per-page, so we would have to build all docs for a entity on every request. # Future work diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index 73e0cc0f66..f3be54a4de 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -36,9 +36,9 @@ techdocs: # techdocs.builder can be either 'local' or 'external. - # If builder is set to 'local' and you open a TechDocs page, techdocs-backend will try to build the docs, publish to storage + # If builder is set to 'local' and you open a TechDocs page, techdocs-backend will try to generate the docs, publish to storage # and show the generated docs afterwords. This is the "Basic" setup of the TechDocs Architecture. - # If builder is set to 'external', techdocs-backend will only fetch the docs and will NOT try to build and publish. In this case of 'external', + # If builder is set to 'external', techdocs-backend will only fetch the docs and will NOT try to generate and publish. In this case of 'external', # we assume that docs are being built by an external process (e.g. in the CI/CD pipeline of the repository). This is the "Recommended" setup of # the architecture. Read more here https://backstage.io/docs/features/techdocs/architecture diff --git a/packages/techdocs-common/src/stages/generate/helpers.ts b/packages/techdocs-common/src/stages/generate/helpers.ts index 13344b8542..773b542517 100644 --- a/packages/techdocs-common/src/stages/generate/helpers.ts +++ b/packages/techdocs-common/src/stages/generate/helpers.ts @@ -209,7 +209,7 @@ export const getRepoUrlFromLocationAnnotation = ( }; /** - * Update the mkdocs.yml file before TechDocs generator uses it to build docs site. + * Update the mkdocs.yml file before TechDocs generator uses it to generate docs site. * * List of tasks: * - Add repo_url if it does not exists diff --git a/plugins/techdocs/src/reader/components/TechDocsNotFound.tsx b/plugins/techdocs/src/reader/components/TechDocsNotFound.tsx index e19cb69631..cdacc1cb7e 100644 --- a/plugins/techdocs/src/reader/components/TechDocsNotFound.tsx +++ b/plugins/techdocs/src/reader/components/TechDocsNotFound.tsx @@ -30,8 +30,8 @@ export const TechDocsNotFound = ({ errorMessage }: Props) => { if (techdocsBuilder !== 'local') { additionalInfo = "Note that techdocs.builder is not set to 'local' in your config, which means this Backstage app will not " + - "build docs if they are not found. Make sure the project's docs are generated and published by some external " + - "process (e.g. CI/CD pipeline). Or change techdocs.builder to 'local' to build docs from this Backstage " + + "generate docs if they are not found. Make sure the project's docs are generated and published by some external " + + "process (e.g. CI/CD pipeline). Or change techdocs.builder to 'local' to generate docs from this Backstage " + 'instance.'; } From 6e8bb3ac076765280a591bc50e76ea6240892a9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Wed, 9 Dec 2020 16:10:30 +0100 Subject: [PATCH 035/188] catalog-backend: leave unknown placeholder-lookalikes untouched --- .changeset/breezy-chefs-report.md | 5 +++ .../processors/PlaceholderProcessor.test.ts | 38 ++++++++----------- .../processors/PlaceholderProcessor.ts | 19 +++++++--- 3 files changed, 34 insertions(+), 28 deletions(-) create mode 100644 .changeset/breezy-chefs-report.md diff --git a/.changeset/breezy-chefs-report.md b/.changeset/breezy-chefs-report.md new file mode 100644 index 0000000000..baaec3021f --- /dev/null +++ b/.changeset/breezy-chefs-report.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +leave unknown placeholder-lookalikes untouched in the catalog processing loop diff --git a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts index 5a3511af6d..8b2632ad5d 100644 --- a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.test.ts @@ -88,7 +88,7 @@ describe('PlaceholderProcessor', () => { ); }); - it('rejects multiple placeholders', async () => { + it('ignores multiple placeholders', async () => { const processor = new PlaceholderProcessor({ resolvers: { foo: jest.fn(), @@ -96,41 +96,35 @@ describe('PlaceholderProcessor', () => { }, reader, }); + const entity: Entity = { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'n', x: { $foo: 'a', $bar: 'b' } }, + }; await expect( - processor.preProcessEntity( - { - apiVersion: 'a', - kind: 'k', - metadata: { name: 'n', x: { $foo: 'a', $bar: 'b' } }, - }, - { type: 'a', target: 'b' }, - ), - ).rejects.toThrow( - 'Placeholders have to be on the form of a single $-prefixed key in an object', - ); + processor.preProcessEntity(entity, { type: 'a', target: 'b' }), + ).resolves.toEqual(entity); expect(read).not.toBeCalled(); }); - it('rejects unknown placeholders', async () => { + it('ignores unknown placeholders', async () => { const processor = new PlaceholderProcessor({ resolvers: { bar: jest.fn(), }, reader, }); + const entity: Entity = { + apiVersion: 'a', + kind: 'k', + metadata: { name: 'n', x: { $foo: 'a' } }, + }; await expect( - processor.preProcessEntity( - { - apiVersion: 'a', - kind: 'k', - metadata: { name: 'n', x: { $foo: 'a' } }, - }, - { type: 'a', target: 'b' }, - ), - ).rejects.toThrow('Encountered unknown placeholder $foo'); + processor.preProcessEntity(entity, { type: 'a', target: 'b' }), + ).resolves.toEqual(entity); expect(read).not.toBeCalled(); }); diff --git a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts index 0e23d51f61..b7b3df7b6e 100644 --- a/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/PlaceholderProcessor.ts @@ -76,21 +76,28 @@ export class PlaceholderProcessor implements CatalogProcessor { ? [data, false] : [Object.fromEntries(entries.map(([k, [v]]) => [k, v])), true]; } else if (keys.length !== 1) { - throw new Error( - 'Placeholders have to be on the form of a single $-prefixed key in an object', - ); + // This was an object that had more than one key, some of which were + // dollar prefixed. We only handle the case where there is exactly one + // such key; anything else is left alone. + return [data, false]; } const resolverKey = keys[0].substr(1); + const resolverValue = data[keys[0]]; const resolver = this.options.resolvers[resolverKey]; - if (!resolver) { - throw new Error(`Encountered unknown placeholder \$${resolverKey}`); + if (!resolver || typeof resolverValue !== 'string') { + // If there was no such placeholder resolver or if the value was not a + // string, we err on the side of safety and assume that this is + // something that's best left alone. For example, if the input contains + // JSONSchema, there may be "$ref": "#/definitions/node" nodes in the + // document. + return [data, false]; } return [ await resolver({ key: resolverKey, - value: data[keys[0]], + value: resolverValue, baseUrl: location.target, read: this.options.reader.read.bind(this.options.reader), }), From 03e444e97e32a80bad3c5f7357ca51fe8bf5e301 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 9 Dec 2020 16:50:36 +0100 Subject: [PATCH 036/188] docs/FAQ: add question about missing docker images --- docs/FAQ.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/FAQ.md b/docs/FAQ.md index d8be258141..da10f9724c 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -152,6 +152,27 @@ plugins to share dependencies between each other when possible. This contributes to Backstage being fast, which is an important part of the user and developer experience. +### Why are there no published Docker images or helm charts for Backstage? + +As mentioned above, Backstage is not a packaged service that you can use out of +the box. In order to get started with Backstage you need to use the +`@backstage/create-app` package to create and customize your own Backstage app. + +In order to build a Docker image from your own app, you can use the +`yarn build-image` command which is included out of the box in the app template. +By default this image will bundle up both the frontend and the backend into a +single image that you can deploy using your favorite tooling. + +There are also some examples that can help you deploy Backstage to kubernetes in +the +[contrib](https://github.com/backstage/backstage/tree/master/contrib/kubernetes) +folder. + +It is possible that example images will be provided in the future, which can be +used to quickly try out a small subset of the functionality of Backstage, but +these would not be able to provide much more functionality on top of what you +can see on a demo site. + ### Do I have to write plugins in TypeScript? No, you can use JavaScript if you prefer. We want to keep the Backstage core From 2b71db211668479ae337446d838aac0f59ad7759 Mon Sep 17 00:00:00 2001 From: Dominik Henneke Date: Tue, 8 Dec 2020 18:55:17 +0100 Subject: [PATCH 037/188] Consider group memberships in the OwnershipCard --- .changeset/witty-rabbits-smell.md | 5 ++ .../Cards/OwnershipCard/OwnershipCard.tsx | 13 ++--- .../org/src/components/getEntityRelations.ts | 42 ++++++++++++++ plugins/org/src/components/isOwnerOf.ts | 55 +++++++++++++++++++ 4 files changed, 107 insertions(+), 8 deletions(-) create mode 100644 .changeset/witty-rabbits-smell.md create mode 100644 plugins/org/src/components/getEntityRelations.ts create mode 100644 plugins/org/src/components/isOwnerOf.ts diff --git a/.changeset/witty-rabbits-smell.md b/.changeset/witty-rabbits-smell.md new file mode 100644 index 0000000000..fbe2234b45 --- /dev/null +++ b/.changeset/witty-rabbits-smell.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-org': patch +--- + +Support transitive ownerships of users and groups. diff --git a/plugins/org/src/components/Cards/OwnershipCard/OwnershipCard.tsx b/plugins/org/src/components/Cards/OwnershipCard/OwnershipCard.tsx index 7c4ad7a56a..16a833cf56 100644 --- a/plugins/org/src/components/Cards/OwnershipCard/OwnershipCard.tsx +++ b/plugins/org/src/components/Cards/OwnershipCard/OwnershipCard.tsx @@ -13,9 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import React from 'react'; import { InfoCard, useApi, Progress } from '@backstage/core'; -import { Entity, RELATION_OWNED_BY } from '@backstage/catalog-model'; +import { Entity } from '@backstage/catalog-model'; import { catalogApiRef } from '@backstage/plugin-catalog'; import { useAsync } from 'react-use'; import Alert from '@material-ui/lab/Alert'; @@ -28,6 +29,7 @@ import { Typography, } from '@material-ui/core'; import { pageTheme } from '@backstage/theme'; +import { isOwnerOf } from '../../isOwnerOf'; type EntitiesKinds = 'Component' | 'API'; type EntitiesTypes = @@ -118,9 +120,6 @@ export const OwnershipCard = ({ entity: Entity; variant: string; }) => { - const { - metadata: { name: groupName }, - } = entity; const catalogApi = useApi(catalogApiRef); const { loading, @@ -129,10 +128,8 @@ export const OwnershipCard = ({ } = useAsync(async () => { const entitiesList = await catalogApi.getEntities(); const ownedEntitiesList = entitiesList.items.filter(component => - component?.relations?.some( - r => r.type === RELATION_OWNED_BY && r.target.name === groupName, - ), - ) as Array; + isOwnerOf(entity, component), + ); return [ { diff --git a/plugins/org/src/components/getEntityRelations.ts b/plugins/org/src/components/getEntityRelations.ts new file mode 100644 index 0000000000..9230aaebee --- /dev/null +++ b/plugins/org/src/components/getEntityRelations.ts @@ -0,0 +1,42 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Entity, EntityName } from '@backstage/catalog-model'; + +// TODO: this file is copied from /packages/app/catalog/src/components/getEntityRelations.ts and +// should be replaced once common relation-functions are introduced. + +/** + * Get the related entity references. + */ +export function getEntityRelations( + entity: Entity | undefined, + relationType: string, + filter?: { kind: string }, +): EntityName[] { + let entityNames = + entity?.relations + ?.filter(r => r.type === relationType) + ?.map(r => r.target) || []; + + if (filter?.kind) { + entityNames = entityNames?.filter( + e => e.kind.toLowerCase() === filter.kind.toLowerCase(), + ); + } + + return entityNames; +} diff --git a/plugins/org/src/components/isOwnerOf.ts b/plugins/org/src/components/isOwnerOf.ts new file mode 100644 index 0000000000..dd7c7d6805 --- /dev/null +++ b/plugins/org/src/components/isOwnerOf.ts @@ -0,0 +1,55 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + Entity, + EntityName, + getEntityName, + RELATION_MEMBER_OF, + RELATION_OWNED_BY, +} from '@backstage/catalog-model'; +import { getEntityRelations } from './getEntityRelations'; + +// TODO: this file is copied from /packages/app/catalog/src/components/isOwnerOf.ts and +// should be replaced once common relation-functions are introduced. + +/** + * Check if one entity is owned by another. Returns true, if the entity is owned by the + * owner directly, or if the entity is owned by a group that the owner is a member of. + */ +export function isOwnerOf(owner: Entity, owned: Entity) { + const possibleOwners: EntityName[] = [ + ...getEntityRelations(owner, RELATION_MEMBER_OF, { kind: 'group' }), + ...(owner ? [getEntityName(owner)] : []), + ]; + + const owners = getEntityRelations(owned, RELATION_OWNED_BY); + + for (const owner of owners) { + if ( + possibleOwners.find( + o => + owner.kind.toLowerCase() === o.kind.toLowerCase() && + owner.namespace.toLowerCase() === o.namespace.toLowerCase() && + owner.name.toLowerCase() === o.name.toLowerCase(), + ) !== undefined + ) { + return true; + } + } + + return false; +} From 24bcdd0829cb7aa27203008737d3df3f58eaa983 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Wed, 9 Dec 2020 18:58:12 +0100 Subject: [PATCH 038/188] techdocs: Use googleGcs for publisher type instead of google_gcs --- .changeset/six-mugs-camp.md | 2 +- app-config.yaml | 2 +- docs/features/techdocs/configuration.md | 8 +-- docs/features/techdocs/using-cloud-storage.md | 20 ++++---- .../src/stages/publish/googleStorage.test.ts | 4 +- .../src/stages/publish/googleStorage.ts | 12 ++--- .../src/stages/publish/publish.test.ts | 4 +- .../src/stages/publish/publish.ts | 2 +- .../src/stages/publish/types.ts | 2 +- .../techdocs-backend/src/service/router.ts | 4 +- plugins/techdocs/package.json | 50 ++++++++++++------- 11 files changed, 61 insertions(+), 49 deletions(-) diff --git a/.changeset/six-mugs-camp.md b/.changeset/six-mugs-camp.md index a27a25a15b..373edd340a 100644 --- a/.changeset/six-mugs-camp.md +++ b/.changeset/six-mugs-camp.md @@ -17,4 +17,4 @@ instead of An instance of `publisher` can either be a local filesystem publisher or a Google Cloud Storage publisher. Read more about the configs here https://backstage.io/docs/features/techdocs/configuration -(You will also have to update `techdocs.storage.type` to `local` or `google_gcs`. And `techdocs.builder` to either `local` or `external`.) +(You will also have to update `techdocs.storage.type` to `local` or `googleGcs`. And `techdocs.builder` to either `local` or `external`.) diff --git a/app-config.yaml b/app-config.yaml index 81538d8f61..3a21faf418 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -76,7 +76,7 @@ techdocs: generators: techdocs: 'docker' # Alternatives - 'local' publisher: - type: 'local' # Alternatives - 'google_gcs'. Read documentation for using alternatives. + type: 'local' # Alternatives - 'googleGcs'. Read documentation for using alternatives. sentry: organization: my-company diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index f3be54a4de..eac6799317 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -50,16 +50,16 @@ techdocs: publisher: - # techdocs.publisher.type can be - 'local' or 'google_gcs' (aws_s3, azure_storage, etc. to be available as well). + # techdocs.publisher.type can be - 'local' or 'googleGcs' (awsS3, azureStorage, etc. to be available as well). # When set to 'local', techdocs-backend will create a 'static' directory at its root to store generated documentation files. - # When set to 'google_gcs', techdocs-backend will use a Google Cloud Storage Bucket to store generated documentation files. + # When set to 'googleGcs', techdocs-backend will use a Google Cloud Storage Bucket to store generated documentation files. type: 'local' - # Required when techdocs.publisher.type is set to 'google_gcs'. Skip otherwise. + # Required when techdocs.publisher.type is set to 'googleGcs'. Skip otherwise. - google: + googleGcs: # An API key is required to write to a storage bucket. pathToKey: '/path/to/google_application_credentials.json', diff --git a/docs/features/techdocs/using-cloud-storage.md b/docs/features/techdocs/using-cloud-storage.md index 07b2982e18..12e3acd1bf 100644 --- a/docs/features/techdocs/using-cloud-storage.md +++ b/docs/features/techdocs/using-cloud-storage.md @@ -22,24 +22,24 @@ for the latest instructions on the following steps involving GCP. **1. Set `techdocs.publisher.type` config in your `app-config.yaml`** -Set `techdocs.publisher.type` to `'google_gcs'`. +Set `techdocs.publisher.type` to `'googleGcs'`. ```yaml techdocs: publisher: - type: 'google_gcs' + type: 'googleGcs' ``` **2. GCP (Google Cloud Platform) Project** Create or choose a dedicated GCP project. Set -`techdocs.publisher.google.projectId` to the project ID. +`techdocs.publisher.googleGcs.projectId` to the project ID. ```yaml techdocs: publisher: - type: 'google_gcs' - google: + type: 'googleGcs' + googleGcs: projectId: 'gcp-project-id ``` @@ -58,13 +58,13 @@ IAM & Admin console), and create a new key. Use JSON format for the key. A `.json` file will be downloaded. This is the secret key TechDocs will use to make API calls. Make it available in your Backstage server and/or your local development server and set it in the app config -`techdocs.publisher.google.pathToKey`. +`techdocs.publisher.googleGcs.pathToKey`. ```yaml techdocs: publisher: - type: 'google_gcs' - google: + type: 'googleGcs' + googleGcs: projectId: 'gcp-project-id' pathToKey: '/path/to/google_application_credentials.json' ``` @@ -80,8 +80,8 @@ Set the name of the bucket to `techdocs.publisher ```yaml techdocs: publisher: - type: 'google_gcs' - google: + type: 'googleGcs' + googleGcs: projectId: 'gcp-project-id' pathToKey: '/path/to/google_application_credentials.json' bucketName: 'name-of-techdocs-storage-bucket' diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts index e618f7edc7..fa7bddc648 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts @@ -49,8 +49,8 @@ beforeEach(() => { techdocs: { requestUrl: 'http://localhost:7000', publisher: { - type: 'google_gcs', - google: { + type: 'googleGcs', + googleGcs: { pathToKey: '/path/to/google-application-credentials.json', projectId: 'gcp-project-id', bucketName: 'bucketName', diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index 3449a7e6e3..c9b4ffd3db 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -29,13 +29,13 @@ export class GoogleGCSPublish implements PublisherBase { let projectId = ''; let bucketName = ''; try { - pathToKey = config.getString('techdocs.publisher.google.pathToKey'); - projectId = config.getString('techdocs.publisher.google.projectId'); - bucketName = config.getString('techdocs.publisher.google.bucketName'); + pathToKey = config.getString('techdocs.publisher.googleGcs.pathToKey'); + projectId = config.getString('techdocs.publisher.googleGcs.projectId'); + bucketName = config.getString('techdocs.publisher.googleGcs.bucketName'); } catch (error) { throw new Error( - "Since techdocs.publisher.type is set to 'google_gcs' in your app config, " + - 'pathToKey, projectId and bucketName are required in techdocs.publisher.google ' + + "Since techdocs.publisher.type is set to 'googleGcs' in your app config, " + + 'pathToKey, projectId and bucketName are required in techdocs.publisher.googleGcs ' + 'required to authenticate with Google Cloud Storage.', ); } @@ -59,7 +59,7 @@ export class GoogleGCSPublish implements PublisherBase { logger.error( `Could not retrieve metadata about the GCS bucket ${bucketName} in the GCP project ${projectId}. ` + 'Make sure the GCP project and the bucket exists and the access key located at the path ' + - "techdocs.publisher.google.pathToKey defined in app config has the role 'Storage Object Creator'. " + + "techdocs.publisher.googleGcs.pathToKey defined in app config has the role 'Storage Object Creator'. " + 'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage', ); throw new Error(`from GCS client library: ${reason.message}`); diff --git a/packages/techdocs-common/src/stages/publish/publish.test.ts b/packages/techdocs-common/src/stages/publish/publish.test.ts index 002719997b..16e2225e46 100644 --- a/packages/techdocs-common/src/stages/publish/publish.test.ts +++ b/packages/techdocs-common/src/stages/publish/publish.test.ts @@ -77,8 +77,8 @@ describe('Publisher', () => { techdocs: { requestUrl: 'http://localhost:7000', publisher: { - type: 'google_gcs', - google: { + type: 'googleGcs', + googleGcs: { pathToKey: '/path/to/google-application-credentials.json', projectId: 'gcp-project-id', bucketName: 'bucketName', diff --git a/packages/techdocs-common/src/stages/publish/publish.ts b/packages/techdocs-common/src/stages/publish/publish.ts index c91ebe8bb3..04a9d89996 100644 --- a/packages/techdocs-common/src/stages/publish/publish.ts +++ b/packages/techdocs-common/src/stages/publish/publish.ts @@ -36,7 +36,7 @@ export class Publisher { ) ?? 'local') as PublisherType; switch (publisherType) { - case 'google_gcs': + case 'googleGcs': logger.info('Creating Google Storage Bucket publisher for TechDocs'); return GoogleGCSPublish.fromConfig(config, logger); case 'local': diff --git a/packages/techdocs-common/src/stages/publish/types.ts b/packages/techdocs-common/src/stages/publish/types.ts index fae61241b2..6722f6c221 100644 --- a/packages/techdocs-common/src/stages/publish/types.ts +++ b/packages/techdocs-common/src/stages/publish/types.ts @@ -19,7 +19,7 @@ import express from 'express'; /** * Key for all the different types of TechDocs publishers that are supported. */ -export type PublisherType = 'local' | 'google_gcs'; +export type PublisherType = 'local' | 'googleGcs'; export type PublishRequest = { entity: Entity; diff --git a/plugins/techdocs-backend/src/service/router.ts b/plugins/techdocs-backend/src/service/router.ts index a13f25790a..a64f581a1e 100644 --- a/plugins/techdocs-backend/src/service/router.ts +++ b/plugins/techdocs-backend/src/service/router.ts @@ -115,7 +115,7 @@ export async function createRouter({ } catch (err) { throw new Error( 'Unable to get techdocs.publisher.type in your app config. Set it to either ' + - "'local', 'google_gcs' or other support storage providers. Read more here " + + "'local', 'googleGcs' or other support storage providers. Read more here " + 'https://backstage.io/docs/features/techdocs/architecture', ); } @@ -136,7 +136,7 @@ export async function createRouter({ if (!(await docsBuilder.docsUpToDate())) { await docsBuilder.build(); } - } else if (publisherType === 'google_gcs') { + } else if (publisherType === 'googleGcs') { // This block should be valid for all external storage implementations. So no need to duplicate in future, // add the publisher type in the list here. if (!(await publisher.hasDocsBeenGenerated(entity))) { diff --git a/plugins/techdocs/package.json b/plugins/techdocs/package.json index 9c6e010f63..da4c239f36 100644 --- a/plugins/techdocs/package.json +++ b/plugins/techdocs/package.json @@ -85,32 +85,44 @@ "visibility": "frontend" }, "publisher": { - "type": "object", - "properties": { - "type": { - "type": "string", - "visibility": "backend" - }, - "google": { + "oneOf": [ + { "type": "object", "properties": { - "pathToKey": { + "type": { "type": "string", - "visibility": "secret" + "const": "local", + "visibility": "backend" + } + } + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "const": "googleGcs", + "visibility": "backend" }, - "projectId": { - "type": "string", - "visibility": "secret" - }, - "bucketName": { - "type": "string", - "visibility": "secret" + "googleGcs": { + "type": "object", + "properties": { + "pathToKey": { + "type": "string", + "visibility": "secret" + }, + "projectId": { + "type": "string", + "visibility": "secret" + }, + "bucketName": { + "type": "string", + "visibility": "secret" + } + } } } } - }, - "required": [ - "type" ] } }, From 511f59f021b070cc9fcfd5b00fab32b0eb464d34 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Wed, 9 Dec 2020 20:09:51 +0100 Subject: [PATCH 039/188] 1. Use $file for googleGcs.credentials 2. Clarify that techdocs-common is only used in backend plugin since it is compiled to commonJS --- docs/features/techdocs/configuration.md | 3 ++- docs/features/techdocs/using-cloud-storage.md | 8 ++++--- packages/techdocs-common/README.md | 2 +- packages/techdocs-common/package.json | 2 +- .../src/stages/publish/googleStorage.test.ts | 10 +-------- .../src/stages/publish/googleStorage.ts | 21 ++++++++++++++----- .../src/stages/publish/publish.test.ts | 9 +------- plugins/techdocs/package.json | 2 +- 8 files changed, 28 insertions(+), 29 deletions(-) diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index eac6799317..b3cb349778 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -61,7 +61,8 @@ techdocs: googleGcs: # An API key is required to write to a storage bucket. - pathToKey: '/path/to/google_application_credentials.json', + credentials: + $file: '/path/to/google_application_credentials.json', # Your GCP Project ID where the Cloud Storage Bucket is hosted. projectId: 'gcp-project-id' diff --git a/docs/features/techdocs/using-cloud-storage.md b/docs/features/techdocs/using-cloud-storage.md index 12e3acd1bf..55352b6c00 100644 --- a/docs/features/techdocs/using-cloud-storage.md +++ b/docs/features/techdocs/using-cloud-storage.md @@ -58,7 +58,7 @@ IAM & Admin console), and create a new key. Use JSON format for the key. A `.json` file will be downloaded. This is the secret key TechDocs will use to make API calls. Make it available in your Backstage server and/or your local development server and set it in the app config -`techdocs.publisher.googleGcs.pathToKey`. +`techdocs.publisher.googleGcs.credentials`. ```yaml techdocs: @@ -66,7 +66,8 @@ techdocs: type: 'googleGcs' googleGcs: projectId: 'gcp-project-id' - pathToKey: '/path/to/google_application_credentials.json' + credentials: + $file: '/path/to/google_application_credentials.json' ``` **4. GCS Bucket** @@ -83,7 +84,8 @@ techdocs: type: 'googleGcs' googleGcs: projectId: 'gcp-project-id' - pathToKey: '/path/to/google_application_credentials.json' + credentials: + $file: '/path/to/google_application_credentials.json' bucketName: 'name-of-techdocs-storage-bucket' ``` diff --git a/packages/techdocs-common/README.md b/packages/techdocs-common/README.md index 2940df2524..e4889d6f79 100644 --- a/packages/techdocs-common/README.md +++ b/packages/techdocs-common/README.md @@ -1,6 +1,6 @@ # @backstage/techdocs-common -Common functionalities for TechDocs, to be shared between techdocs plugins and techdocs-cli +Common functionalities for TechDocs, to be shared between techdocs-backend plugin and techdocs-cli This package is used by `techdocs-backend` to serve docs from different types of publishers (Google GCS, Local, etc.). It is also used to build docs and publish them to storage, by both `techdocs-backend` and `techdocs-cli`. diff --git a/packages/techdocs-common/package.json b/packages/techdocs-common/package.json index eb917edc2b..1bec23c17e 100644 --- a/packages/techdocs-common/package.json +++ b/packages/techdocs-common/package.json @@ -1,6 +1,6 @@ { "name": "@backstage/techdocs-common", - "description": "Common functionalities for TechDocs, to be shared between techdocs plugins and techdocs-cli", + "description": "Common functionalities for TechDocs, to be shared between techdocs-backend plugin and techdocs-cli", "version": "0.1.1", "main": "src/index.ts", "types": "src/index.ts", diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts index fa7bddc648..a61dcc2767 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts @@ -38,10 +38,6 @@ jest.spyOn(logger, 'info').mockReturnValue(logger); let publisher: PublisherBase; beforeEach(() => { - mockFs({ - '/path/to/google-application-credentials.json': '{}', - }); - const mockConfig = ConfigReader.fromConfigs([ { context: '', @@ -51,7 +47,7 @@ beforeEach(() => { publisher: { type: 'googleGcs', googleGcs: { - pathToKey: '/path/to/google-application-credentials.json', + credentials: '{}', projectId: 'gcp-project-id', bucketName: 'bucketName', }, @@ -64,10 +60,6 @@ beforeEach(() => { publisher = GoogleGCSPublish.fromConfig(mockConfig, logger); }); -afterEach(() => { - mockFs.restore(); -}); - describe('GoogleGCSPublish', () => { it('should publish a directory', () => { mockFs({ diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index c9b4ffd3db..d6868c1103 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -25,24 +25,35 @@ import { PublisherBase, PublishRequest } from './types'; export class GoogleGCSPublish implements PublisherBase { static fromConfig(config: Config, logger: Logger): PublisherBase { - let pathToKey = ''; + let credentials = ''; let projectId = ''; let bucketName = ''; try { - pathToKey = config.getString('techdocs.publisher.googleGcs.pathToKey'); + credentials = config.getString( + 'techdocs.publisher.googleGcs.credentials', + ); projectId = config.getString('techdocs.publisher.googleGcs.projectId'); bucketName = config.getString('techdocs.publisher.googleGcs.bucketName'); } catch (error) { throw new Error( "Since techdocs.publisher.type is set to 'googleGcs' in your app config, " + - 'pathToKey, projectId and bucketName are required in techdocs.publisher.googleGcs ' + + 'credentials, projectId and bucketName are required in techdocs.publisher.googleGcs ' + 'required to authenticate with Google Cloud Storage.', ); } + let credentialsJson = {}; + try { + credentialsJson = JSON.parse(credentials); + } catch (err) { + throw new Error( + 'Error in parsing techdocs.publisher.googleGcs.credentials config to JSON.', + ); + } + const storageClient = new Storage({ + credentials: credentialsJson, projectId: projectId, - keyFilename: pathToKey, }); // Check if the defined bucket exists. Being able to connect means the configuration is good @@ -59,7 +70,7 @@ export class GoogleGCSPublish implements PublisherBase { logger.error( `Could not retrieve metadata about the GCS bucket ${bucketName} in the GCP project ${projectId}. ` + 'Make sure the GCP project and the bucket exists and the access key located at the path ' + - "techdocs.publisher.googleGcs.pathToKey defined in app config has the role 'Storage Object Creator'. " + + "techdocs.publisher.googleGcs.credentials defined in app config has the role 'Storage Object Creator'. " + 'Refer to https://backstage.io/docs/features/techdocs/using-cloud-storage', ); throw new Error(`from GCS client library: ${reason.message}`); diff --git a/packages/techdocs-common/src/stages/publish/publish.test.ts b/packages/techdocs-common/src/stages/publish/publish.test.ts index 16e2225e46..244a606129 100644 --- a/packages/techdocs-common/src/stages/publish/publish.test.ts +++ b/packages/techdocs-common/src/stages/publish/publish.test.ts @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import mockFs from 'mock-fs'; import { getVoidLogger, PluginEndpointDiscovery, @@ -66,10 +65,6 @@ describe('Publisher', () => { }); it('should create google gcs publisher from config', () => { - mockFs({ - '/path/to/google-application-credentials.json': '{}', - }); - const mockConfig = ConfigReader.fromConfigs([ { context: '', @@ -79,7 +74,7 @@ describe('Publisher', () => { publisher: { type: 'googleGcs', googleGcs: { - pathToKey: '/path/to/google-application-credentials.json', + credentials: '{}', projectId: 'gcp-project-id', bucketName: 'bucketName', }, @@ -91,7 +86,5 @@ describe('Publisher', () => { const publisher = Publisher.fromConfig(mockConfig, logger, testDiscovery); expect(publisher).toBeInstanceOf(GoogleGCSPublish); - - mockFs.restore(); }); }); diff --git a/plugins/techdocs/package.json b/plugins/techdocs/package.json index da4c239f36..5b95bd64d2 100644 --- a/plugins/techdocs/package.json +++ b/plugins/techdocs/package.json @@ -107,7 +107,7 @@ "googleGcs": { "type": "object", "properties": { - "pathToKey": { + "credentials": { "type": "string", "visibility": "secret" }, From 92ecddbe24ab150fddb52546d20560d501f1e429 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Wed, 9 Dec 2020 20:32:41 +0100 Subject: [PATCH 040/188] techdocs-common: Unify response type of Local and googleGcs publisher --- docs/features/techdocs/architecture.md | 12 ++++++------ .../src/stages/publish/googleStorage.test.ts | 2 +- .../src/stages/publish/googleStorage.ts | 4 ++-- packages/techdocs-common/src/stages/publish/local.ts | 7 ++----- packages/techdocs-common/src/stages/publish/types.ts | 5 ++++- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/features/techdocs/architecture.md b/docs/features/techdocs/architecture.md index 87fe969d46..3969002a46 100644 --- a/docs/features/techdocs/architecture.md +++ b/docs/features/techdocs/architecture.md @@ -50,8 +50,8 @@ built. We assume each entity lives in a repository somewhere (GitHub, GitLab, etc.). We recommend using a CI/CD pipeline with the repository that has a dedicated -step/job to generate docs for TechDocs. The generated static files are then stored -in a cloud storage solution of your choice. +step/job to generate docs for TechDocs. The generated static files are then +stored in a cloud storage solution of your choice. [Track progress here](https://github.com/backstage/backstage/issues/3096). Similar to how it is done in the Basic setup, the TechDocs Reader requests @@ -62,8 +62,8 @@ TechDocs Reader. We will provide instructions, scripts and/or templates (e.g. GitHub Actions) to generate docs in your CI/CD system. [Track progress here.](https://github.com/backstage/backstage/issues/3400) You -will be able to use `techdocs-cli` to generate docs and publish the generated docs -site files to your cloud storage system. +will be able to use `techdocs-cli` to generate docs and publish the generated +docs site files to your cloud storage system. Note about caching: We have noticed internally that some storage providers can be quite slow, which is why we are recommending a cache that sits between the @@ -120,8 +120,8 @@ docs site in real-time?** A: Generating the content from Markdown on the fly is not optimal (although that is how the basic out-of-the-box setup is implemented). Storage solutions act as a cache for the generated static content. TechDocs is also currently built on -MkDocs which does not allow us to generate docs per-page, so we would have to build -all docs for a entity on every request. +MkDocs which does not allow us to generate docs per-page, so we would have to +build all docs for a entity on every request. # Future work diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts index a61dcc2767..6ab208ad03 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts @@ -75,6 +75,6 @@ describe('GoogleGCSPublish', () => { const entity = createMockEntity(); return expect( publisher.publish({ entity, directory: '/path/to/generatedDirectory' }), - ).resolves.toStrictEqual({}); + ).resolves.toBeUndefined(); }); }); diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index d6868c1103..342b0a2687 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -93,7 +93,7 @@ export class GoogleGCSPublish implements PublisherBase { * Upload all the files from the generated `directory` to the GCS bucket. * Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html */ - publish({ entity, directory }: PublishRequest): Promise<{}> { + publish({ entity, directory }: PublishRequest): Promise { return new Promise((resolve, reject) => { // Path of all files to upload, relative to the root of the source directory // e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png'] @@ -140,7 +140,7 @@ export class GoogleGCSPublish implements PublisherBase { this.logger.info( `Successfully uploaded all the generated files for Entity ${entityRootDir}. Total number of files: ${allFilesToUpload.length}`, ); - resolve({}); + resolve(undefined); }); }); } diff --git a/packages/techdocs-common/src/stages/publish/local.ts b/packages/techdocs-common/src/stages/publish/local.ts index 7bb670f2d4..bc24867f50 100644 --- a/packages/techdocs-common/src/stages/publish/local.ts +++ b/packages/techdocs-common/src/stages/publish/local.ts @@ -23,10 +23,7 @@ import { PluginEndpointDiscovery, } from '@backstage/backend-common'; import { Config } from '@backstage/config'; -import { PublisherBase, PublishRequest } from './types'; - -/* `remoteUrl` is the URL which serves files from the local publisher's static directory. */ -export type LocalPublishReturn = Promise<{ remoteUrl: string }>; +import { PublisherBase, PublishRequest, PublishResponse } from './types'; const staticDocsDir = resolvePackagePath( '@backstage/plugin-techdocs-backend', @@ -52,7 +49,7 @@ export class LocalPublish implements PublisherBase { this.discovery = discovery; } - publish({ entity, directory }: PublishRequest): LocalPublishReturn { + publish({ entity, directory }: PublishRequest): Promise { const entityNamespace = entity.metadata.namespace ?? 'default'; const publishDir = resolvePackagePath( diff --git a/packages/techdocs-common/src/stages/publish/types.ts b/packages/techdocs-common/src/stages/publish/types.ts index 6722f6c221..9bfd8cb334 100644 --- a/packages/techdocs-common/src/stages/publish/types.ts +++ b/packages/techdocs-common/src/stages/publish/types.ts @@ -27,7 +27,10 @@ export type PublishRequest = { directory: string; }; -export type PublishResponse = {}; +/* `remoteUrl` is the URL which serves files from the local publisher's static directory. */ +export type PublishResponse = { + remoteUrl?: string; +} | void; /** * Base class for a TechDocs publisher (e.g. Local, Google GCS Bucket, AWS S3, etc.) From 59a879115887bc9f9d79f941572ce6668911c63d Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Wed, 9 Dec 2020 15:51:16 -0500 Subject: [PATCH 041/188] Fix comment typos --- .../catalog/src/components/EntityPageLayout/Tabbed/Tabbed.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/catalog/src/components/EntityPageLayout/Tabbed/Tabbed.tsx b/plugins/catalog/src/components/EntityPageLayout/Tabbed/Tabbed.tsx index 096b2fa708..9a4e872d8f 100644 --- a/plugins/catalog/src/components/EntityPageLayout/Tabbed/Tabbed.tsx +++ b/plugins/catalog/src/components/EntityPageLayout/Tabbed/Tabbed.tsx @@ -39,8 +39,8 @@ const getSelectedIndexOrDefault = ( /** * Compound component, which allows you to define layout - * for EntityPage using Tabs as a subnavigation mechanism - * Constists of 2 parts: Tabbed.Layout and Tabbed.Content. + * for EntityPage using Tabs as a sub-navigation mechanism + * Consists of 2 parts: Tabbed.Layout and Tabbed.Content. * Takes care of: tabs, routes, document titles, spacing around content * * @example From 6fde301a993c0342581b98878a4f9129ba59a927 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Wed, 9 Dec 2020 15:56:27 -0500 Subject: [PATCH 042/188] FIx unneeded defensive code --- packages/config/src/reader.ts | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/packages/config/src/reader.ts b/packages/config/src/reader.ts index eb8c91e366..cd0a6f3221 100644 --- a/packages/config/src/reader.ts +++ b/packages/config/src/reader.ts @@ -272,23 +272,17 @@ export class ConfigReader implements Config { if (value === undefined) { return this.fallback?.readConfigValue(key, validate); } - if (value !== undefined) { - const result = validate(value); - if (result !== true) { - const { - key: keyName = key, - value: theValue = value, + const result = validate(value); + if (result !== true) { + const { key: keyName = key, value: theValue = value, expected } = result; + throw new TypeError( + errors.type( + this.fullKey(keyName), + this.context, + typeOf(theValue), expected, - } = result; - throw new TypeError( - errors.type( - this.fullKey(keyName), - this.context, - typeOf(theValue), - expected, - ), - ); - } + ), + ); } return value as T; From a6b8d6163d742767ed398286e4fa3d00b7d76a63 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Wed, 9 Dec 2020 16:02:52 -0500 Subject: [PATCH 043/188] Fix useless conditional --- plugins/catalog/src/components/Router.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog/src/components/Router.tsx b/plugins/catalog/src/components/Router.tsx index fe1ba3a593..298aa28d52 100644 --- a/plugins/catalog/src/components/Router.tsx +++ b/plugins/catalog/src/components/Router.tsx @@ -50,7 +50,7 @@ const EntityPageSwitch = ({ EntityPage }: { EntityPage: ComponentType }) => { const { entity, loading, error } = useEntity(); // Loading and error states if (loading) return ; - if (error || (!loading && !entity)) return ; + if (error || !entity) return ; // Otherwise EntityPage provided from the App // Note that EntityPage will include EntityPageLayout already From e3bd9fc2f480b2bfc8326c9598d5b6c87638cf6a Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Wed, 9 Dec 2020 16:12:43 -0500 Subject: [PATCH 044/188] Add changesets --- .changeset/poor-spies-grin.md | 5 +++++ .changeset/violet-windows-run.md | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 .changeset/poor-spies-grin.md create mode 100644 .changeset/violet-windows-run.md diff --git a/.changeset/poor-spies-grin.md b/.changeset/poor-spies-grin.md new file mode 100644 index 0000000000..cab5df9029 --- /dev/null +++ b/.changeset/poor-spies-grin.md @@ -0,0 +1,5 @@ +--- +'@backstage/config': patch +--- + +Fix unneeded defensive code diff --git a/.changeset/violet-windows-run.md b/.changeset/violet-windows-run.md new file mode 100644 index 0000000000..5f385d4dfe --- /dev/null +++ b/.changeset/violet-windows-run.md @@ -0,0 +1,5 @@ +--- +'@backstage/config': patch +--- + +Fix useless conditional From a48f2704c91670713e9fe80abea19c54234fdde1 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Wed, 9 Dec 2020 16:17:48 -0500 Subject: [PATCH 045/188] Fix string template literal --- .../src/ingestion/processors/microsoftGraph/client.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.ts b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.ts index 6dded56aa1..9db76542f4 100644 --- a/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.ts +++ b/plugins/catalog-backend/src/ingestion/processors/microsoftGraph/client.ts @@ -184,7 +184,7 @@ export class MicrosoftGraphClient { const response = await this.requestApi(`organization/${tenantId}`); if (response.status !== 200) { - await this.handleError('organization/${tenantId}', response); + await this.handleError(`organization/${tenantId}`, response); } return await response.json(); From 38d63fbe1cc04cf8cc7e272865e179e72128cf73 Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Wed, 9 Dec 2020 16:18:43 -0500 Subject: [PATCH 046/188] Add catalog-backend changeset --- .changeset/seven-tigers-mate.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/seven-tigers-mate.md diff --git a/.changeset/seven-tigers-mate.md b/.changeset/seven-tigers-mate.md new file mode 100644 index 0000000000..c48bcf649a --- /dev/null +++ b/.changeset/seven-tigers-mate.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Fix string template literal From 9507d72a38d6ae0f5846501758665042fb6ccb64 Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Thu, 10 Dec 2020 00:09:30 +0100 Subject: [PATCH 047/188] 1. Replace klaw with recursive-readdir 2. Break out reading files logic to a helper function (add tests) 3. Use Promise.all to make sure all files are uploaded --- packages/techdocs-common/package.json | 5 +- .../src/stages/publish/googleStorage.ts | 77 ++++++++----------- .../src/stages/publish/helpers.test.ts | 43 +++++++++++ .../src/stages/publish/helpers.ts | 36 +++++++++ yarn.lock | 23 +----- 5 files changed, 114 insertions(+), 70 deletions(-) create mode 100644 packages/techdocs-common/src/stages/publish/helpers.test.ts diff --git a/packages/techdocs-common/package.json b/packages/techdocs-common/package.json index 1bec23c17e..141921a0cb 100644 --- a/packages/techdocs-common/package.json +++ b/packages/techdocs-common/package.json @@ -48,14 +48,13 @@ "fs-extra": "^9.0.1", "git-url-parse": "^11.4.0", "js-yaml": "^3.14.0", - "klaw": "^3.0.0", "mock-fs": "^4.13.0", "nodegit": "^0.27.0", + "recursive-readdir": "^2.2.2", "winston": "^3.2.1" }, "devDependencies": { - "@backstage/cli": "^0.4.0", - "@types/klaw": "^3.0.1" + "@backstage/cli": "^0.4.0" }, "jest": { "roots": [ diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index 342b0a2687..21742a272e 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -13,14 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import path from 'path'; import express from 'express'; -import walk from 'klaw'; import { Storage, UploadResponse } from '@google-cloud/storage'; import { Logger } from 'winston'; import { Entity, EntityName } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; -import { getHeadersForFileExtension, supportedFileType } from './helpers'; +import { + getHeadersForFileExtension, + supportedFileType, + getFileTreeRecursively, +} from './helpers'; import { PublisherBase, PublishRequest } from './types'; export class GoogleGCSPublish implements PublisherBase { @@ -94,53 +96,38 @@ export class GoogleGCSPublish implements PublisherBase { * Directory structure used in the bucket is - entityNamespace/entityKind/entityName/index.html */ publish({ entity, directory }: PublishRequest): Promise { - return new Promise((resolve, reject) => { - // Path of all files to upload, relative to the root of the source directory - // e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png'] - const allFilesToUpload: Array = []; + return new Promise(async (resolve, reject) => { + // Note: GCS manages creation of parent directories if they do not exist. + // So collecting path of only the files is good enough. + const allFilesToUpload = await getFileTreeRecursively(directory); - // Iterate on all the files in the directory and its sub-directories - walk(directory) - .on('data', (item: walk.Item) => { - // GCS manages creation of parent directories if they do not exist. - // So collecting path of only the files is good enough. - if (item.stats.isFile()) { - // Remove the absolute path prefix of the source directory - const relativeFilePath = item.path.replace(`${directory}/`, ''); - allFilesToUpload.push(relativeFilePath); - } - }) - .on('error', (err: Error, item: walk.Item) => { - const errorMessage = `Unable to read file at ${item.path}. Error ${err.message}`; - this.logger.error(errorMessage); - reject(errorMessage); - }) - .on('end', () => { - // 'end' event happens when all the files have been read. - const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; - allFilesToUpload.forEach(filePath => { - const source = path.join(directory, filePath); // Local file absolute path - const destination = `${entityRootDir}/${filePath}`; // GCS Bucket file relative path - this.storageClient - .bucket(this.bucketName) - .upload(source, { destination }) - .then( - (uploadResp: UploadResponse) => ({ - fileName: destination, - status: uploadResp[0], - }), - (err: Error) => { - const errorMessage = `Unable to upload file ${destination} to GCS. Error ${err.message}`; - this.logger.error(errorMessage); - reject(errorMessage); - }, - ); - }); + const uploadPromises: Array> = []; + allFilesToUpload.forEach(filePath => { + // Remove the absolute path prefix of the source directory + // Path of all files to upload, relative to the root of the source directory + // e.g. ['index.html', 'sub-page/index.html', 'assets/images/favicon.png'] + const relativeFilePath = filePath.replace(`${directory}/`, ''); + const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + const destination = `${entityRootDir}/${relativeFilePath}`; // GCS Bucket file relative path + // TODO: Upload in chunks of ~10 files instead of all files at once. + uploadPromises.push( + this.storageClient.bucket(this.bucketName).upload(filePath, { + destination, + }), + ); + }); + Promise.all(uploadPromises) + .then(() => { this.logger.info( - `Successfully uploaded all the generated files for Entity ${entityRootDir}. Total number of files: ${allFilesToUpload.length}`, + `Successfully uploaded all the generated files for Entity ${entity.metadata.name}. Total number of files: ${allFilesToUpload.length}`, ); resolve(undefined); + }) + .catch((err: Error) => { + const errorMessage = `Unable to upload file(s) to Google Cloud Storage. Error ${err.message}`; + this.logger.error(errorMessage); + reject(errorMessage); }); }); } diff --git a/packages/techdocs-common/src/stages/publish/helpers.test.ts b/packages/techdocs-common/src/stages/publish/helpers.test.ts new file mode 100644 index 0000000000..2f8fbadae0 --- /dev/null +++ b/packages/techdocs-common/src/stages/publish/helpers.test.ts @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import mockFs from 'mock-fs'; +import { getFileTreeRecursively } from './helpers'; + +describe('getFileTreeRecursively', () => { + beforeEach(() => { + mockFs({ + '/rootDir': { + file1: '', + subDirA: { + file2: '', + emptyDir1: mockFs.directory(), + }, + emptyDir2: mockFs.directory(), + }, + }); + }); + + afterEach(() => { + mockFs.restore(); + }); + + it('returns complete file tree of a path', async () => { + const fileList = await getFileTreeRecursively('/rootDir'); + expect(fileList.length).toBe(2); + expect(fileList).toContain('/rootDir/file1'); + expect(fileList).toContain('/rootDir/subDirA/file2'); + }); +}); diff --git a/packages/techdocs-common/src/stages/publish/helpers.ts b/packages/techdocs-common/src/stages/publish/helpers.ts index cca5d85150..2939bbfc1c 100644 --- a/packages/techdocs-common/src/stages/publish/helpers.ts +++ b/packages/techdocs-common/src/stages/publish/helpers.ts @@ -13,6 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import recursiveReadDir from 'recursive-readdir'; + export type supportedFileType = 'html' | 'css'; export type responseHeadersType = { @@ -44,3 +46,37 @@ export const getHeadersForFileExtension = ( return headersCommon; } }; + +/** + * Recursively traverse all the sub-directories of a path and return + * a list of absolute paths of all the files. e.g. tree command in Unix + * + * @example + * + * /User/username/my_dir + * dirA + * | subDirA + * | | file1 + * EmptyDir + * dirB + * | file2 + * file3 + * + * getFileListRecursively('/Users/username/myDir') + * // returns + * [ + * '/User/username/my_dir/dirA/subDirA/file1', + * '/User/username/my_dir/dirB/file2', + * '/User/username/my_dir/file3' + * ] + * @param rootDirPath Absolute path to the root directory. + */ +export const getFileTreeRecursively = async ( + rootDirPath: string, +): Promise => { + // Iterate on all the files in the directory and its sub-directories + const fileList = await recursiveReadDir(rootDirPath).catch(error => { + throw new Error(`Failed to read template directory: ${error.message}`); + }); + return fileList; +}; diff --git a/yarn.lock b/yarn.lock index 399a84dd17..e2cfe16504 100644 --- a/yarn.lock +++ b/yarn.lock @@ -844,14 +844,7 @@ dependencies: "@babel/helper-plugin-utils" "^7.8.0" -"@babel/plugin-syntax-top-level-await@^7.12.1": - version "7.12.1" - resolved "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.12.1.tgz#dd6c0b357ac1bb142d98537450a319625d13d2a0" - integrity sha512-i7ooMZFS+a/Om0crxZodrTzNEPJHZrlMVGMTEpFAj6rYY/bKCddB0Dk/YxfPuYXOopuhKk/e1jV6h+WUU9XN3A== - dependencies: - "@babel/helper-plugin-utils" "^7.10.4" - -"@babel/plugin-syntax-top-level-await@^7.8.3": +"@babel/plugin-syntax-top-level-await@^7.12.1", "@babel/plugin-syntax-top-level-await@^7.8.3": version "7.12.1" resolved "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.12.1.tgz#dd6c0b357ac1bb142d98537450a319625d13d2a0" integrity sha512-i7ooMZFS+a/Om0crxZodrTzNEPJHZrlMVGMTEpFAj6rYY/bKCddB0Dk/YxfPuYXOopuhKk/e1jV6h+WUU9XN3A== @@ -5894,13 +5887,6 @@ dependencies: "@types/node" "*" -"@types/klaw@^3.0.1": - version "3.0.1" - resolved "https://registry.npmjs.org/@types/klaw/-/klaw-3.0.1.tgz#29f90021c0234976aa4eb97efced9cb6db9fa8b3" - integrity sha512-acnF3n9mYOr1aFJKFyvfNX0am9EtPUsYPq22QUCGdJE+MVt6UyAN1jwo+PmOPqXD4K7ZS9MtxDEp/un0lxFccA== - dependencies: - "@types/node" "*" - "@types/koa-compose@*": version "3.2.5" resolved "https://registry.npmjs.org/@types/koa-compose/-/koa-compose-3.2.5.tgz#85eb2e80ac50be95f37ccf8c407c09bbe3468e9d" @@ -16255,13 +16241,6 @@ klaw@^1.0.0: optionalDependencies: graceful-fs "^4.1.9" -klaw@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/klaw/-/klaw-3.0.0.tgz#b11bec9cf2492f06756d6e809ab73a2910259146" - integrity sha512-0Fo5oir+O9jnXu5EefYbVK+mHMBeEVEy2cmctR1O1NECcCkPRreJKrS6Qt/j3KC2C148Dfo9i3pCmCMsdqGr0g== - dependencies: - graceful-fs "^4.1.9" - kleur@^3.0.3: version "3.0.3" resolved "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz#a79c9ecc86ee1ce3fa6206d1216c501f147fc07e" From 8ef71ed32e8c3cd7eb6f4533fc4c476023da5b24 Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Wed, 9 Dec 2020 15:59:22 +0100 Subject: [PATCH 048/188] Add a Avatar component to @backstage/core --- .changeset/gorgeous-scissors-jog.md | 6 +++ .../src/components/Avatar/Avatar.stories.tsx | 42 +++++++++++++++++++ .../src/components/Avatar/Avatar.test.tsx | 27 ++++++++++++ .../core}/src/components/Avatar/Avatar.tsx | 30 ++++--------- .../core}/src/components/Avatar/index.ts | 0 .../core/src/components/Avatar/util.test.ts | 37 ++++++++++++++++ packages/core/src/components/Avatar/utils.ts | 32 ++++++++++++++ packages/core/src/components/index.ts | 1 + .../Group/MembersList/MembersListCard.tsx | 21 +++++----- .../User/UserProfileCard/UserProfileCard.tsx | 13 +++--- 10 files changed, 169 insertions(+), 40 deletions(-) create mode 100644 .changeset/gorgeous-scissors-jog.md create mode 100644 packages/core/src/components/Avatar/Avatar.stories.tsx create mode 100644 packages/core/src/components/Avatar/Avatar.test.tsx rename {plugins/org => packages/core}/src/components/Avatar/Avatar.tsx (67%) rename {plugins/org => packages/core}/src/components/Avatar/index.ts (100%) create mode 100644 packages/core/src/components/Avatar/util.test.ts create mode 100644 packages/core/src/components/Avatar/utils.ts diff --git a/.changeset/gorgeous-scissors-jog.md b/.changeset/gorgeous-scissors-jog.md new file mode 100644 index 0000000000..3f192808c1 --- /dev/null +++ b/.changeset/gorgeous-scissors-jog.md @@ -0,0 +1,6 @@ +--- +'@backstage/core': patch +'@backstage/plugin-org': patch +--- + +Add a `` component to `@backstage/core`. diff --git a/packages/core/src/components/Avatar/Avatar.stories.tsx b/packages/core/src/components/Avatar/Avatar.stories.tsx new file mode 100644 index 0000000000..5ac628d72b --- /dev/null +++ b/packages/core/src/components/Avatar/Avatar.stories.tsx @@ -0,0 +1,42 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import { Avatar } from './Avatar'; + +export default { + title: 'Data Display/Avatar', + component: Avatar, +}; + +export const Default = () => ( + +); + +export const NameFallback = () => ; + +export const Empty = () => ; + +export const CustomStyling = () => ( + +); diff --git a/packages/core/src/components/Avatar/Avatar.test.tsx b/packages/core/src/components/Avatar/Avatar.test.tsx new file mode 100644 index 0000000000..da6ca8f42e --- /dev/null +++ b/packages/core/src/components/Avatar/Avatar.test.tsx @@ -0,0 +1,27 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { render } from '@testing-library/react'; +import React from 'react'; +import { Avatar } from './Avatar'; + +describe('', () => { + it('renders without exploding', async () => { + const { getByText } = render(); + + expect(getByText('JD')).toBeInTheDocument(); + }); +}); diff --git a/plugins/org/src/components/Avatar/Avatar.tsx b/packages/core/src/components/Avatar/Avatar.tsx similarity index 67% rename from plugins/org/src/components/Avatar/Avatar.tsx rename to packages/core/src/components/Avatar/Avatar.tsx index 637973ec3b..95aa4a8ced 100644 --- a/plugins/org/src/components/Avatar/Avatar.tsx +++ b/packages/core/src/components/Avatar/Avatar.tsx @@ -20,6 +20,7 @@ import { makeStyles, Theme, } from '@material-ui/core'; +import { extractInitials, stringToColor } from './utils'; const useStyles = makeStyles((theme: Theme) => createStyles({ @@ -34,28 +35,13 @@ const useStyles = makeStyles((theme: Theme) => }), ); -const stringToColour = (str: string) => { - let hash = 0; - for (let i = 0; i < str.length; i++) { - hash = str.charCodeAt(i) + ((hash << 5) - hash); - } - let colour = '#'; - for (let i = 0; i < 3; i++) { - const value = (hash >> (i * 8)) & 0xff; - colour += `00${value.toString(16)}`.substr(-2); - } - return colour; +export type AvatarProps = { + displayName?: string; + picture?: string; + customStyles?: CSSProperties; }; -export const Avatar = ({ - displayName, - picture, - customStyles, -}: { - displayName: string | undefined; - picture: string | undefined; - customStyles?: CSSProperties; -}) => { +export const Avatar = ({ displayName, picture, customStyles }: AvatarProps) => { const classes = useStyles(); return ( - {displayName && displayName.match(/\b\w/g)!.join('').substring(0, 2)} + {displayName && extractInitials(displayName)} ); }; diff --git a/plugins/org/src/components/Avatar/index.ts b/packages/core/src/components/Avatar/index.ts similarity index 100% rename from plugins/org/src/components/Avatar/index.ts rename to packages/core/src/components/Avatar/index.ts diff --git a/packages/core/src/components/Avatar/util.test.ts b/packages/core/src/components/Avatar/util.test.ts new file mode 100644 index 0000000000..94de957e8e --- /dev/null +++ b/packages/core/src/components/Avatar/util.test.ts @@ -0,0 +1,37 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { extractInitials, stringToColor } from './utils'; + +describe('stringToColor', () => { + it('extract color', async () => { + expect(stringToColor('Jenny Doe')).toEqual('#7809fa'); + }); +}); + +describe('extractInitials', () => { + it('extract initials', async () => { + expect(extractInitials('Jenny Doe')).toEqual('JD'); + }); + + it('extract single letter for short name', async () => { + expect(extractInitials('Doe')).toEqual('D'); + }); + + it('limit the initials to two letters', async () => { + expect(extractInitials('John Jonathan Doe')).toEqual('JJ'); + }); +}); diff --git a/packages/core/src/components/Avatar/utils.ts b/packages/core/src/components/Avatar/utils.ts new file mode 100644 index 0000000000..5990a72955 --- /dev/null +++ b/packages/core/src/components/Avatar/utils.ts @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function stringToColor(str: string) { + let hash = 0; + for (let i = 0; i < str.length; i++) { + hash = str.charCodeAt(i) + ((hash << 5) - hash); + } + let color = '#'; + for (let i = 0; i < 3; i++) { + const value = (hash >> (i * 8)) & 0xff; + color += `00${value.toString(16)}`.substr(-2); + } + return color; +} + +export function extractInitials(value: string) { + return value.match(/\b\w/g)!.join('').substring(0, 2); +} diff --git a/packages/core/src/components/index.ts b/packages/core/src/components/index.ts index bcc56debea..a8ae0213c4 100644 --- a/packages/core/src/components/index.ts +++ b/packages/core/src/components/index.ts @@ -15,6 +15,7 @@ */ export * from './AlertDisplay'; +export * from './Avatar'; export * from './Button'; export * from './CodeSnippet'; export * from './CopyTextButton'; diff --git a/plugins/org/src/components/Cards/Group/MembersList/MembersListCard.tsx b/plugins/org/src/components/Cards/Group/MembersList/MembersListCard.tsx index cd39241ca8..f16f3701c2 100644 --- a/plugins/org/src/components/Cards/Group/MembersList/MembersListCard.tsx +++ b/plugins/org/src/components/Cards/Group/MembersList/MembersListCard.tsx @@ -13,8 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import React from 'react'; -import Alert from '@material-ui/lab/Alert'; +import { + Entity, + RELATION_MEMBER_OF, + UserEntity, +} from '@backstage/catalog-model'; +import { Avatar, InfoCard, Progress, useApi } from '@backstage/core'; +import { catalogApiRef, entityRouteParams } from '@backstage/plugin-catalog'; import { Box, createStyles, @@ -24,16 +29,10 @@ import { Theme, Typography, } from '@material-ui/core'; -import { InfoCard, Progress, useApi } from '@backstage/core'; -import { - UserEntity, - RELATION_MEMBER_OF, - Entity, -} from '@backstage/catalog-model'; -import { Link as RouterLink, generatePath } from 'react-router-dom'; -import { catalogApiRef, entityRouteParams } from '@backstage/plugin-catalog'; +import Alert from '@material-ui/lab/Alert'; +import React from 'react'; +import { generatePath, Link as RouterLink } from 'react-router-dom'; import { useAsync } from 'react-use'; -import { Avatar } from '../../../Avatar'; const useStyles = makeStyles((theme: Theme) => createStyles({ diff --git a/plugins/org/src/components/Cards/User/UserProfileCard/UserProfileCard.tsx b/plugins/org/src/components/Cards/User/UserProfileCard/UserProfileCard.tsx index 0f32835d70..889933a58b 100644 --- a/plugins/org/src/components/Cards/User/UserProfileCard/UserProfileCard.tsx +++ b/plugins/org/src/components/Cards/User/UserProfileCard/UserProfileCard.tsx @@ -13,21 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import React from 'react'; -import { Box, Grid, Link, Tooltip, Typography } from '@material-ui/core'; -import Alert from '@material-ui/lab/Alert'; -import { InfoCard } from '@backstage/core'; -import { entityRouteParams } from '@backstage/plugin-catalog'; import { Entity, RELATION_MEMBER_OF, UserEntity, } from '@backstage/catalog-model'; +import { Avatar, InfoCard } from '@backstage/core'; +import { entityRouteParams } from '@backstage/plugin-catalog'; +import { Box, Grid, Link, Tooltip, Typography } from '@material-ui/core'; import EmailIcon from '@material-ui/icons/Email'; import GroupIcon from '@material-ui/icons/Group'; import PersonIcon from '@material-ui/icons/Person'; -import { Link as RouterLink, generatePath } from 'react-router-dom'; -import { Avatar } from '../../../Avatar'; +import Alert from '@material-ui/lab/Alert'; +import React from 'react'; +import { generatePath, Link as RouterLink } from 'react-router-dom'; const GroupLink = ({ groupName, From 6e47fb65685b27807a568eca5eeead84dea60aa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikael=20=C3=96stberg?= Date: Wed, 9 Dec 2020 15:31:43 +0100 Subject: [PATCH 049/188] Use parseGitUri --- plugins/catalog-import/src/util/useGithubRepos.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/plugins/catalog-import/src/util/useGithubRepos.ts b/plugins/catalog-import/src/util/useGithubRepos.ts index 740f0581b1..30e77e18a8 100644 --- a/plugins/catalog-import/src/util/useGithubRepos.ts +++ b/plugins/catalog-import/src/util/useGithubRepos.ts @@ -18,6 +18,7 @@ import * as YAML from 'yaml'; import { useApi, configApiRef } from '@backstage/core'; import { catalogImportApiRef } from '../api/CatalogImportApi'; import { ConfigSpec } from '../components/ImportComponentPage'; +import parseGitUri from 'git-url-parse'; // TODO: (O5ten) Refactor into a core API instead of direct usage like this // https://github.com/backstage/backstage/pull/3613#issuecomment-7408929430 @@ -28,9 +29,12 @@ export function useGithubRepos() { const config = useApi(configApiRef); const submitPrToRepo = async (selectedRepo: ConfigSpec) => { - const [hostname, ownerName, repoName] = selectedRepo.location - .split('/') - .slice(-3); + const { + name: repoName, + owner: ownerName, + resource: hostname, + } = parseGitUri(selectedRepo.location); + const configs = readGitHubIntegrationConfigs( config.getOptionalConfigArray('integrations.github') ?? [], ); From 9724bc1a12ead22a29b912307d79f11dbbb26b6d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 10 Dec 2020 10:37:40 +0100 Subject: [PATCH 050/188] build(deps-dev): bump nodemon from 2.0.4 to 2.0.6 (#3658) Bumps [nodemon](https://github.com/remy/nodemon) from 2.0.4 to 2.0.6. - [Release notes](https://github.com/remy/nodemon/releases) - [Commits](https://github.com/remy/nodemon/compare/v2.0.4...v2.0.6) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- yarn.lock | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/yarn.lock b/yarn.lock index 73405beac2..3e95d570a0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -844,14 +844,7 @@ dependencies: "@babel/helper-plugin-utils" "^7.8.0" -"@babel/plugin-syntax-top-level-await@^7.12.1": - version "7.12.1" - resolved "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.12.1.tgz#dd6c0b357ac1bb142d98537450a319625d13d2a0" - integrity sha512-i7ooMZFS+a/Om0crxZodrTzNEPJHZrlMVGMTEpFAj6rYY/bKCddB0Dk/YxfPuYXOopuhKk/e1jV6h+WUU9XN3A== - dependencies: - "@babel/helper-plugin-utils" "^7.10.4" - -"@babel/plugin-syntax-top-level-await@^7.8.3": +"@babel/plugin-syntax-top-level-await@^7.12.1", "@babel/plugin-syntax-top-level-await@^7.8.3": version "7.12.1" resolved "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.12.1.tgz#dd6c0b357ac1bb142d98537450a319625d13d2a0" integrity sha512-i7ooMZFS+a/Om0crxZodrTzNEPJHZrlMVGMTEpFAj6rYY/bKCddB0Dk/YxfPuYXOopuhKk/e1jV6h+WUU9XN3A== @@ -17930,9 +17923,9 @@ nodegit@0.27.0, nodegit@^0.27.0: tar-fs "^1.16.3" nodemon@^2.0.2: - version "2.0.4" - resolved "https://registry.npmjs.org/nodemon/-/nodemon-2.0.4.tgz#55b09319eb488d6394aa9818148c0c2d1c04c416" - integrity sha512-Ltced+hIfTmaS28Zjv1BM552oQ3dbwPqI4+zI0SLgq+wpJhSyqgYude/aZa/3i31VCQWMfXJVxvu86abcam3uQ== + version "2.0.6" + resolved "https://registry.npmjs.org/nodemon/-/nodemon-2.0.6.tgz#1abe1937b463aaf62f0d52e2b7eaadf28cc2240d" + integrity sha512-4I3YDSKXg6ltYpcnZeHompqac4E6JeAMpGm8tJnB9Y3T0ehasLa4139dJOcCrB93HHrUMsCrKtoAlXTqT5n4AQ== dependencies: chokidar "^3.2.2" debug "^3.2.6" @@ -17942,8 +17935,8 @@ nodemon@^2.0.2: semver "^5.7.1" supports-color "^5.5.0" touch "^3.1.0" - undefsafe "^2.0.2" - update-notifier "^4.0.0" + undefsafe "^2.0.3" + update-notifier "^4.1.0" "nopt@2 || 3": version "3.0.6" @@ -24060,7 +24053,7 @@ unc-path-regex@^0.1.2: resolved "https://registry.npmjs.org/unc-path-regex/-/unc-path-regex-0.1.2.tgz#e73dd3d7b0d7c5ed86fbac6b0ae7d8c6a69d50fa" integrity sha1-5z3T17DXxe2G+6xrCufYxqadUPo= -undefsafe@^2.0.2: +undefsafe@^2.0.3: version "2.0.3" resolved "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.3.tgz#6b166e7094ad46313b2202da7ecc2cd7cc6e7aae" integrity sha512-nrXZwwXrD/T/JXeygJqdCO6NZZ1L66HrxM/Z7mIq2oPanoN0F1nLx3lwJMu6AwJY69hdixaFQOuoYsMjE5/C2A== @@ -24272,10 +24265,10 @@ upath@^1.1.1, upath@^1.2.0: resolved "https://registry.npmjs.org/upath/-/upath-1.2.0.tgz#8f66dbcd55a883acdae4408af8b035a5044c1894" integrity sha512-aZwGpamFO61g3OlfT7OQCHqhGnW43ieH9WZeP7QxN/G/jS4jfqUkZxoryvJgVPEcrl5NL/ggHsSmLMHuH64Lhg== -update-notifier@^4.0.0: - version "4.1.0" - resolved "https://registry.npmjs.org/update-notifier/-/update-notifier-4.1.0.tgz#4866b98c3bc5b5473c020b1250583628f9a328f3" - integrity sha512-w3doE1qtI0/ZmgeoDoARmI5fjDoT93IfKgEGqm26dGUOh8oNpaSTsGNdYRN/SjOuo10jcJGwkEL3mroKzktkew== +update-notifier@^4.1.0: + version "4.1.3" + resolved "https://registry.npmjs.org/update-notifier/-/update-notifier-4.1.3.tgz#be86ee13e8ce48fb50043ff72057b5bd598e1ea3" + integrity sha512-Yld6Z0RyCYGB6ckIjffGOSOmHXj1gMeE7aROz4MG+XMkmixBX4jUngrGXNYz7wPKBmtoD4MnBa2Anu7RSKht/A== dependencies: boxen "^4.2.0" chalk "^3.0.0" From e1f4e24ef196341085588e6512de82ef6f57a2c8 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 10 Dec 2020 11:06:23 +0100 Subject: [PATCH 051/188] dev-utils,test-utils: move @backstage/cli to dev deps --- .changeset/popular-flies-swim.md | 6 ++++++ packages/dev-utils/package.json | 2 +- packages/test-utils/package.json | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 .changeset/popular-flies-swim.md diff --git a/.changeset/popular-flies-swim.md b/.changeset/popular-flies-swim.md new file mode 100644 index 0000000000..42a0944dce --- /dev/null +++ b/.changeset/popular-flies-swim.md @@ -0,0 +1,6 @@ +--- +'@backstage/dev-utils': patch +'@backstage/test-utils': patch +--- + +Fix @backstage/cli not being a devDependency diff --git a/packages/dev-utils/package.json b/packages/dev-utils/package.json index 4ceb0cec5e..21d6732ed5 100644 --- a/packages/dev-utils/package.json +++ b/packages/dev-utils/package.json @@ -29,7 +29,6 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/cli": "^0.4.0", "@backstage/core": "^0.3.1", "@backstage/test-utils": "^0.1.4", "@backstage/theme": "^0.2.1", @@ -46,6 +45,7 @@ "react-router-dom": "6.0.0-beta.0" }, "devDependencies": { + "@backstage/cli": "^0.4.0", "@types/jest": "^26.0.7", "@types/node": "^12.0.0" }, diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index 86997a6767..d35080f204 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -29,7 +29,6 @@ "clean": "backstage-cli clean" }, "dependencies": { - "@backstage/cli": "^0.4.0", "@backstage/core-api": "^0.2.4", "@backstage/test-utils-core": "^0.1.1", "@backstage/theme": "^0.2.0", @@ -46,6 +45,7 @@ "zen-observable": "^0.8.15" }, "devDependencies": { + "@backstage/cli": "^0.4.0", "@types/jest": "^26.0.7", "@types/node": "^12.0.0" }, From 1bb8afc61efdc788e3ea1f7f97c7c2977b5c29da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 10 Dec 2020 10:17:15 +0100 Subject: [PATCH 052/188] catalog-backend: move github util to own folder --- .../processors/GithubOrgReaderProcessor.ts | 2 +- .../processors/{util => github}/github.test.ts | 2 +- .../processors/{util => github}/github.ts | 0 .../src/ingestion/processors/github/index.ts | 17 +++++++++++++++++ .../src/ingestion/processors/index.ts | 8 +++----- 5 files changed, 22 insertions(+), 7 deletions(-) rename plugins/catalog-backend/src/ingestion/processors/{util => github}/github.test.ts (100%) rename plugins/catalog-backend/src/ingestion/processors/{util => github}/github.ts (100%) create mode 100644 plugins/catalog-backend/src/ingestion/processors/github/index.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts index 9aecb8b86e..27d5424e4e 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GithubOrgReaderProcessor.ts @@ -18,9 +18,9 @@ import { LocationSpec } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { graphql } from '@octokit/graphql'; import { Logger } from 'winston'; +import { getOrganizationTeams, getOrganizationUsers } from './github'; import * as results from './results'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; -import { getOrganizationTeams, getOrganizationUsers } from './util/github'; import { buildOrgHierarchy } from './util/org'; /** diff --git a/plugins/catalog-backend/src/ingestion/processors/util/github.test.ts b/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/util/github.test.ts rename to plugins/catalog-backend/src/ingestion/processors/github/github.test.ts index 1e35f44712..d10aba8acb 100644 --- a/plugins/catalog-backend/src/ingestion/processors/util/github.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/github/github.test.ts @@ -14,10 +14,10 @@ * limitations under the License. */ +import { msw } from '@backstage/test-utils'; import { graphql } from '@octokit/graphql'; import { graphql as graphqlMsw } from 'msw'; import { setupServer } from 'msw/node'; -import { msw } from '@backstage/test-utils'; import { getOrganizationTeams, getOrganizationUsers, diff --git a/plugins/catalog-backend/src/ingestion/processors/util/github.ts b/plugins/catalog-backend/src/ingestion/processors/github/github.ts similarity index 100% rename from plugins/catalog-backend/src/ingestion/processors/util/github.ts rename to plugins/catalog-backend/src/ingestion/processors/github/github.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/github/index.ts b/plugins/catalog-backend/src/ingestion/processors/github/index.ts new file mode 100644 index 0000000000..e903ab669a --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/github/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2020 Spotify AB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { getOrganizationTeams, getOrganizationUsers } from './github'; diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index 49d4e03b64..2b477f18f9 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -16,11 +16,6 @@ import * as results from './results'; -export { results }; -export * from './types'; - -export { parseEntityYaml } from './util/parse'; - export { AnnotateLocationEntityProcessor } from './AnnotateLocationEntityProcessor'; export { BuiltinKindsEntityProcessor } from './BuiltinKindsEntityProcessor'; export { CodeOwnersProcessor } from './CodeOwnersProcessor'; @@ -32,4 +27,7 @@ export { MicrosoftGraphOrgReaderProcessor } from './MicrosoftGraphOrgReaderProce export { PlaceholderProcessor } from './PlaceholderProcessor'; export type { PlaceholderResolver } from './PlaceholderProcessor'; export { StaticLocationProcessor } from './StaticLocationProcessor'; +export * from './types'; export { UrlReaderProcessor } from './UrlReaderProcessor'; +export { parseEntityYaml } from './util/parse'; +export { results }; From 2bf71d60fbc74714e4a9ea15ed1c898f948a2b22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Thu, 10 Dec 2020 11:23:02 +0100 Subject: [PATCH 053/188] catalog-backend: get rid of winston warnings --- .../src/ingestion/processors/CodeOwnersProcessor.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts index 068a1e6e9a..be4f15e501 100644 --- a/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/CodeOwnersProcessor.test.ts @@ -14,9 +14,9 @@ * limitations under the License. */ +import { getVoidLogger } from '@backstage/backend-common'; import { LocationSpec } from '@backstage/catalog-model'; import { CodeOwnersEntry } from 'codeowners-utils'; -import { createLogger } from 'winston'; import { buildCodeOwnerUrl, buildUrl, @@ -28,7 +28,7 @@ import { resolveCodeOwner, } from './CodeOwnersProcessor'; -const logger = createLogger(); +const logger = getVoidLogger(); describe('CodeOwnersProcessor', () => { const mockUrl = ({ basePath = '' } = {}): string => From a6a2ca62049b7699b951133a02a9edb44ab04550 Mon Sep 17 00:00:00 2001 From: Askar Date: Thu, 10 Dec 2020 11:23:29 +0100 Subject: [PATCH 054/188] remove React's FC type from codebase (#3527) * WIP-packages: remove React's FC type from codebase * remove FC from other directories * fix build failures * add types to required packages --- .../quickstart-app-plugin/ExampleComponent.md | 4 +- .../ExampleFetchComponent.md | 6 +-- docs/api/utility-apis.md | 4 +- docs/reference/createPlugin-feature-flags.md | 2 +- docs/tutorials/quickstart-app-plugin.md | 10 ++--- packages/app/src/App.tsx | 4 +- packages/app/src/components/Root/LogoFull.tsx | 4 +- packages/app/src/components/Root/LogoIcon.tsx | 4 +- packages/app/src/components/Root/Root.tsx | 11 ++--- .../ExampleComponent/ExampleComponent.tsx.hbs | 4 +- .../ExampleFetchComponent.tsx.hbs | 6 +-- packages/core-api/package.json | 1 + .../core-api/src/apis/system/ApiProvider.tsx | 14 ++++-- packages/core-api/src/app/App.tsx | 13 +++--- packages/core-api/src/app/AppContext.tsx | 7 ++- .../core-api/src/app/AppThemeProvider.tsx | 6 +-- packages/core-api/src/icons/icons.tsx | 4 +- packages/core/package.json | 1 + packages/core/src/api-wrappers/createApp.tsx | 4 +- .../components/AlertDisplay/AlertDisplay.tsx | 6 +-- .../CopyTextButton/CopyTextButton.tsx | 4 +- .../FeatureCalloutCircular.tsx | 6 +-- .../HorizontalScrollGrid.tsx | 4 +- .../src/components/Lifecycle/Lifecycle.tsx | 4 +- packages/core/src/components/Link/Link.tsx | 2 +- .../LoginRequestListItem.tsx | 4 +- .../OAuthRequestDialog/OAuthRequestDialog.tsx | 6 +-- .../core/src/components/Progress/Progress.tsx | 4 +- .../src/components/ProgressBars/Gauge.tsx | 4 +- .../src/components/ProgressBars/GaugeCard.tsx | 4 +- .../components/ProgressBars/LinearGauge.tsx | 4 +- .../SimpleStepper/SimpleStepper.tsx | 6 +-- .../SimpleStepper/SimpleStepperFooter.tsx | 43 +++++++++++-------- .../SimpleStepper/SimpleStepperStep.tsx | 6 +-- .../core/src/components/Status/Status.tsx | 14 +++--- .../StructuredMetadataTable.stories.tsx | 4 +- .../SupportButton/SupportButton.tsx | 13 ++++-- .../src/components/Table/SubvalueCell.tsx | 4 +- packages/core/src/components/Tabs/TabBar.tsx | 4 +- .../core/src/components/Tabs/TabPanel.tsx | 5 +-- packages/core/src/components/Tabs/Tabs.tsx | 10 +---- .../src/components/TrendLine/TrendLine.tsx | 4 +- .../core/src/layout/BottomLink/BottomLink.tsx | 4 +- packages/core/src/layout/Content/Content.tsx | 6 +-- .../layout/ContentHeader/ContentHeader.tsx | 10 ++--- packages/core/src/layout/Header/Header.tsx | 18 +++----- .../HeaderActionMenu/HeaderActionMenu.tsx | 10 ++--- .../src/layout/HeaderLabel/HeaderLabel.tsx | 11 +++-- .../core/src/layout/HeaderTabs/HeaderTabs.tsx | 9 +++- .../core/src/layout/ItemCard/ItemCard.tsx | 6 +-- packages/core/src/layout/Page/Page.tsx | 4 +- packages/core/src/layout/Sidebar/Bar.tsx | 6 +-- packages/core/src/layout/Sidebar/Intro.tsx | 8 ++-- packages/core/src/layout/Sidebar/Page.tsx | 9 +++- .../core/src/layout/SignInPage/SignInPage.tsx | 6 +-- .../core/src/layout/TabbedCard/TabbedCard.tsx | 13 ++++-- .../default-app/packages/app/src/App.tsx | 4 +- .../default-app/packages/app/src/LogoFull.tsx | 4 +- .../default-app/packages/app/src/LogoIcon.tsx | 4 +- .../default-app/packages/app/src/sidebar.tsx | 4 +- packages/dev-utils/src/devApp/render.tsx | 4 +- .../src/testUtils/appWrappers.test.tsx | 10 ++--- .../test-utils/src/testUtils/appWrappers.tsx | 6 +-- .../EntityContextMenu/EntityContextMenu.tsx | 4 +- .../FavouriteEntity/FavouriteEntity.tsx | 2 +- .../UnregisterEntityDialog.tsx | 6 +-- .../src/hooks/useStarredEntities.test.tsx | 4 +- .../lib/ActionOutput/ActionOutput.tsx | 11 +++-- plugins/circleci/src/route-refs.tsx | 4 +- .../WorkflowRunsTable/WorkflowRunsTable.tsx | 6 +-- .../AlertActionCardList.tsx | 4 +- .../explore/src/components/ExploreCard.tsx | 4 +- .../WorkflowRunsTable/WorkflowRunsTable.tsx | 6 +-- .../components/ClusterList/ClusterList.tsx | 4 +- .../components/ClusterPage/ClusterPage.tsx | 4 +- .../components/ClusterTable/ClusterTable.tsx | 4 +- .../ClusterTemplateCard.tsx | 4 +- .../ClusterTemplateCardList.tsx | 4 +- .../components/ProfileCard/ProfileCard.tsx | 4 +- .../ProfileCardList/ProfileCardList.tsx | 4 +- .../ProfileCatalog/ProfileCatalog.tsx | 4 +- .../GraphiQLBrowser/GraphiQLBrowser.tsx | 4 +- .../lib/ActionOutput/ActionOutput.tsx | 8 ++-- .../BuildsPage/lib/CITable/CITable.tsx | 6 +-- plugins/lighthouse/package.json | 2 +- .../components/AuditList/AuditListTable.tsx | 4 +- .../src/components/AuditList/index.tsx | 4 +- .../src/components/AuditStatusIcon/index.tsx | 4 +- .../src/components/AuditView/index.tsx | 11 ++--- .../Cards/LastLighthouseAuditCard.tsx | 18 +++++--- .../src/components/CreateAudit/index.tsx | 4 +- .../src/hooks/useWebsiteForEntity.test.tsx | 4 +- .../NewRelicComponent/NewRelicComponent.tsx | 4 +- .../NewRelicFetchComponent.tsx | 8 ++-- .../src/components/ErrorCell/ErrorCell.tsx | 6 +-- .../src/components/ErrorGraph/ErrorGraph.tsx | 6 +-- 96 files changed, 316 insertions(+), 290 deletions(-) diff --git a/contrib/docs/tutorials/quickstart-app-plugin/ExampleComponent.md b/contrib/docs/tutorials/quickstart-app-plugin/ExampleComponent.md index 5d633544e3..9b5d77bc7c 100644 --- a/contrib/docs/tutorials/quickstart-app-plugin/ExampleComponent.md +++ b/contrib/docs/tutorials/quickstart-app-plugin/ExampleComponent.md @@ -3,7 +3,7 @@ ExampleComponent.tsx reference ```tsx -import React, { FC } from 'react'; +import React from 'react'; import { Typography, Grid } from '@material-ui/core'; import { InfoCard, @@ -18,7 +18,7 @@ import { import { useApi } from '@backstage/core-api'; import ExampleFetchComponent from '../ExampleFetchComponent'; -const ExampleComponent: FC<{}> = () => { +const ExampleComponent = () => { const identityApi = useApi(identityApiRef); const userId = identityApi.getUserId(); const profile = identityApi.getProfile(); diff --git a/contrib/docs/tutorials/quickstart-app-plugin/ExampleFetchComponent.md b/contrib/docs/tutorials/quickstart-app-plugin/ExampleFetchComponent.md index 08d14e8c4c..6061b69e93 100644 --- a/contrib/docs/tutorials/quickstart-app-plugin/ExampleFetchComponent.md +++ b/contrib/docs/tutorials/quickstart-app-plugin/ExampleFetchComponent.md @@ -3,7 +3,7 @@ ExampleFetchComponent.tsx reference ```tsx -import React, { FC } from 'react'; +import React from 'react'; import { useAsync } from 'react-use'; import Alert from '@material-ui/lab/Alert'; import { @@ -57,7 +57,7 @@ type DenseTableProps = { viewer: Viewer; }; -export const DenseTable: FC = ({ viewer }) => { +export const DenseTable = ({ viewer }: DenseTableProps) => { const columns: TableColumn[] = [ { title: 'Name', field: 'name' }, { title: 'Created', field: 'createdAt' }, @@ -76,7 +76,7 @@ export const DenseTable: FC = ({ viewer }) => { ); }; -const ExampleFetchComponent: FC<{}> = () => { +const ExampleFetchComponent = () => { const auth = useApi(githubAuthApiRef); const { value, loading, error } = useAsync(async (): Promise => { diff --git a/docs/api/utility-apis.md b/docs/api/utility-apis.md index 2547db5c5c..ff322c136f 100644 --- a/docs/api/utility-apis.md +++ b/docs/api/utility-apis.md @@ -33,10 +33,10 @@ hook exported by `@backstage/core`, or the `withApis` HOC if you prefer class components. For example, the `ErrorApi` can be accessed like this: ```tsx -import React, { FC } from 'react'; +import React from 'react'; import { useApi, errorApiRef } from '@backstage/core'; -export const MyComponent: FC<{}> = () => { +export const MyComponent = () => { const errorApi = useApi(errorApiRef); // Signal to the app that something went wrong, and display the error to the user. diff --git a/docs/reference/createPlugin-feature-flags.md b/docs/reference/createPlugin-feature-flags.md index bcea80e26b..622c085291 100644 --- a/docs/reference/createPlugin-feature-flags.md +++ b/docs/reference/createPlugin-feature-flags.md @@ -27,7 +27,7 @@ To inspect the state of a feature flag inside your plugin, you can use the `FeatureFlagsApi`, accessed via the `featureFlagsApiRef`. For example: ```tsx -import React, { FC } from 'react'; +import React from 'react'; import { Button } from '@material-ui/core'; import { featureFlagsApiRef, useApi } from '@backstage/core'; diff --git a/docs/tutorials/quickstart-app-plugin.md b/docs/tutorials/quickstart-app-plugin.md index 216ff9659f..0f8d9bb84b 100644 --- a/docs/tutorials/quickstart-app-plugin.md +++ b/docs/tutorials/quickstart-app-plugin.md @@ -81,13 +81,13 @@ import { useApi } from '@backstage/core-api'; _from inline:_ ```tsx -const ExampleComponent: FC<{}> = () => ( ... ) +const ExampleComponent = () => ( ... ) ``` _to block:_ ```tsx -const ExampleComponent: FC<{}> = () => { +const ExampleComponent = () => { return ( ... @@ -135,7 +135,7 @@ changes, let's start by wiping this component clean. 1. Replace everything in the file with the following: ```tsx -import React, { FC } from 'react'; +import React from 'react'; import { useAsync } from 'react-use'; import Alert from '@material-ui/lab/Alert'; import { @@ -147,7 +147,7 @@ import { import { useApi } from '@backstage/core-api'; import { graphql } from '@octokit/graphql'; -const ExampleFetchComponent: FC<{}> = () => { +const ExampleFetchComponent = () => { return
Nothing to see yet
; }; @@ -223,7 +223,7 @@ type DenseTableProps = { viewer: Viewer; }; -export const DenseTable: FC = ({ viewer }) => { +export const DenseTable = ({ viewer }: DenseTableProps) => { const columns: TableColumn[] = [ { title: 'Name', field: 'name' }, { title: 'Created', field: 'createdAt' }, diff --git a/packages/app/src/App.tsx b/packages/app/src/App.tsx index 28aebeeb79..627b5d19aa 100644 --- a/packages/app/src/App.tsx +++ b/packages/app/src/App.tsx @@ -21,7 +21,7 @@ import { SignInPage, createRouteRef, } from '@backstage/core'; -import React, { FC } from 'react'; +import React from 'react'; import Root from './components/Root'; import * as plugins from './plugins'; import { apis } from './apis'; @@ -92,7 +92,7 @@ const AppRoutes = () => ( ); -const App: FC<{}> = () => ( +const App = () => ( diff --git a/packages/app/src/components/Root/LogoFull.tsx b/packages/app/src/components/Root/LogoFull.tsx index d2b1bf1080..2fb767465b 100644 --- a/packages/app/src/components/Root/LogoFull.tsx +++ b/packages/app/src/components/Root/LogoFull.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import React, { FC } from 'react'; +import React from 'react'; import { makeStyles } from '@material-ui/core'; const useStyles = makeStyles({ @@ -26,7 +26,7 @@ const useStyles = makeStyles({ fill: '#7df3e1', }, }); -const LogoFull: FC<{}> = () => { +const LogoFull = () => { const classes = useStyles(); return ( diff --git a/packages/app/src/components/Root/LogoIcon.tsx b/packages/app/src/components/Root/LogoIcon.tsx index d70be3dd32..507e47ddb9 100644 --- a/packages/app/src/components/Root/LogoIcon.tsx +++ b/packages/app/src/components/Root/LogoIcon.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import React, { FC } from 'react'; +import React from 'react'; import { makeStyles } from '@material-ui/core'; const useStyles = makeStyles({ @@ -27,7 +27,7 @@ const useStyles = makeStyles({ }, }); -const LogoIcon: FC<{}> = () => { +const LogoIcon = () => { const classes = useStyles(); return ( diff --git a/packages/app/src/components/Root/Root.tsx b/packages/app/src/components/Root/Root.tsx index 47e7d32d1a..52dd397418 100644 --- a/packages/app/src/components/Root/Root.tsx +++ b/packages/app/src/components/Root/Root.tsx @@ -14,8 +14,7 @@ * limitations under the License. */ -import React, { FC, useContext } from 'react'; -import PropTypes from 'prop-types'; +import React, { useContext, PropsWithChildren } from 'react'; import { Link, makeStyles } from '@material-ui/core'; import HomeIcon from '@material-ui/icons/Home'; import ExtensionIcon from '@material-ui/icons/Extension'; @@ -55,7 +54,7 @@ const useSidebarLogoStyles = makeStyles({ }, }); -const SidebarLogo: FC<{}> = () => { +const SidebarLogo = () => { const classes = useSidebarLogoStyles(); const { isOpen } = useContext(SidebarContext); @@ -73,7 +72,7 @@ const SidebarLogo: FC<{}> = () => { ); }; -const Root: FC<{}> = ({ children }) => ( +const Root = ({ children }: PropsWithChildren<{}>) => ( @@ -102,8 +101,4 @@ const Root: FC<{}> = ({ children }) => ( ); -Root.propTypes = { - children: PropTypes.node, -}; - export default Root; diff --git a/packages/cli/templates/default-plugin/src/components/ExampleComponent/ExampleComponent.tsx.hbs b/packages/cli/templates/default-plugin/src/components/ExampleComponent/ExampleComponent.tsx.hbs index e08f1650d5..5f90f2de1e 100644 --- a/packages/cli/templates/default-plugin/src/components/ExampleComponent/ExampleComponent.tsx.hbs +++ b/packages/cli/templates/default-plugin/src/components/ExampleComponent/ExampleComponent.tsx.hbs @@ -1,4 +1,4 @@ -import React, { FC } from 'react'; +import React from 'react'; import { Typography, Grid } from '@material-ui/core'; import { InfoCard, @@ -11,7 +11,7 @@ import { } from '@backstage/core'; import ExampleFetchComponent from '../ExampleFetchComponent'; -const ExampleComponent: FC<{}> = () => ( +const ExampleComponent = () => (
diff --git a/packages/cli/templates/default-plugin/src/components/ExampleFetchComponent/ExampleFetchComponent.tsx.hbs b/packages/cli/templates/default-plugin/src/components/ExampleFetchComponent/ExampleFetchComponent.tsx.hbs index 0af27a5935..8cc5ed2ab7 100644 --- a/packages/cli/templates/default-plugin/src/components/ExampleFetchComponent/ExampleFetchComponent.tsx.hbs +++ b/packages/cli/templates/default-plugin/src/components/ExampleFetchComponent/ExampleFetchComponent.tsx.hbs @@ -1,4 +1,4 @@ -import React, { FC } from 'react'; +import React from 'react'; import { makeStyles } from '@material-ui/core/styles'; import { Table, TableColumn, Progress } from '@backstage/core'; import Alert from '@material-ui/lab/Alert'; @@ -38,7 +38,7 @@ type DenseTableProps = { users: User[]; }; -export const DenseTable: FC = ({ users }) => { +export const DenseTable = ({ users }: DenseTableProps) => { const classes = useStyles(); const columns: TableColumn[] = [ @@ -73,7 +73,7 @@ export const DenseTable: FC = ({ users }) => { ); }; -const ExampleFetchComponent: FC<{}> = () => { +const ExampleFetchComponent = () => { const { value, loading, error } = useAsync(async (): Promise => { const response = await fetch('https://randomuser.me/api/?results=20'); const data = await response.json(); diff --git a/packages/core-api/package.json b/packages/core-api/package.json index bfff89f27a..b1397f2e38 100644 --- a/packages/core-api/package.json +++ b/packages/core-api/package.json @@ -35,6 +35,7 @@ "@material-ui/core": "^4.11.0", "@material-ui/icons": "^4.9.1", "@types/react": "^16.9", + "@types/prop-types": "^15.7.3", "prop-types": "^15.7.2", "react": "^16.12.0", "react-router-dom": "6.0.0-beta.0", diff --git a/packages/core-api/src/apis/system/ApiProvider.tsx b/packages/core-api/src/apis/system/ApiProvider.tsx index f2aa70244e..91d35e5ee7 100644 --- a/packages/core-api/src/apis/system/ApiProvider.tsx +++ b/packages/core-api/src/apis/system/ApiProvider.tsx @@ -14,7 +14,12 @@ * limitations under the License. */ -import React, { FC, createContext, useContext, ReactNode } from 'react'; +import React, { + createContext, + useContext, + ReactNode, + PropsWithChildren, +} from 'react'; import PropTypes from 'prop-types'; import { ApiRef, ApiHolder, TypesToApiRefs } from './types'; import { ApiAggregator } from './ApiAggregator'; @@ -26,7 +31,10 @@ type ApiProviderProps = { const Context = createContext(undefined); -export const ApiProvider: FC = ({ apis, children }) => { +export const ApiProvider = ({ + apis, + children, +}: PropsWithChildren) => { const parentHolder = useContext(Context); const holder = parentHolder ? new ApiAggregator(apis, parentHolder) : apis; @@ -62,7 +70,7 @@ export function withApis(apis: TypesToApiRefs) { return function withApisWrapper

( WrappedComponent: React.ComponentType

, ) { - const Hoc: FC> = props => { + const Hoc = (props: PropsWithChildren>) => { const apiHolder = useContext(Context); if (!apiHolder) { diff --git a/packages/core-api/src/app/App.tsx b/packages/core-api/src/app/App.tsx index 3d32259337..3ca91966bf 100644 --- a/packages/core-api/src/app/App.tsx +++ b/packages/core-api/src/app/App.tsx @@ -15,10 +15,10 @@ */ import React, { ComponentType, - FC, useMemo, useState, ReactElement, + PropsWithChildren, } from 'react'; import { Route, Routes, Navigate } from 'react-router-dom'; import { AppContextProvider } from './AppContext'; @@ -196,7 +196,7 @@ export class PrivateAppImpl implements BackstageApp { } getProvider(): ComponentType<{}> { - const Provider: FC<{}> = ({ children }) => { + const Provider = ({ children }: PropsWithChildren<{}>) => { const appThemeApi = useMemo( () => AppThemeSelector.createWithStorage(this.themes), [], @@ -233,10 +233,13 @@ export class PrivateAppImpl implements BackstageApp { } = this.components; // This wraps the sign-in page and waits for sign-in to be completed before rendering the app - const SignInPageWrapper: FC<{ + const SignInPageWrapper = ({ + component: Component, + children, + }: { component: ComponentType; children: ReactElement; - }> = ({ component: Component, children }) => { + }) => { const [result, setResult] = useState(); if (result) { @@ -247,7 +250,7 @@ export class PrivateAppImpl implements BackstageApp { return ; }; - const AppRouter: FC<{}> = ({ children }) => { + const AppRouter = ({ children }: PropsWithChildren<{}>) => { const configApi = useApi(configApiRef); let { pathname } = new URL( diff --git a/packages/core-api/src/app/AppContext.tsx b/packages/core-api/src/app/AppContext.tsx index ec3831992f..e0659d9e29 100644 --- a/packages/core-api/src/app/AppContext.tsx +++ b/packages/core-api/src/app/AppContext.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import React, { createContext, useContext, FC } from 'react'; +import React, { createContext, PropsWithChildren, useContext } from 'react'; import { BackstageApp } from './types'; const Context = createContext(undefined); @@ -23,7 +23,10 @@ type Props = { app: BackstageApp; }; -export const AppContextProvider: FC = ({ app, children }) => ( +export const AppContextProvider = ({ + app, + children, +}: PropsWithChildren) => ( ); diff --git a/packages/core-api/src/app/AppThemeProvider.tsx b/packages/core-api/src/app/AppThemeProvider.tsx index 6bbcaea93a..993de23a7a 100644 --- a/packages/core-api/src/app/AppThemeProvider.tsx +++ b/packages/core-api/src/app/AppThemeProvider.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import React, { FC, useMemo, useEffect, useState } from 'react'; +import React, { useMemo, useEffect, useState, PropsWithChildren } from 'react'; import { ThemeProvider, CssBaseline } from '@material-ui/core'; import { useApi, appThemeApiRef, AppTheme } from '../apis'; import { useObservable } from 'react-use'; @@ -68,7 +68,7 @@ const useShouldPreferDarkTheme = () => { return shouldPreferDark; }; -export const AppThemeProvider: FC<{}> = ({ children }) => { +export function AppThemeProvider({ children }: PropsWithChildren<{}>) { const appThemeApi = useApi(appThemeApiRef); const themeId = useObservable( appThemeApi.activeThemeId$(), @@ -94,4 +94,4 @@ export const AppThemeProvider: FC<{}> = ({ children }) => { {children} ); -}; +} diff --git a/packages/core-api/src/icons/icons.tsx b/packages/core-api/src/icons/icons.tsx index 488973b664..50c4b68e43 100644 --- a/packages/core-api/src/icons/icons.tsx +++ b/packages/core-api/src/icons/icons.tsx @@ -17,7 +17,7 @@ import { SvgIconProps } from '@material-ui/core'; import PeopleIcon from '@material-ui/icons/People'; import PersonIcon from '@material-ui/icons/Person'; -import React, { FC } from 'react'; +import React from 'react'; import { useApp } from '../app/AppContext'; import { IconComponent, SystemIconKey, SystemIcons } from './types'; @@ -27,7 +27,7 @@ export const defaultSystemIcons: SystemIcons = { }; const overridableSystemIcon = (key: SystemIconKey): IconComponent => { - const Component: FC = props => { + const Component = (props: SvgIconProps) => { const app = useApp(); const Icon = app.getSystemIcon(key); return ; diff --git a/packages/core/package.json b/packages/core/package.json index dd48a5d4ce..dc9ed4c540 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -38,6 +38,7 @@ "@types/dagre": "^0.7.44", "@types/react": "^16.9", "@types/react-sparklines": "^1.7.0", + "@types/prop-types": "^15.7.3", "classnames": "^2.2.6", "clsx": "^1.1.0", "d3-selection": "^2.0.0", diff --git a/packages/core/src/api-wrappers/createApp.tsx b/packages/core/src/api-wrappers/createApp.tsx index 4a1e58db42..c99ba0ed03 100644 --- a/packages/core/src/api-wrappers/createApp.tsx +++ b/packages/core/src/api-wrappers/createApp.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import React, { FC } from 'react'; +import React from 'react'; import privateExports, { AppOptions, defaultSystemIcons, @@ -93,7 +93,7 @@ export function createApp(options?: AppOptions) { const DefaultNotFoundPage = () => ( ); - const DefaultBootErrorPage: FC = ({ step, error }) => { + const DefaultBootErrorPage = ({ step, error }: BootErrorPageProps) => { let message = ''; if (step === 'load-config') { message = `The configuration failed to load, someone should have a look at this error: ${error.message}`; diff --git a/packages/core/src/components/AlertDisplay/AlertDisplay.tsx b/packages/core/src/components/AlertDisplay/AlertDisplay.tsx index 30940f68ac..6d6646fa18 100644 --- a/packages/core/src/components/AlertDisplay/AlertDisplay.tsx +++ b/packages/core/src/components/AlertDisplay/AlertDisplay.tsx @@ -14,16 +14,14 @@ * limitations under the License. */ -import React, { FC, useEffect, useState } from 'react'; +import React, { useEffect, useState } from 'react'; import { Snackbar, IconButton } from '@material-ui/core'; import CloseIcon from '@material-ui/icons/Close'; import { Alert } from '@material-ui/lab'; import { AlertMessage, useApi, alertApiRef } from '@backstage/core-api'; -type Props = {}; - // TODO: improve on this and promote to a shared component for use by all apps. -export const AlertDisplay: FC = () => { +export const AlertDisplay = () => { const [messages, setMessages] = useState>([]); const alertApi = useApi(alertApiRef); diff --git a/packages/core/src/components/CopyTextButton/CopyTextButton.tsx b/packages/core/src/components/CopyTextButton/CopyTextButton.tsx index cb322e5b56..9f6cdd7e7c 100644 --- a/packages/core/src/components/CopyTextButton/CopyTextButton.tsx +++ b/packages/core/src/components/CopyTextButton/CopyTextButton.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import React, { FC, useRef, useState, MouseEventHandler } from 'react'; +import React, { useRef, useState, MouseEventHandler } from 'react'; import { IconButton, makeStyles, Tooltip } from '@material-ui/core'; import PropTypes from 'prop-types'; import CopyIcon from '@material-ui/icons/FileCopy'; @@ -56,7 +56,7 @@ const defaultProps = { tooltipText: 'Text copied to clipboard', }; -export const CopyTextButton: FC = props => { +export const CopyTextButton = (props: Props) => { const { text, tooltipDelay, tooltipText } = { ...defaultProps, ...props, diff --git a/packages/core/src/components/FeatureDiscovery/FeatureCalloutCircular.tsx b/packages/core/src/components/FeatureDiscovery/FeatureCalloutCircular.tsx index 9dc0d25681..64722c24b0 100644 --- a/packages/core/src/components/FeatureDiscovery/FeatureCalloutCircular.tsx +++ b/packages/core/src/components/FeatureDiscovery/FeatureCalloutCircular.tsx @@ -16,7 +16,7 @@ import { ClickAwayListener, makeStyles, Typography } from '@material-ui/core'; import React, { - FC, + PropsWithChildren, useCallback, useEffect, useLayoutEffect, @@ -93,12 +93,12 @@ type Placement = { textWidth: number; }; -export const FeatureCalloutCircular: FC = ({ +export const FeatureCalloutCircular = ({ featureId, title, description, children, -}) => { +}: PropsWithChildren) => { const { show, hide } = useShowCallout(featureId); const portalElement = usePortal('core.callout'); const wrapperRef = useRef(null); diff --git a/packages/core/src/components/HorizontalScrollGrid/HorizontalScrollGrid.tsx b/packages/core/src/components/HorizontalScrollGrid/HorizontalScrollGrid.tsx index 4e321dfa95..073fb960e1 100644 --- a/packages/core/src/components/HorizontalScrollGrid/HorizontalScrollGrid.tsx +++ b/packages/core/src/components/HorizontalScrollGrid/HorizontalScrollGrid.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import React, { FC } from 'react'; +import React, { PropsWithChildren } from 'react'; import classNames from 'classnames'; import ChevronLeftIcon from '@material-ui/icons/ChevronLeft'; import ChevronRightIcon from '@material-ui/icons/ChevronRight'; @@ -181,7 +181,7 @@ function useSmoothScroll( return setScrollTarget; } -export const HorizontalScrollGrid: FC = props => { +export const HorizontalScrollGrid = (props: PropsWithChildren) => { const { scrollStep = 100, scrollSpeed = 50, diff --git a/packages/core/src/components/Lifecycle/Lifecycle.tsx b/packages/core/src/components/Lifecycle/Lifecycle.tsx index 416df27a30..d388452cdc 100644 --- a/packages/core/src/components/Lifecycle/Lifecycle.tsx +++ b/packages/core/src/components/Lifecycle/Lifecycle.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import React, { FC } from 'react'; +import React from 'react'; import CSS from 'csstype'; import { makeStyles } from '@material-ui/core'; @@ -38,7 +38,7 @@ const useStyles = makeStyles({ }, }); -export const Lifecycle: FC = props => { +export const Lifecycle = (props: Props) => { const classes = useStyles(props); const { shorthand, alpha } = props; return shorthand ? ( diff --git a/packages/core/src/components/Link/Link.tsx b/packages/core/src/components/Link/Link.tsx index 59a9604fa7..c5426ac434 100644 --- a/packages/core/src/components/Link/Link.tsx +++ b/packages/core/src/components/Link/Link.tsx @@ -19,7 +19,7 @@ import { Link as MaterialLink } from '@material-ui/core'; import { Link as RouterLink } from 'react-router-dom'; type Props = ComponentProps & - ComponentProps & { component?: React.FC }; + ComponentProps & { component?: React.ReactNode }; /** * Thin wrapper on top of material-ui's Link component diff --git a/packages/core/src/components/OAuthRequestDialog/LoginRequestListItem.tsx b/packages/core/src/components/OAuthRequestDialog/LoginRequestListItem.tsx index cc7d99660b..cc1df3b529 100644 --- a/packages/core/src/components/OAuthRequestDialog/LoginRequestListItem.tsx +++ b/packages/core/src/components/OAuthRequestDialog/LoginRequestListItem.tsx @@ -22,7 +22,7 @@ import { Typography, Theme, } from '@material-ui/core'; -import React, { FC, useState } from 'react'; +import React, { useState } from 'react'; import { PendingAuthRequest } from '@backstage/core-api'; const useItemStyles = makeStyles(theme => ({ @@ -37,7 +37,7 @@ type RowProps = { setBusy: (busy: boolean) => void; }; -const LoginRequestListItem: FC = ({ request, busy, setBusy }) => { +const LoginRequestListItem = ({ request, busy, setBusy }: RowProps) => { const classes = useItemStyles(); const [error, setError] = useState(); diff --git a/packages/core/src/components/OAuthRequestDialog/OAuthRequestDialog.tsx b/packages/core/src/components/OAuthRequestDialog/OAuthRequestDialog.tsx index 07078c3fa2..06b53332dc 100644 --- a/packages/core/src/components/OAuthRequestDialog/OAuthRequestDialog.tsx +++ b/packages/core/src/components/OAuthRequestDialog/OAuthRequestDialog.tsx @@ -24,7 +24,7 @@ import { Theme, Button, } from '@material-ui/core'; -import React, { FC, useMemo, useState } from 'react'; +import React, { useMemo, useState } from 'react'; import { useObservable } from 'react-use'; import LoginRequestListItem from './LoginRequestListItem'; import { useApi, oauthRequestApiRef } from '@backstage/core-api'; @@ -41,9 +41,7 @@ const useStyles = makeStyles(theme => ({ }, })); -type OAuthRequestDialogProps = {}; - -export const OAuthRequestDialog: FC = () => { +export const OAuthRequestDialog = () => { const classes = useStyles(); const [busy, setBusy] = useState(false); const oauthRequestApi = useApi(oauthRequestApiRef); diff --git a/packages/core/src/components/Progress/Progress.tsx b/packages/core/src/components/Progress/Progress.tsx index 80f4f38cc9..aacdf8821b 100644 --- a/packages/core/src/components/Progress/Progress.tsx +++ b/packages/core/src/components/Progress/Progress.tsx @@ -14,10 +14,10 @@ * limitations under the License. */ -import React, { FC, useState, useEffect } from 'react'; +import React, { useState, useEffect, PropsWithChildren } from 'react'; import { LinearProgress, LinearProgressProps } from '@material-ui/core'; -export const Progress: FC = props => { +export const Progress = (props: PropsWithChildren) => { const [isVisible, setIsVisible] = useState(false); useEffect(() => { diff --git a/packages/core/src/components/ProgressBars/Gauge.tsx b/packages/core/src/components/ProgressBars/Gauge.tsx index 4c339bf8e1..ca6a3a66ab 100644 --- a/packages/core/src/components/ProgressBars/Gauge.tsx +++ b/packages/core/src/components/ProgressBars/Gauge.tsx @@ -17,7 +17,7 @@ import { makeStyles, useTheme } from '@material-ui/core'; import { BackstageTheme } from '@backstage/theme'; import { Circle } from 'rc-progress'; -import React, { FC } from 'react'; +import React from 'react'; const useStyles = makeStyles(theme => ({ root: { @@ -77,7 +77,7 @@ export function getProgressColor( return palette.status.ok; } -export const Gauge: FC = props => { +export const Gauge = (props: Props) => { const classes = useStyles(props); const theme = useTheme(); const { value, fractional, inverse, unit, max } = { diff --git a/packages/core/src/components/ProgressBars/GaugeCard.tsx b/packages/core/src/components/ProgressBars/GaugeCard.tsx index 4eb4b2e075..7f281c67a5 100644 --- a/packages/core/src/components/ProgressBars/GaugeCard.tsx +++ b/packages/core/src/components/ProgressBars/GaugeCard.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import React, { FC } from 'react'; +import React from 'react'; import { makeStyles } from '@material-ui/core'; import { InfoCard } from '../../layout/InfoCard'; import { BottomLinkProps } from '../../layout/BottomLink'; @@ -36,7 +36,7 @@ const useStyles = makeStyles({ }, }); -export const GaugeCard: FC = props => { +export const GaugeCard = (props: Props) => { const classes = useStyles(props); const { title, subheader, progress, deepLink, variant } = props; diff --git a/packages/core/src/components/ProgressBars/LinearGauge.tsx b/packages/core/src/components/ProgressBars/LinearGauge.tsx index a6aea59f19..9bb7b34c09 100644 --- a/packages/core/src/components/ProgressBars/LinearGauge.tsx +++ b/packages/core/src/components/ProgressBars/LinearGauge.tsx @@ -14,7 +14,7 @@ * limitations under the License. */ -import React, { FC } from 'react'; +import React from 'react'; import { Tooltip, useTheme } from '@material-ui/core'; // @ts-ignore import { Line } from 'rc-progress'; @@ -28,7 +28,7 @@ type Props = { value: number; }; -export const LinearGauge: FC = ({ value }) => { +export const LinearGauge = ({ value }: Props) => { const theme = useTheme(); if (isNaN(value)) { return null; diff --git a/packages/core/src/components/SimpleStepper/SimpleStepper.tsx b/packages/core/src/components/SimpleStepper/SimpleStepper.tsx index 75fc1e5e62..ff65534843 100644 --- a/packages/core/src/components/SimpleStepper/SimpleStepper.tsx +++ b/packages/core/src/components/SimpleStepper/SimpleStepper.tsx @@ -16,9 +16,9 @@ import React, { Children, isValidElement, - FC, useState, useEffect, + PropsWithChildren, } from 'react'; import { Stepper as MuiStepper } from '@material-ui/core'; @@ -47,12 +47,12 @@ export interface StepperProps { activeStep?: number; } -export const SimpleStepper: FC = ({ +export const SimpleStepper = ({ children, elevated, onStepChange, activeStep = 0, -}) => { +}: PropsWithChildren) => { const [stepIndex, setStepIndex] = useState(activeStep); const [stepHistory, setStepHistory] = useState([0]); diff --git a/packages/core/src/components/SimpleStepper/SimpleStepperFooter.tsx b/packages/core/src/components/SimpleStepper/SimpleStepperFooter.tsx index c8cddb375d..a49e20913c 100644 --- a/packages/core/src/components/SimpleStepper/SimpleStepperFooter.tsx +++ b/packages/core/src/components/SimpleStepper/SimpleStepperFooter.tsx @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import React, { useContext, FC, ReactNode } from 'react'; +import React, { useContext, ReactNode, PropsWithChildren } from 'react'; import { Button, makeStyles } from '@material-ui/core'; import { StepActions } from './SimpleStepperStep'; import { VerticalStepperContext } from './SimpleStepper'; @@ -27,20 +27,33 @@ const useStyles = makeStyles(theme => ({ }, })); -export const RestartBtn: FC<{ +interface CommonBtnProps { text?: string; handleClick?: () => void; stepIndex: number; -}> = ({ text, handleClick }) => ( - -); -const NextBtn: FC<{ - text?: string; - handleClick?: () => void; +} +interface RestartBtnProps extends CommonBtnProps {} + +interface NextBtnProps extends CommonBtnProps { disabled?: boolean; last?: boolean; stepIndex: number; -}> = ({ text, handleClick, disabled, last, stepIndex }) => ( +} +interface BackBtnProps extends CommonBtnProps { + disabled?: boolean; + stepIndex: number; +} +export const RestartBtn = ({ text, handleClick }: RestartBtnProps) => ( + +); + +const NextBtn = ({ + text, + handleClick, + disabled, + last, + stepIndex, +}: NextBtnProps) => (