From cd5932bc667d7b770093999d118a72b0ca3e5528 Mon Sep 17 00:00:00 2001 From: Tom Opdebeeck Date: Mon, 19 Jul 2021 16:35:00 +0200 Subject: [PATCH 001/343] feat: setup gitlab auth provider factory * add username to ProfileInfo * expose gitlab provider * add signIn handler * refactor resultHandler usage Signed-off-by: Tom Opdebeeck --- .../lib/passport/PassportStrategyHelper.ts | 7 +- .../src/providers/gitlab/provider.ts | 178 ++++++++++++++---- plugins/auth-backend/src/providers/index.ts | 1 + plugins/auth-backend/src/providers/types.ts | 4 + 4 files changed, 156 insertions(+), 34 deletions(-) diff --git a/plugins/auth-backend/src/lib/passport/PassportStrategyHelper.ts b/plugins/auth-backend/src/lib/passport/PassportStrategyHelper.ts index 313b4a79bd..e6ef47c2c0 100644 --- a/plugins/auth-backend/src/lib/passport/PassportStrategyHelper.ts +++ b/plugins/auth-backend/src/lib/passport/PassportStrategyHelper.ts @@ -30,7 +30,7 @@ export const makeProfileInfo = ( profile: passport.Profile, idToken?: string, ): ProfileInfo => { - let { displayName } = profile; + let { displayName, username } = profile; let email: string | undefined = undefined; if (profile.emails && profile.emails.length > 0) { @@ -56,6 +56,10 @@ export const makeProfileInfo = ( if (!displayName && decoded.name) { displayName = decoded.name; } + + if (!username && decoded.username) { + username = decoded.username; + } } catch (e) { throw new Error(`Failed to parse id token and get profile info, ${e}`); } @@ -65,6 +69,7 @@ export const makeProfileInfo = ( email, picture, displayName, + username, }; }; diff --git a/plugins/auth-backend/src/providers/gitlab/provider.ts b/plugins/auth-backend/src/providers/gitlab/provider.ts index 606d2c76a6..0fa99252b7 100644 --- a/plugins/auth-backend/src/providers/gitlab/provider.ts +++ b/plugins/auth-backend/src/providers/gitlab/provider.ts @@ -16,6 +16,8 @@ import express from 'express'; import { Strategy as GitlabStrategy } from 'passport-gitlab2'; +import { Logger } from 'winston'; + import { executeRedirectStrategy, executeFrameHandlerStrategy, @@ -24,7 +26,12 @@ import { makeProfileInfo, PassportDoneCallback, } from '../../lib/passport'; -import { RedirectInfo, AuthProviderFactory } from '../types'; +import { + RedirectInfo, + AuthProviderFactory, + SignInResolver, + AuthHandler, +} from '../types'; import { OAuthAdapter, OAuthProviderOptions, @@ -36,6 +43,8 @@ import { encodeState, OAuthResult, } from '../../lib/oauth'; +import { TokenIssuer } from '../../identity'; +import { CatalogIdentityClient } from '../../lib/catalog'; type FullProfile = OAuthResult['fullProfile'] & { avatarUrl?: string; @@ -47,29 +56,72 @@ type PrivateInfo = { export type GitlabAuthProviderOptions = OAuthProviderOptions & { baseUrl: string; + signInResolver?: SignInResolver; + authHandler: AuthHandler; + tokenIssuer: TokenIssuer; + catalogIdentityClient: CatalogIdentityClient; + logger: Logger; }; -function transformProfile(fullProfile: FullProfile) { +function transformResult(result: OAuthResult): OAuthResult { + const { fullProfile, ...authResult } = result; + const profile = makeProfileInfo({ ...fullProfile, photos: [ ...(fullProfile.photos ?? []), - ...(fullProfile.avatarUrl ? [{ value: fullProfile.avatarUrl }] : []), + ...((fullProfile as FullProfile).avatarUrl + ? [{ value: (fullProfile as FullProfile).avatarUrl as string }] + : []), ], }); let id = fullProfile.id; + if (profile.email) { id = profile.email.split('@')[0]; } - return { id, profile }; + return { + ...authResult, + fullProfile, + }; } +export const gitlabDefaultSignInResolver: SignInResolver = async ( + info, + ctx, +) => { + const { profile } = info; + + if (!profile.email) { + throw new Error('Profile contained no email'); + } + + const userId = profile.email.split('@')[0]; + + const token = await ctx.tokenIssuer.issueToken({ + claims: { sub: userId, ent: [`user:default/${userId}`] }, + }); + + return { id: userId, token }; +}; + export class GitlabAuthProvider implements OAuthHandlers { private readonly _strategy: GitlabStrategy; + private readonly signInResolver?: SignInResolver; + private readonly authHandler: AuthHandler; + private readonly tokenIssuer: TokenIssuer; + private readonly catalogIdentityClient: CatalogIdentityClient; + private readonly logger: Logger; constructor(options: GitlabAuthProviderOptions) { + this.signInResolver = options.signInResolver; + this.authHandler = options.authHandler; + this.tokenIssuer = options.tokenIssuer; + this.logger = options.logger; + this.catalogIdentityClient = options.catalogIdentityClient; + this._strategy = new GitlabStrategy( { clientID: options.clientId, @@ -109,23 +161,9 @@ export class GitlabAuthProvider implements OAuthHandlers { OAuthResult, PrivateInfo >(req, this._strategy); - const { accessToken, params } = result; - - const { id, profile } = transformProfile(result.fullProfile); return { - response: { - profile, - providerInfo: { - accessToken, - scope: params.scope, - expiresInSeconds: params.expires_in, - idToken: params.id_token, - }, - backstageIdentity: { - id, - }, - }, + response: await this.handleResult(result), refreshToken: privateInfo.refreshToken, }; } @@ -145,30 +183,78 @@ export class GitlabAuthProvider implements OAuthHandlers { this._strategy, accessToken, ); - const { id, profile } = transformProfile(fullProfile); - return { - profile, + return this.handleResult({ + fullProfile, + params, + accessToken, + refreshToken: newRefreshToken, + }); + } + + private async handleResult(result: OAuthResult): Promise { + const { profile } = await this.authHandler(transformResult(result)); + + const response: OAuthResponse = { providerInfo: { - accessToken, - refreshToken: newRefreshToken, // GitLab expires the old refresh token when used - idToken: params.id_token, - expiresInSeconds: params.expires_in, - scope: params.scope, - }, - backstageIdentity: { - id, + idToken: result.params.id_token, + accessToken: result.accessToken, + scope: result.params.scope, + expiresInSeconds: result.params.expires_in, }, + profile, }; + + if (this.signInResolver) { + response.backstageIdentity = await this.signInResolver( + { + result, + profile, + }, + { + tokenIssuer: this.tokenIssuer, + catalogIdentityClient: this.catalogIdentityClient, + logger: this.logger, + }, + ); + } + + return response; } } -export type GitlabProviderOptions = {}; +export type GitlabProviderOptions = { + /** + * The profile transformation function used to verify and convert the auth response + * into the profile that will be presented to the user. + */ + authHandler?: AuthHandler; + + /** + * Configure sign-in for this provider, without it the provider can not be used to sign users in. + */ + /** + * Maps an auth result to a Backstage identity for the user. + * + * Set to `'email'` to use the default email-based sign in resolver, which will search + * the catalog for a single user entity that has a matching `microsoft.com/email` annotation. + */ + signIn?: { + resolver?: SignInResolver; + }; +}; export const createGitlabProvider = ( - _options?: GitlabProviderOptions, + options?: GitlabProviderOptions, ): AuthProviderFactory => { - return ({ providerId, globalConfig, config, tokenIssuer }) => + return ({ + providerId, + globalConfig, + config, + tokenIssuer, + catalogApi, + logger, + }) => OAuthEnvironmentHandler.mapConfig(config, envConfig => { const clientId = envConfig.getString('clientId'); const clientSecret = envConfig.getString('clientSecret'); @@ -176,11 +262,37 @@ export const createGitlabProvider = ( const baseUrl = audience || 'https://gitlab.com'; const callbackUrl = `${globalConfig.baseUrl}/${providerId}/handler/frame`; + const catalogIdentityClient = new CatalogIdentityClient({ + catalogApi, + tokenIssuer, + }); + + const authHandler: AuthHandler = options?.authHandler + ? options.authHandler + : async ({ fullProfile, params }) => ({ + profile: makeProfileInfo(fullProfile, params.id_token), + }); + + const signInResolverFn = + options?.signIn?.resolver ?? gitlabDefaultSignInResolver; + + const signInResolver: SignInResolver = info => + signInResolverFn(info, { + catalogIdentityClient, + tokenIssuer, + logger, + }); + const provider = new GitlabAuthProvider({ clientId, clientSecret, callbackUrl, baseUrl, + authHandler, + signInResolver, + catalogIdentityClient, + logger, + tokenIssuer, }); return OAuthAdapter.fromConfig(globalConfig, provider, { diff --git a/plugins/auth-backend/src/providers/index.ts b/plugins/auth-backend/src/providers/index.ts index 6b262dcf68..bf5615a72e 100644 --- a/plugins/auth-backend/src/providers/index.ts +++ b/plugins/auth-backend/src/providers/index.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +export * from './gitlab'; export * from './google'; export * from './microsoft'; export { factories as defaultAuthProviderFactories } from './factories'; diff --git a/plugins/auth-backend/src/providers/types.ts b/plugins/auth-backend/src/providers/types.ts index c838005c00..a8b087156d 100644 --- a/plugins/auth-backend/src/providers/types.ts +++ b/plugins/auth-backend/src/providers/types.ts @@ -196,6 +196,10 @@ export type ProfileInfo = { * signed in user. */ picture?: string; + /** + * Username of the signed in user. + */ + username?: string; }; export type SignInInfo = { From aae88bc3fa1b000a254326ea3f475f43da012307 Mon Sep 17 00:00:00 2001 From: Tom Opdebeeck Date: Mon, 19 Jul 2021 16:45:52 +0200 Subject: [PATCH 002/343] fix: cleanup username/email check Signed-off-by: Tom Opdebeeck --- .../src/providers/gitlab/provider.ts | 38 ++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/plugins/auth-backend/src/providers/gitlab/provider.ts b/plugins/auth-backend/src/providers/gitlab/provider.ts index 0fa99252b7..25440dab75 100644 --- a/plugins/auth-backend/src/providers/gitlab/provider.ts +++ b/plugins/auth-backend/src/providers/gitlab/provider.ts @@ -31,6 +31,7 @@ import { AuthProviderFactory, SignInResolver, AuthHandler, + ProfileInfo, } from '../types'; import { OAuthAdapter, @@ -63,25 +64,28 @@ export type GitlabAuthProviderOptions = OAuthProviderOptions & { logger: Logger; }; +const extractUserId = (profile: ProfileInfo): string => { + return profile.username || (profile.email?.split('@')[0] as string); +}; + function transformResult(result: OAuthResult): OAuthResult { const { fullProfile, ...authResult } = result; - const profile = makeProfileInfo({ - ...fullProfile, - photos: [ - ...(fullProfile.photos ?? []), - ...((fullProfile as FullProfile).avatarUrl - ? [{ value: (fullProfile as FullProfile).avatarUrl as string }] - : []), - ], - }); + fullProfile.photos = [ + ...(fullProfile.photos ?? []), + ...((fullProfile as FullProfile).avatarUrl + ? [{ value: (fullProfile as FullProfile).avatarUrl as string }] + : []), + ]; - let id = fullProfile.id; + const profile = makeProfileInfo(fullProfile); - if (profile.email) { - id = profile.email.split('@')[0]; + if (!profile.username && !profile.email) { + throw new Error('Profile contained no username or email'); } + fullProfile.id = extractUserId(profile); + return { ...authResult, fullProfile, @@ -94,17 +98,17 @@ export const gitlabDefaultSignInResolver: SignInResolver = async ( ) => { const { profile } = info; - if (!profile.email) { - throw new Error('Profile contained no email'); + if (!profile.username && !profile.email) { + throw new Error('Profile contained no username or email'); } - const userId = profile.email.split('@')[0]; + const id = extractUserId(profile); const token = await ctx.tokenIssuer.issueToken({ - claims: { sub: userId, ent: [`user:default/${userId}`] }, + claims: { sub: id, ent: [`user:default/${id}`] }, }); - return { id: userId, token }; + return { id, token }; }; export class GitlabAuthProvider implements OAuthHandlers { From 9b55010a460ca812aed44dae678458030a1a50ab Mon Sep 17 00:00:00 2001 From: Tom Opdebeeck Date: Mon, 19 Jul 2021 17:16:17 +0200 Subject: [PATCH 003/343] fix: cleanup provider methods Signed-off-by: Tom Opdebeeck --- .../src/providers/gitlab/provider.ts | 108 ++++++++++-------- 1 file changed, 58 insertions(+), 50 deletions(-) diff --git a/plugins/auth-backend/src/providers/gitlab/provider.ts b/plugins/auth-backend/src/providers/gitlab/provider.ts index 25440dab75..d229165562 100644 --- a/plugins/auth-backend/src/providers/gitlab/provider.ts +++ b/plugins/auth-backend/src/providers/gitlab/provider.ts @@ -58,40 +58,16 @@ type PrivateInfo = { export type GitlabAuthProviderOptions = OAuthProviderOptions & { baseUrl: string; signInResolver?: SignInResolver; - authHandler: AuthHandler; + authHandler?: AuthHandler; tokenIssuer: TokenIssuer; catalogIdentityClient: CatalogIdentityClient; logger: Logger; }; -const extractUserId = (profile: ProfileInfo): string => { +export const extractGitLabUserId = (profile: ProfileInfo): string => { return profile.username || (profile.email?.split('@')[0] as string); }; -function transformResult(result: OAuthResult): OAuthResult { - const { fullProfile, ...authResult } = result; - - fullProfile.photos = [ - ...(fullProfile.photos ?? []), - ...((fullProfile as FullProfile).avatarUrl - ? [{ value: (fullProfile as FullProfile).avatarUrl as string }] - : []), - ]; - - const profile = makeProfileInfo(fullProfile); - - if (!profile.username && !profile.email) { - throw new Error('Profile contained no username or email'); - } - - fullProfile.id = extractUserId(profile); - - return { - ...authResult, - fullProfile, - }; -} - export const gitlabDefaultSignInResolver: SignInResolver = async ( info, ctx, @@ -102,7 +78,7 @@ export const gitlabDefaultSignInResolver: SignInResolver = async ( throw new Error('Profile contained no username or email'); } - const id = extractUserId(profile); + const id = extractGitLabUserId(profile); const token = await ctx.tokenIssuer.issueToken({ claims: { sub: id, ent: [`user:default/${id}`] }, @@ -111,6 +87,13 @@ export const gitlabDefaultSignInResolver: SignInResolver = async ( return { id, token }; }; +export const gitlabDefaultAuthHandler: AuthHandler = async ({ + fullProfile, + params, +}) => ({ + profile: makeProfileInfo(fullProfile, params.id_token), +}); + export class GitlabAuthProvider implements OAuthHandlers { private readonly _strategy: GitlabStrategy; private readonly signInResolver?: SignInResolver; @@ -120,11 +103,11 @@ export class GitlabAuthProvider implements OAuthHandlers { private readonly logger: Logger; constructor(options: GitlabAuthProviderOptions) { - this.signInResolver = options.signInResolver; - this.authHandler = options.authHandler; - this.tokenIssuer = options.tokenIssuer; - this.logger = options.logger; this.catalogIdentityClient = options.catalogIdentityClient; + this.logger = options.logger; + this.tokenIssuer = options.tokenIssuer; + this.authHandler = options.authHandler || gitlabDefaultAuthHandler; + this.signInResolver = this.createSignInResolverFn(options); this._strategy = new GitlabStrategy( { @@ -197,7 +180,8 @@ export class GitlabAuthProvider implements OAuthHandlers { } private async handleResult(result: OAuthResult): Promise { - const { profile } = await this.authHandler(transformResult(result)); + const transformedResult = this.transformResult(result); + const { profile } = await this.authHandler(transformedResult); const response: OAuthResponse = { providerInfo: { @@ -225,6 +209,46 @@ export class GitlabAuthProvider implements OAuthHandlers { return response; } + + private createSignInResolverFn({ + signInResolver, + catalogIdentityClient, + tokenIssuer, + logger, + }: GitlabAuthProviderOptions): SignInResolver { + const resolver = signInResolver || gitlabDefaultSignInResolver; + + return info => + resolver(info, { + catalogIdentityClient, + tokenIssuer, + logger, + }); + } + + private transformResult(result: OAuthResult): OAuthResult { + const { fullProfile, ...authResult } = result; + + fullProfile.photos = [ + ...(fullProfile.photos ?? []), + ...((fullProfile as FullProfile).avatarUrl + ? [{ value: (fullProfile as FullProfile).avatarUrl as string }] + : []), + ]; + + const profile = makeProfileInfo(fullProfile); + + if (!profile.username && !profile.email) { + throw new Error('Profile contained no username or email'); + } + + fullProfile.id = extractGitLabUserId(profile); + + return { + ...authResult, + fullProfile, + }; + } } export type GitlabProviderOptions = { @@ -271,29 +295,13 @@ export const createGitlabProvider = ( tokenIssuer, }); - const authHandler: AuthHandler = options?.authHandler - ? options.authHandler - : async ({ fullProfile, params }) => ({ - profile: makeProfileInfo(fullProfile, params.id_token), - }); - - const signInResolverFn = - options?.signIn?.resolver ?? gitlabDefaultSignInResolver; - - const signInResolver: SignInResolver = info => - signInResolverFn(info, { - catalogIdentityClient, - tokenIssuer, - logger, - }); - const provider = new GitlabAuthProvider({ clientId, clientSecret, callbackUrl, baseUrl, - authHandler, - signInResolver, + authHandler: options?.authHandler, + signInResolver: options?.signIn?.resolver, catalogIdentityClient, logger, tokenIssuer, From db469242bec05d59cd4206a3255fa0ddcd719351 Mon Sep 17 00:00:00 2001 From: Tom Opdebeeck Date: Mon, 19 Jul 2021 17:16:26 +0200 Subject: [PATCH 004/343] fix: tests Signed-off-by: Tom Opdebeeck --- .../src/providers/github/provider.test.ts | 4 ++++ .../src/providers/gitlab/provider.test.ts | 16 ++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/plugins/auth-backend/src/providers/github/provider.test.ts b/plugins/auth-backend/src/providers/github/provider.test.ts index 4e53ce4026..5faa508f88 100644 --- a/plugins/auth-backend/src/providers/github/provider.test.ts +++ b/plugins/auth-backend/src/providers/github/provider.test.ts @@ -72,6 +72,7 @@ describe('GithubAuthProvider', () => { }, profile: { email: 'jimmymarkum@gmail.com', + username: 'jimmymarkum', displayName: 'Jimmy Markum', picture: 'https://a1cf74336522e87f135f-2f21ace9a6cf0052456644b80fa06d4f.ssl.cf2.rackcdn.com/images/characters_opt/p-mystic-river-sean-penn.jpg', @@ -117,6 +118,7 @@ describe('GithubAuthProvider', () => { }, profile: { displayName: 'Jimmy Markum', + username: 'jimmymarkum', picture: 'https://a1cf74336522e87f135f-2f21ace9a6cf0052456644b80fa06d4f.ssl.cf2.rackcdn.com/images/characters_opt/p-mystic-river-sean-penn.jpg', }, @@ -160,6 +162,7 @@ describe('GithubAuthProvider', () => { }, profile: { displayName: 'jimmymarkum', + username: 'jimmymarkum', picture: 'https://a1cf74336522e87f135f-2f21ace9a6cf0052456644b80fa06d4f.ssl.cf2.rackcdn.com/images/characters_opt/p-mystic-river-sean-penn.jpg', }, @@ -205,6 +208,7 @@ describe('GithubAuthProvider', () => { profile: { displayName: 'Dave Boyle', email: 'daveboyle@gitlab.org', + username: 'daveboyle', }, }; diff --git a/plugins/auth-backend/src/providers/gitlab/provider.test.ts b/plugins/auth-backend/src/providers/gitlab/provider.test.ts index 2277515cc9..814a060e90 100644 --- a/plugins/auth-backend/src/providers/gitlab/provider.test.ts +++ b/plugins/auth-backend/src/providers/gitlab/provider.test.ts @@ -17,6 +17,9 @@ import { GitlabAuthProvider } from './provider'; import * as helpers from '../../lib/passport/PassportStrategyHelper'; import { OAuthResult } from '../../lib/oauth'; +import { getVoidLogger } from '../../../../../packages/backend-common/src'; +import { TokenIssuer } from '../../identity'; +import { CatalogIdentityClient } from '../../lib/catalog'; const mockFrameHandler = (jest.spyOn( helpers, @@ -64,6 +67,7 @@ describe('GitlabAuthProvider', () => { profile: { email: 'jimmymarkum@gmail.com', displayName: 'Jimmy Markum', + username: 'jimmymarkum', picture: 'https://a1cf74336522e87f135f-2f21ace9a6cf0052456644b80fa06d4f.ssl.cf2.rackcdn.com/images/characters_opt/p-mystic-river-sean-penn.jpg', }, @@ -107,16 +111,28 @@ describe('GitlabAuthProvider', () => { profile: { displayName: 'Dave Boyle', email: 'daveboyle@gitlab.org', + username: 'daveboyle', }, }, }, ]; + const tokenIssuer = { + issueToken: jest.fn(), + listPublicKeys: jest.fn(), + }; + const catalogIdentityClient = { + findUser: jest.fn(), + }; + const provider = new GitlabAuthProvider({ clientId: 'mock', clientSecret: 'mock', callbackUrl: 'mock', baseUrl: 'mock', + catalogIdentityClient: (catalogIdentityClient as unknown) as CatalogIdentityClient, + tokenIssuer: (tokenIssuer as unknown) as TokenIssuer, + logger: getVoidLogger(), }); for (const test of tests) { mockFrameHandler.mockResolvedValueOnce(test.input); From 39fc3d7f807f790e178e8d5ed4383944efd73da5 Mon Sep 17 00:00:00 2001 From: Tom Opdebeeck Date: Mon, 19 Jul 2021 17:23:30 +0200 Subject: [PATCH 005/343] chore: add changeset Signed-off-by: Tom Opdebeeck --- .changeset/strong-swans-wave.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/strong-swans-wave.md diff --git a/.changeset/strong-swans-wave.md b/.changeset/strong-swans-wave.md new file mode 100644 index 0000000000..c24aceb8df --- /dev/null +++ b/.changeset/strong-swans-wave.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-auth-backend': minor +--- + +Add Sign In and Handler resolver for GitLab provider From af8e1d1b6d098a60e0c95b95110427a257ae6701 Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Wed, 4 Aug 2021 11:44:24 +0200 Subject: [PATCH 006/343] feat(techdocs-common): lowercase entity triplet path for GCS Signed-off-by: Camila Belo --- .../src/stages/publish/googleStorage.ts | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index 9def24722a..3b8ed36edd 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -13,7 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { + Entity, + EntityName, + ENTITY_DEFAULT_NAMESPACE, +} from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import { FileExistsResponse, @@ -26,7 +30,11 @@ import createLimiter from 'p-limit'; import path from 'path'; import { Readable } from 'stream'; import { Logger } from 'winston'; -import { getFileTreeRecursively, getHeadersForFileExtension } from './helpers'; +import { + getFileTreeRecursively, + getHeadersForFileExtension, + lowerCaseEntityTripletInStoragePath, +} from './helpers'; import { MigrateWriteStream } from './migrations'; import { PublisherBase, @@ -135,8 +143,13 @@ export class GoogleGCSPublish implements PublisherBase { .join(path.posix.sep); // The / delimiter is intentional since it represents the cloud storage and not the local file system. - const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; - const destination = `${entityRootDir}/${relativeFilePathPosix}`; // GCS Bucket file relative path + const entityRootDir = `${ + entity.metadata?.namespace ?? ENTITY_DEFAULT_NAMESPACE + }/${entity.kind}/${entity.metadata.name}`; + + const destination = lowerCaseEntityTripletInStoragePath( + `${entityRootDir}/${relativeFilePathPosix}`, + ); // GCS Bucket file relative path // Rate limit the concurrent execution of file uploads to batches of 10 (per publish) const uploadFile = limiter(() => @@ -190,8 +203,10 @@ export class GoogleGCSPublish implements PublisherBase { docsRouter(): express.Handler { return (req, res) => { // Decode and trim the leading forward slash - // filePath example - /default/Component/documented-component/index.html - const filePath = decodeURI(req.path.replace(/^\//, '')); + const decodedUri = decodeURI(req.path.replace(/^\//, '')); + + // filePath example - /default/component/documented-component/index.html + const filePath = lowerCaseEntityTripletInStoragePath(decodedUri); // Files with different extensions (CSS, HTML) need to be served with different headers const fileExtension = path.extname(filePath); From 1fb4b4eb44311bd4375bbeb78a7c252c50e1ed6f Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Wed, 4 Aug 2021 12:05:10 +0200 Subject: [PATCH 007/343] feat(techdocs-common): lowercase entity triplet path for AWS Signed-off-by: Camila Belo --- .../src/stages/publish/awsS3.ts | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts index 7bdc91ffd4..d08a7d55e7 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.ts @@ -13,7 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Entity, EntityName } from '@backstage/catalog-model'; +import { + Entity, + EntityName, + ENTITY_DEFAULT_NAMESPACE, +} from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import aws, { Credentials } from 'aws-sdk'; import { ListObjectsV2Output, ManagedUpload } from 'aws-sdk/clients/s3'; @@ -196,8 +200,12 @@ export class AwsS3Publish implements PublisherBase { .join(path.posix.sep); // The / delimiter is intentional since it represents the cloud storage and not the local file system. - const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; - const destination = `${entityRootDir}/${relativeFilePathPosix}`; // S3 Bucket file relative path + const entityRootDir = `${ + entity.metadata?.namespace ?? ENTITY_DEFAULT_NAMESPACE + }/${entity.kind}/${entity.metadata.name}`; + const destination = lowerCaseEntityTripletInStoragePath( + `${entityRootDir}/${relativeFilePathPosix}`, + ); // S3 Bucket file relative path // Rate limit the concurrent execution of file uploads to batches of 10 (per publish) const uploadFile = limiter(() => { @@ -268,8 +276,10 @@ export class AwsS3Publish implements PublisherBase { docsRouter(): express.Handler { return async (req, res) => { // Decode and trim the leading forward slash - // filePath example - /default/Component/documented-component/index.html - const filePath = decodeURI(req.path.replace(/^\//, '')); + const decodedUri = decodeURI(req.path.replace(/^\//, '')); + + // filePath example - /default/component/documented-component/index.html + const filePath = lowerCaseEntityTripletInStoragePath(decodedUri); // Files with different extensions (CSS, HTML) need to be served with different headers const fileExtension = path.extname(filePath); From a72470c1932067928336943c9ef307aff4ef296f Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Wed, 4 Aug 2021 12:11:19 +0200 Subject: [PATCH 008/343] feat(techdocs-common): lowercase entity triplet path for Azure Signed-off-by: Camila Belo --- .../src/stages/publish/azureBlobStorage.ts | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts index fe8fe9c2ce..8af6137e0c 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts @@ -18,7 +18,11 @@ import { BlobServiceClient, StorageSharedKeyCredential, } from '@azure/storage-blob'; -import { Entity, EntityName } from '@backstage/catalog-model'; +import { + Entity, + EntityName, + ENTITY_DEFAULT_NAMESPACE, +} from '@backstage/catalog-model'; import { Config } from '@backstage/config'; import express from 'express'; import JSON5 from 'json5'; @@ -159,8 +163,12 @@ export class AzureBlobStoragePublish implements PublisherBase { .join(path.posix.sep); // The / delimiter is intentional since it represents the cloud storage and not the local file system. - const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; - const destination = `${entityRootDir}/${relativeFilePathPosix}`; // Azure Blob Storage Container file relative path + const entityRootDir = `${ + entity.metadata?.namespace ?? ENTITY_DEFAULT_NAMESPACE + }/${entity.kind}/${entity.metadata.name}`; + const destination = lowerCaseEntityTripletInStoragePath( + `${entityRootDir}/${relativeFilePathPosix}`, + ); // Azure Blob Storage Container file relative path return limiter(async () => { const response = await this.storageClient .getContainerClient(this.containerName) @@ -259,8 +267,11 @@ export class AzureBlobStoragePublish implements PublisherBase { docsRouter(): express.Handler { return (req, res) => { // Decode and trim the leading forward slash + const decodedUri = decodeURI(req.path.replace(/^\//, '')); + // filePath example - /default/Component/documented-component/index.html - const filePath = decodeURI(req.path.replace(/^\//, '')); + const filePath = lowerCaseEntityTripletInStoragePath(decodedUri); + // Files with different extensions (CSS, HTML) need to be served with different headers const fileExtension = platformPath.extname(filePath); const responseHeaders = getHeadersForFileExtension(fileExtension); From a7a55632d398e6ef2beafee9076d45b7f3d9ebd9 Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Wed, 4 Aug 2021 22:23:06 +0200 Subject: [PATCH 009/343] test(techdocs-common): fix failing tests for all providers Co-authored-by: Eric Peterson Signed-off-by: Camila Belo --- packages/techdocs-common/__mocks__/aws-sdk.ts | 9 +- .../src/stages/publish/awsS3.test.ts | 319 +++++++---------- .../src/stages/publish/awsS3.ts | 9 +- .../stages/publish/azureBlobStorage.test.ts | 333 +++++++----------- .../src/stages/publish/azureBlobStorage.ts | 9 +- .../src/stages/publish/googleStorage.test.ts | 25 +- .../src/stages/publish/googleStorage.ts | 38 +- .../src/stages/publish/helpers.ts | 29 +- 8 files changed, 360 insertions(+), 411 deletions(-) diff --git a/packages/techdocs-common/__mocks__/aws-sdk.ts b/packages/techdocs-common/__mocks__/aws-sdk.ts index 2826a8d8bb..763fad0bda 100644 --- a/packages/techdocs-common/__mocks__/aws-sdk.ts +++ b/packages/techdocs-common/__mocks__/aws-sdk.ts @@ -42,12 +42,6 @@ const checkFileExists = async (Key: string): Promise => { }; export class S3 { - private readonly options; - - constructor(options: S3Types.ClientConfiguration) { - this.options = options; - } - headObject({ Key }: { Key: string }) { return { promise: async () => { @@ -61,7 +55,7 @@ export class S3 { getObject({ Key }: { Key: string }) { const filePath = path.join(rootDir, Key); return { - promise: () => checkFileExists(filePath), + promise: async () => await checkFileExists(filePath), createReadStream: () => { const emitter = new EventEmitter(); process.nextTick(() => { @@ -86,7 +80,6 @@ export class S3 { if (Bucket === 'errorBucket') { throw new Error('Bucket does not exist'); } - return {}; }, }; diff --git a/packages/techdocs-common/src/stages/publish/awsS3.test.ts b/packages/techdocs-common/src/stages/publish/awsS3.test.ts index 81f06716b1..d25b54c529 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.test.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.test.ts @@ -15,11 +15,7 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import { - Entity, - EntityName, - ENTITY_DEFAULT_NAMESPACE, -} from '@backstage/catalog-model'; +import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import express from 'express'; import request from 'supertest'; @@ -27,30 +23,9 @@ import mockFs from 'mock-fs'; import os from 'os'; import path from 'path'; import { AwsS3Publish } from './awsS3'; -import { PublisherBase, TechDocsMetadata } from './types'; // NOTE: /packages/techdocs-common/__mocks__ is being used to mock aws-sdk client library -const createMockEntity = (annotations = {}): Entity => { - return { - apiVersion: 'version', - kind: 'TestKind', - metadata: { - name: 'test-component-name', - namespace: 'test-namespace', - annotations: { - ...annotations, - }, - }, - }; -}; - -const createMockEntityName = (): EntityName => ({ - kind: 'TestKind', - name: 'test-component-name', - namespace: 'test-namespace', -}); - const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; const getEntityRootDir = (entity: Entity) => { @@ -64,10 +39,7 @@ const getEntityRootDir = (entity: Entity) => { const logger = getVoidLogger(); -let publisher: PublisherBase; - -beforeEach(() => { - mockFs.restore(); +const createPublisherFromConfig = (bucketName: string = 'bucketName') => { const mockConfig = new ConfigReader({ techdocs: { requestUrl: 'http://localhost:7000', @@ -78,80 +50,119 @@ beforeEach(() => { accessKeyId: 'accessKeyId', secretAccessKey: 'secretAccessKey', }, - bucketName: 'bucketName', + bucketName, }, }, }, }); - publisher = AwsS3Publish.fromConfig(mockConfig, logger); -}); + return AwsS3Publish.fromConfig(mockConfig, logger); +}; describe('AwsS3Publish', () => { + const entity = { + apiVersion: '1', + kind: 'Component', + metadata: { + name: 'backstage', + + namespace: 'default', + annotations: {}, + }, + }; + + const entityName = { + kind: 'Component', + name: 'backstage', + namespace: 'default', + }; + + const techdocsMetadata = { + site_name: 'backstage', + site_description: 'site_content', + etag: 'etag', + }; + + const localRootDir = getEntityRootDir(entity); + const storageRootDir = getEntityRootDir({ + ...entity, + kind: entity.kind.toLowerCase(), + }); + const storageSingleQuoteDir = getEntityRootDir({ + metadata: { + namespace: 'storage', + name: 'quote', + }, + kind: 'single', + } as Entity); + + beforeAll(() => { + mockFs({ + [localRootDir]: { + 'index.html': '', + '404.html': '', + assets: { + 'main.css': '', + }, + attachments: { + 'image.png': Buffer.from([2, 3, 5, 3, 5, 3, 5, 9, 1]), + }, + 'techdocs_metadata.json': JSON.stringify(techdocsMetadata), + }, + [storageRootDir]: { + 'index.html': '', + '404.html': '', + 'techdocs_metadata.json': JSON.stringify(techdocsMetadata), + assets: { + 'main.css': '', + }, + attachments: { + 'image.png': Buffer.from([2, 3, 5, 3, 5, 3, 5, 9, 1]), + }, + html: { + 'unsafe.html': '', + }, + img: { + 'with spaces.png': 'found it', + 'unsafe.svg': '', + }, + 'some folder': { + 'also with spaces.js': 'found it too', + }, + }, + [storageSingleQuoteDir]: { + 'techdocs_metadata.json': `{'site_name': 'backstage', 'site_description': 'site_content', 'etag': 'etag'}`, + }, + }); + }); + + afterAll(() => { + mockFs.restore(); + }); + describe('getReadiness', () => { it('should validate correct config', async () => { + const publisher = createPublisherFromConfig(); expect(await publisher.getReadiness()).toEqual({ isAvailable: true, }); }); it('should reject incorrect config', async () => { - const mockConfig = new ConfigReader({ - techdocs: { - requestUrl: 'http://localhost:7000', - publisher: { - type: 'awsS3', - awsS3: { - credentials: { - accessKeyId: 'accessKeyId', - secretAccessKey: 'secretAccessKey', - }, - // this bucket name will throw an error - bucketName: 'errorBucket', - }, - }, - }, - }); - - const errorPublisher = AwsS3Publish.fromConfig(mockConfig, logger); - - expect(await errorPublisher.getReadiness()).toEqual({ + const publisher = createPublisherFromConfig('errorBucket'); + expect(await publisher.getReadiness()).toEqual({ isAvailable: false, }); }); }); describe('publish', () => { - beforeEach(() => { - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - - mockFs({ - [entityRootDir]: { - 'index.html': '', - '404.html': '', - assets: { - 'main.css': '', - }, - attachments: { - 'image.png': Buffer.from([2, 3, 5, 3, 5, 3, 5, 9, 1]), - }, - }, - }); - }); - - afterEach(() => { - mockFs.restore(); - }); - it('should publish a directory', async () => { - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - + const publisher = createPublisherFromConfig(); expect( await publisher.publish({ entity, - directory: entityRootDir, + directory: localRootDir, }), ).toBeUndefined(); }); @@ -165,172 +176,114 @@ describe('AwsS3Publish', () => { 'generatedDirectory', ); - const entity = createMockEntity(); - await expect( - publisher.publish({ - entity, - directory: wrongPathToGeneratedDirectory, - }), - ).rejects.toThrowError(); + const publisher = createPublisherFromConfig(); const fails = publisher.publish({ entity, directory: wrongPathToGeneratedDirectory, }); - // Can not do exact error message match due to mockFs adding unexpected characters in the path when throwing the error - // Issue reported https://github.com/tschaub/mock-fs/issues/118 await expect(fails).rejects.toMatchObject({ message: expect.stringContaining( 'Unable to upload file(s) to AWS S3. Error: Failed to read template directory: ENOENT, no such file or directory', ), }); + await expect(fails).rejects.toMatchObject({ message: expect.stringContaining(wrongPathToGeneratedDirectory), }); - - mockFs.restore(); }); }); describe('hasDocsBeenGenerated', () => { it('should return true if docs has been generated', async () => { - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - - mockFs({ - [entityRootDir]: { - 'index.html': 'file-content', - }, - }); - + const publisher = createPublisherFromConfig(); expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); - mockFs.restore(); }); it('should return false if docs has not been generated', async () => { - const entity = createMockEntity(); - - expect(await publisher.hasDocsBeenGenerated(entity)).toBe(false); + const publisher = createPublisherFromConfig(); + expect( + await publisher.hasDocsBeenGenerated({ + kind: 'entity', + metadata: { + namespace: 'invalid', + name: 'triplet', + }, + } as Entity), + ).toBe(false); }); }); describe('fetchTechDocsMetadata', () => { it('should return tech docs metadata', async () => { - const entityNameMock = createMockEntityName(); - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - - mockFs({ - [entityRootDir]: { - 'techdocs_metadata.json': - '{"site_name": "backstage", "site_description": "site_content", "etag": "etag"}', - }, - }); - - const expectedMetadata: TechDocsMetadata = { - site_name: 'backstage', - site_description: 'site_content', - etag: 'etag', - }; - expect( - await publisher.fetchTechDocsMetadata(entityNameMock), - ).toStrictEqual(expectedMetadata); - mockFs.restore(); + const publisher = createPublisherFromConfig(); + expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( + techdocsMetadata, + ); }); it('should return tech docs metadata when json encoded with single quotes', async () => { - const entityNameMock = createMockEntityName(); - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - - mockFs({ - [entityRootDir]: { - 'techdocs_metadata.json': `{'site_name': 'backstage', 'site_description': 'site_content', 'etag': 'etag'}`, - }, - }); - - const expectedMetadata: TechDocsMetadata = { - site_name: 'backstage', - site_description: 'site_content', - etag: 'etag', - }; + const publisher = createPublisherFromConfig(); expect( - await publisher.fetchTechDocsMetadata(entityNameMock), - ).toStrictEqual(expectedMetadata); - mockFs.restore(); + await publisher.fetchTechDocsMetadata({ + namespace: 'storage', + kind: 'single', + name: 'quote', + }), + ).toStrictEqual(techdocsMetadata); }); it('should return an error if the techdocs_metadata.json file is not present', async () => { - const entityNameMock = createMockEntityName(); - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); + const publisher = createPublisherFromConfig(); - const fails = publisher.fetchTechDocsMetadata(entityNameMock); + const invalidEntityName = { + namespace: 'invalid', + kind: 'triplet', + name: 'path', + }; + + const techDocsMetadaFilePath = path.join( + rootDir, + ...Object.values(invalidEntityName), + 'techdocs_metadata.json', + ); + + const fails = publisher.fetchTechDocsMetadata(invalidEntityName); - const errorPath = path.join(entityRootDir, 'techdocs_metadata.json'); await expect(fails).rejects.toMatchObject({ - message: `TechDocs metadata fetch failed, The file ${errorPath} does not exist !`, + message: `TechDocs metadata fetch failed, The file ${techDocsMetadaFilePath} does not exist !`, }); }); }); describe('docsRouter', () => { + const entityTripletPath = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + let app: express.Express; - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); beforeEach(() => { + const publisher = createPublisherFromConfig(); app = express().use(publisher.docsRouter()); - - mockFs.restore(); - mockFs({ - [entityRootDir]: { - html: { - 'unsafe.html': '', - }, - img: { - 'with spaces.png': 'found it', - 'unsafe.svg': '', - }, - 'some folder': { - 'also with spaces.js': 'found it too', - }, - }, - }); - }); - - afterEach(() => { - mockFs.restore(); }); it('should pass expected object path to bucket', async () => { - const { - kind, - metadata: { namespace, name }, - } = entity; - // Ensures leading slash is trimmed and encoded path is decoded. const pngResponse = await request(app).get( - `/${namespace}/${kind}/${name}/img/with%20spaces.png`, + `/${entityTripletPath}/img/with%20spaces.png`, ); expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual( 'found it', ); const jsResponse = await request(app).get( - `/${namespace}/${kind}/${name}/some%20folder/also%20with%20spaces.js`, + `/${entityTripletPath}/some%20folder/also%20with%20spaces.js`, ); expect(jsResponse.text).toEqual('found it too'); }); it('should pass text/plain content-type for html', async () => { - const { - kind, - metadata: { namespace, name }, - } = entity; - const htmlResponse = await request(app).get( - `/${namespace}/${kind}/${name}/html/unsafe.html`, + `/${entityTripletPath}/html/unsafe.html`, ); expect(htmlResponse.text).toEqual(''); expect(htmlResponse.header).toMatchObject({ @@ -338,7 +291,7 @@ describe('AwsS3Publish', () => { }); const svgResponse = await request(app).get( - `/${namespace}/${kind}/${name}/img/unsafe.svg`, + `/${entityTripletPath}/img/unsafe.svg`, ); expect(svgResponse.text).toEqual(''); expect(svgResponse.header).toMatchObject({ diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts index d08a7d55e7..26a896cbac 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.ts @@ -32,6 +32,7 @@ import { Logger } from 'winston'; import { getFileTreeRecursively, getHeadersForFileExtension, + lowerCaseEntityTriplet, lowerCaseEntityTripletInStoragePath, } from './helpers'; import { @@ -238,7 +239,9 @@ export class AwsS3Publish implements PublisherBase { ): Promise { try { return await new Promise(async (resolve, reject) => { - const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; + const entityRootDir = lowerCaseEntityTriplet( + `${entityName.namespace}/${entityName.kind}/${entityName.name}`, + ); const stream = this.storageClient .getObject({ @@ -310,7 +313,9 @@ export class AwsS3Publish implements PublisherBase { */ async hasDocsBeenGenerated(entity: Entity): Promise { try { - const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + const entityRootDir = lowerCaseEntityTriplet( + `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`, + ); await this.storageClient .headObject({ Bucket: this.bucketName, diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts index e55314a71b..60a70a2e0d 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts @@ -15,11 +15,7 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import { - Entity, - EntityName, - ENTITY_DEFAULT_NAMESPACE, -} from '@backstage/catalog-model'; +import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import express from 'express'; import request from 'supertest'; @@ -27,30 +23,9 @@ import mockFs from 'mock-fs'; import os from 'os'; import path from 'path'; import { AzureBlobStoragePublish } from './azureBlobStorage'; -import { PublisherBase, TechDocsMetadata } from './types'; // NOTE: /packages/techdocs-common/__mocks__ is being used to mock Azure client library -const createMockEntity = (annotations = {}) => { - return { - apiVersion: 'version', - kind: 'TestKind', - metadata: { - name: 'test-component-name', - namespace: 'test-namespace', - annotations: { - ...annotations, - }, - }, - }; -}; - -const createMockEntityName = (): EntityName => ({ - kind: 'TestKind', - name: 'test-component-name', - namespace: 'test-namespace', -}); - const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; const getEntityRootDir = (entity: Entity) => { @@ -66,9 +41,10 @@ const logger = getVoidLogger(); jest.spyOn(logger, 'info').mockReturnValue(logger); jest.spyOn(logger, 'error').mockReturnValue(logger); -let publisher: PublisherBase; -beforeEach(async () => { - mockFs.restore(); +const createPublisherFromConfig = ({ + accountName = 'accountName', + containerName = 'containerName', +}: undefined | { accountName?: string; containerName?: string } = {}) => { const mockConfig = new ConfigReader({ techdocs: { requestUrl: 'http://localhost:7000', @@ -76,49 +52,107 @@ beforeEach(async () => { type: 'azureBlobStorage', azureBlobStorage: { credentials: { - accountName: 'accountName', + accountName, accountKey: 'accountKey', }, - containerName: 'containerName', + containerName, }, }, }, }); - publisher = AzureBlobStoragePublish.fromConfig(mockConfig, logger); -}); + return AzureBlobStoragePublish.fromConfig(mockConfig, logger); +}; describe('publishing with valid credentials', () => { + const entity = { + apiVersion: '1', + kind: 'Component', + metadata: { + name: 'backstage', + namespace: 'default', + annotations: {}, + }, + }; + + const entityName = { + kind: 'Component', + name: 'backstage', + namespace: 'default', + }; + + const techdocsMetadata = { + site_name: 'backstage', + site_description: 'site_content', + etag: 'etag', + }; + + const localRootDir = getEntityRootDir(entity); + const storageRootDir = getEntityRootDir({ + ...entity, + kind: entity.kind.toLowerCase(), + }); + const storageSingleQuoteDir = getEntityRootDir({ + metadata: { + namespace: 'storage', + name: 'quote', + }, + kind: 'single', + } as Entity); + + beforeEach(() => { + (logger.info as jest.Mock).mockClear(); + (logger.error as jest.Mock).mockClear(); + }); + + beforeAll(async () => { + mockFs({ + [localRootDir]: { + 'index.html': 'file-content', + '404.html': '', + assets: { + 'main.css': '', + }, + 'techdocs_metadata.json': JSON.stringify(techdocsMetadata), + }, + [storageRootDir]: { + 'index.html': '', + 'techdocs_metadata.json': JSON.stringify(techdocsMetadata), + html: { + 'unsafe.html': '', + }, + img: { + 'with spaces.png': 'found it', + 'unsafe.svg': '', + }, + 'some folder': { + 'also with spaces.js': 'found it too', + }, + }, + [storageSingleQuoteDir]: { + 'techdocs_metadata.json': `{'site_name': 'backstage', 'site_description': 'site_content', 'etag': 'etag'}`, + }, + }); + }); + + afterAll(() => { + mockFs.restore(); + }); + describe('getReadiness', () => { it('should validate correct config', async () => { + const publisher = createPublisherFromConfig(); expect(await publisher.getReadiness()).toEqual({ isAvailable: true, }); }); it('should reject incorrect config', async () => { - const mockConfig = new ConfigReader({ - techdocs: { - requestUrl: 'http://localhost:7000', - publisher: { - type: 'azureBlobStorage', - azureBlobStorage: { - credentials: { - accountName: 'accountName', - accountKey: 'accountKey', - }, - containerName: 'bad_container', - }, - }, - }, + const publisher = createPublisherFromConfig({ + containerName: 'bad_container', }); - const errorPublisher = await AzureBlobStoragePublish.fromConfig( - mockConfig, - logger, - ); - - expect(await errorPublisher.getReadiness()).toEqual({ + expect(await publisher.getReadiness()).toEqual({ isAvailable: false, }); @@ -131,32 +165,14 @@ describe('publishing with valid credentials', () => { }); describe('publish', () => { - beforeEach(() => { - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - - mockFs({ - [entityRootDir]: { - 'index.html': '', - '404.html': '', - assets: { - 'main.css': '', - }, - }, - }); - }); - it('should publish a directory', async () => { - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - + const publisher = createPublisherFromConfig(); expect( await publisher.publish({ entity, - directory: entityRootDir, + directory: localRootDir, }), ).toBeUndefined(); - mockFs.restore(); }); it('should fail to publish a directory', async () => { @@ -168,7 +184,9 @@ describe('publishing with valid credentials', () => { 'generatedDirectory', ); - const entity = createMockEntity(); + const publisher = createPublisherFromConfig({ + containerName: 'bad_container', + }); const fails = publisher.publish({ entity, @@ -180,46 +198,22 @@ describe('publishing with valid credentials', () => { `Unable to upload file(s) to Azure Blob Storage. Error: Failed to read template directory: ENOENT, no such file or directory`, ), }); + await expect(fails).rejects.toMatchObject({ message: expect.stringContaining(wrongPathToGeneratedDirectory), }); - - mockFs.restore(); }); it('reports an error when bad account credentials', async () => { - const mockConfig = new ConfigReader({ - techdocs: { - requestUrl: 'http://localhost:7000', - publisher: { - type: 'azureBlobStorage', - azureBlobStorage: { - credentials: { - accountName: 'failupload', - accountKey: 'accountKey', - }, - containerName: 'containerName', - }, - }, - }, - }); - - publisher = await AzureBlobStoragePublish.fromConfig(mockConfig, logger); - - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - - mockFs({ - [entityRootDir]: { - 'index.html': '', - }, + const publisher = createPublisherFromConfig({ + accountName: 'failupload', }); let error; try { await publisher.publish({ entity, - directory: entityRootDir, + directory: localRootDir, }); } catch (e) { error = e; @@ -232,91 +226,64 @@ describe('publishing with valid credentials', () => { expect(logger.error).toHaveBeenCalledWith( expect.stringContaining( `Unable to upload file(s) to Azure Blob Storage. Error: Upload failed for ${path.join( - entityRootDir, - 'index.html', + localRootDir, + '404.html', )} with status code 500`, ), ); - - mockFs.restore(); }); }); describe('hasDocsBeenGenerated', () => { it('should return true if docs has been generated', async () => { - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - - mockFs({ - [entityRootDir]: { - 'index.html': 'file-content', - }, - }); - + const publisher = createPublisherFromConfig(); expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); - mockFs.restore(); }); it('should return false if docs has not been generated', async () => { - const entity = createMockEntity(); - - expect(await publisher.hasDocsBeenGenerated(entity)).toBe(false); + const publisher = createPublisherFromConfig(); + expect( + await publisher.hasDocsBeenGenerated({ + kind: 'triplet', + metadata: { + namespace: 'invalid', + name: 'path', + }, + } as Entity), + ).toBe(false); }); }); describe('fetchTechDocsMetadata', () => { it('should return tech docs metadata', async () => { - const entityNameMock = createMockEntityName(); - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - - mockFs({ - [entityRootDir]: { - 'techdocs_metadata.json': - '{"site_name": "backstage", "site_description": "site_content", "etag": "etag"}', - }, - }); - const expectedMetadata: TechDocsMetadata = { - site_name: 'backstage', - site_description: 'site_content', - etag: 'etag', - }; - expect( - await publisher.fetchTechDocsMetadata(entityNameMock), - ).toStrictEqual(expectedMetadata); - mockFs.restore(); + const publisher = createPublisherFromConfig(); + expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( + techdocsMetadata, + ); }); it('should return tech docs metadata when json encoded with single quotes', async () => { - const entityNameMock = createMockEntityName(); - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - - mockFs({ - [entityRootDir]: { - 'techdocs_metadata.json': `{'site_name': 'backstage', 'site_description': 'site_content', 'etag': 'etag'}`, - }, - }); - - const expectedMetadata: TechDocsMetadata = { - site_name: 'backstage', - site_description: 'site_content', - etag: 'etag', - }; + const publisher = createPublisherFromConfig(); expect( - await publisher.fetchTechDocsMetadata(entityNameMock), - ).toStrictEqual(expectedMetadata); - mockFs.restore(); + await publisher.fetchTechDocsMetadata({ + namespace: 'storage', + kind: 'single', + name: 'quote', + }), + ).toStrictEqual(techdocsMetadata); }); it('should return an error if the techdocs_metadata.json file is not present', async () => { - const entityNameMock = createMockEntityName(); - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); + const publisher = createPublisherFromConfig(); + const invalidEntityName = { + namespace: 'invalid', + kind: 'triplet', + name: 'path', + }; let error; try { - await publisher.fetchTechDocsMetadata(entityNameMock); + await publisher.fetchTechDocsMetadata(invalidEntityName); } catch (e) { error = e; } @@ -324,7 +291,8 @@ describe('publishing with valid credentials', () => { expect(error).toEqual( new Error( `TechDocs metadata fetch failed, The file ${path.join( - entityRootDir, + rootDir, + ...Object.values(invalidEntityName), 'techdocs_metadata.json', )} does not exist !`, ), @@ -333,61 +301,32 @@ describe('publishing with valid credentials', () => { }); describe('docsRouter', () => { + const entityTripletPath = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + let app: express.Express; - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); beforeEach(() => { + const publisher = createPublisherFromConfig(); app = express().use(publisher.docsRouter()); - - mockFs.restore(); - mockFs({ - [entityRootDir]: { - html: { - 'unsafe.html': '', - }, - img: { - 'with spaces.png': 'found it', - 'unsafe.svg': '', - }, - 'some folder': { - 'also with spaces.js': 'found it too', - }, - }, - }); - }); - - afterEach(() => { - mockFs.restore(); }); it('should pass expected object path to bucket', async () => { - const { - kind, - metadata: { namespace, name }, - } = entity; - // Ensures leading slash is trimmed and encoded path is decoded. const pngResponse = await request(app).get( - `/${namespace}/${kind}/${name}/img/with%20spaces.png`, + `/${entityTripletPath}/img/with%20spaces.png`, ); expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual( 'found it', ); const jsResponse = await request(app).get( - `/${namespace}/${kind}/${name}/some%20folder/also%20with%20spaces.js`, + `/${entityTripletPath}/some%20folder/also%20with%20spaces.js`, ); expect(jsResponse.text).toEqual('found it too'); }); it('should pass text/plain content-type for html', async () => { - const { - kind, - metadata: { namespace, name }, - } = entity; - const htmlResponse = await request(app).get( - `/${namespace}/${kind}/${name}/html/unsafe.html`, + `/${entityTripletPath}/html/unsafe.html`, ); expect(htmlResponse.text).toEqual(''); expect(htmlResponse.header).toMatchObject({ @@ -395,7 +334,7 @@ describe('publishing with valid credentials', () => { }); const svgResponse = await request(app).get( - `/${namespace}/${kind}/${name}/img/unsafe.svg`, + `/${entityTripletPath}/img/unsafe.svg`, ); expect(svgResponse.text).toEqual(''); expect(svgResponse.header).toMatchObject({ diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts index 8af6137e0c..76f91cff66 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts @@ -32,6 +32,7 @@ import { Logger } from 'winston'; import { getFileTreeRecursively, getHeadersForFileExtension, + lowerCaseEntityTriplet, lowerCaseEntityTripletInStoragePath, } from './helpers'; import { @@ -241,7 +242,9 @@ export class AzureBlobStoragePublish implements PublisherBase { async fetchTechDocsMetadata( entityName: EntityName, ): Promise { - const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; + const entityRootDir = lowerCaseEntityTriplet( + `${entityName.namespace}/${entityName.kind}/${entityName.name}`, + ); try { const techdocsMetadataJson = await this.download( this.containerName, @@ -298,7 +301,9 @@ export class AzureBlobStoragePublish implements PublisherBase { * can be used to verify if there are any pre-generated docs available to serve. */ hasDocsBeenGenerated(entity: Entity): Promise { - const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + const entityRootDir = lowerCaseEntityTriplet( + `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`, + ); return this.storageClient .getContainerClient(this.containerName) .getBlockBlobClient(`${entityRootDir}/index.html`) diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts index 8a00fc9708..a1c9d70fd3 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts @@ -189,7 +189,10 @@ describe('GoogleGCSPublish', () => { describe('hasDocsBeenGenerated', () => { it('should return true if docs has been generated', async () => { const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); + const entityRootDir = getEntityRootDir({ + ...entity, + kind: entity.kind.toLowerCase(), + }); mockFs({ [entityRootDir]: { @@ -216,7 +219,10 @@ describe('GoogleGCSPublish', () => { it('should return tech docs metadata', async () => { const entityNameMock = createMockEntityName(); const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); + const entityRootDir = getEntityRootDir({ + ...entity, + kind: entity.kind.toLowerCase(), + }); mockFs({ [entityRootDir]: { @@ -238,7 +244,10 @@ describe('GoogleGCSPublish', () => { it('should return tech docs metadata when json encoded with single quotes', async () => { const entityNameMock = createMockEntityName(); const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); + const entityRootDir = getEntityRootDir({ + ...entity, + kind: entity.kind.toLowerCase(), + }); mockFs({ [entityRootDir]: { @@ -261,7 +270,10 @@ describe('GoogleGCSPublish', () => { it('should return an error if the techdocs_metadata.json file is not present', async () => { const entityNameMock = createMockEntityName(); const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); + const entityRootDir = getEntityRootDir({ + ...entity, + kind: entity.kind.toLowerCase(), + }); const fails = publisher.fetchTechDocsMetadata(entityNameMock); @@ -277,7 +289,10 @@ describe('GoogleGCSPublish', () => { describe('docsRouter', () => { let app: express.Express; const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); + const entityRootDir = getEntityRootDir({ + ...entity, + kind: entity.kind.toLowerCase(), + }); beforeEach(() => { app = express().use(publisher.docsRouter()); diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index 3b8ed36edd..3aa7b09b74 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -33,6 +33,7 @@ import { Logger } from 'winston'; import { getFileTreeRecursively, getHeadersForFileExtension, + lowerCaseEntityTriplet, lowerCaseEntityTripletInStoragePath, } from './helpers'; import { MigrateWriteStream } from './migrations'; @@ -77,17 +78,29 @@ export class GoogleGCSPublish implements PublisherBase { }), }); - return new GoogleGCSPublish(storageClient, bucketName, logger); + const legacyPathCasing = + config.getOptionalBoolean( + 'techdocs.legacyUseCaseSensitiveTripletPaths', + ) || false; + + return new GoogleGCSPublish( + storageClient, + bucketName, + legacyPathCasing, + logger, + ); } constructor( private readonly storageClient: Storage, private readonly bucketName: string, + private readonly legacyPathCasing: boolean, private readonly logger: Logger, ) { this.storageClient = storageClient; this.bucketName = bucketName; this.logger = logger; + this.legacyPathCasing = legacyPathCasing; } /** @@ -147,9 +160,11 @@ export class GoogleGCSPublish implements PublisherBase { entity.metadata?.namespace ?? ENTITY_DEFAULT_NAMESPACE }/${entity.kind}/${entity.metadata.name}`; - const destination = lowerCaseEntityTripletInStoragePath( - `${entityRootDir}/${relativeFilePathPosix}`, - ); // GCS Bucket file relative path + const destination = this.legacyPathCasing + ? lowerCaseEntityTripletInStoragePath( + `${entityRootDir}/${relativeFilePathPosix}`, + ) + : `${entityRootDir}/${relativeFilePathPosix}`; // GCS Bucket file relative path // Rate limit the concurrent execution of file uploads to batches of 10 (per publish) const uploadFile = limiter(() => @@ -174,7 +189,10 @@ export class GoogleGCSPublish implements PublisherBase { fetchTechDocsMetadata(entityName: EntityName): Promise { return new Promise((resolve, reject) => { - const entityRootDir = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; + const entityTriplet = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; + const entityRootDir = this.legacyPathCasing + ? entityTriplet + : lowerCaseEntityTriplet(entityTriplet); const fileStreamChunks: Array = []; this.storageClient @@ -206,7 +224,9 @@ export class GoogleGCSPublish implements PublisherBase { const decodedUri = decodeURI(req.path.replace(/^\//, '')); // filePath example - /default/component/documented-component/index.html - const filePath = lowerCaseEntityTripletInStoragePath(decodedUri); + const filePath = this.legacyPathCasing + ? decodedUri + : lowerCaseEntityTripletInStoragePath(decodedUri); // Files with different extensions (CSS, HTML) need to be served with different headers const fileExtension = path.extname(filePath); @@ -239,7 +259,11 @@ export class GoogleGCSPublish implements PublisherBase { */ async hasDocsBeenGenerated(entity: Entity): Promise { return new Promise(resolve => { - const entityRootDir = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + const entityTriplet = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + const entityRootDir = this.legacyPathCasing + ? entityTriplet + : lowerCaseEntityTriplet(entityTriplet); + this.storageClient .bucket(this.bucketName) .file(`${entityRootDir}/index.html`) diff --git a/packages/techdocs-common/src/stages/publish/helpers.ts b/packages/techdocs-common/src/stages/publish/helpers.ts index ed681eb8ba..dd62865468 100644 --- a/packages/techdocs-common/src/stages/publish/helpers.ts +++ b/packages/techdocs-common/src/stages/publish/helpers.ts @@ -83,6 +83,23 @@ export const getFileTreeRecursively = async ( return fileList; }; +/** + * Returns a lower-cased version of entity's triplet in the form of a path. + * + * Path must not include a starting slash. + * + * @example + * lowerCaseEntityTriplet('default/Component/backstage') + * // return default/component/backstage + */ +export const lowerCaseEntityTriplet = (originalPath: string): string => { + const [namespace, kind, name, ...parts] = originalPath.split('/'); + const lowerNamespace = namespace.toLowerCase(); + const lowerKind = kind.toLowerCase(); + const lowerName = name.toLowerCase(); + return [lowerNamespace, lowerKind, lowerName, ...parts].join('/'); +}; + /** * Returns the version of an object's storage path where the first three parts * of the path (the entity triplet of namespace, kind, and name) are @@ -90,9 +107,11 @@ export const getFileTreeRecursively = async ( * * Path must not include a starting slash. * + * Throws an error if the path does not appear to be an entity triplet. + * * @example - * lowerCaseEntityTripletInStoragePath('default/Component/backstage') - * // return default/component/backstage + * lowerCaseEntityTripletInStoragePath('default/Component/backstage/file.txt') + * // return default/component/backstage/file.txt */ export const lowerCaseEntityTripletInStoragePath = ( originalPath: string, @@ -105,9 +124,5 @@ export const lowerCaseEntityTripletInStoragePath = ( `Encountered file unmanaged by TechDocs ${originalPath}. Skipping.`, ); } - const [namespace, kind, name, ...parts] = originalPath.split('/'); - const lowerNamespace = namespace.toLowerCase(); - const lowerKind = kind.toLowerCase(); - const lowerName = name.toLowerCase(); - return [lowerNamespace, lowerKind, lowerName, ...parts].join('/'); + return lowerCaseEntityTriplet(originalPath); }; From 5fd7316f5d285d2d1ebb53a3f4ba04e3d9054e8f Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Thu, 5 Aug 2021 21:19:51 -0400 Subject: [PATCH 010/343] Improve search logging Signed-off-by: Adam Harvey --- plugins/search-backend/src/service/router.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/search-backend/src/service/router.ts b/plugins/search-backend/src/service/router.ts index 27b8acf9cb..3a0a43522e 100644 --- a/plugins/search-backend/src/service/router.ts +++ b/plugins/search-backend/src/service/router.ts @@ -38,7 +38,7 @@ export async function createRouter({ ) => { const { term, filters = {}, pageCursor = '' } = req.query; logger.info( - `Search request received: ${term}, ${JSON.stringify( + `Search request received: term="${term}", filters=${JSON.stringify( filters, )}, ${pageCursor}`, ); From 64baedea523bd5a7b73452bec7f524cad1c4c0bd Mon Sep 17 00:00:00 2001 From: Adam Harvey Date: Thu, 5 Aug 2021 21:20:59 -0400 Subject: [PATCH 011/343] Add changeset Signed-off-by: Adam Harvey --- .changeset/nasty-queens-juggle.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/nasty-queens-juggle.md diff --git a/.changeset/nasty-queens-juggle.md b/.changeset/nasty-queens-juggle.md new file mode 100644 index 0000000000..8204190cee --- /dev/null +++ b/.changeset/nasty-queens-juggle.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-search-backend': patch +--- + +Improve search query logging message From bf4ab567502c0bb29a425e775e6f211a4d1d6d84 Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Thu, 5 Aug 2021 15:55:41 +0200 Subject: [PATCH 012/343] feat(techdocs-common): add legacy casing config for all providers Co-authored-by: Eric Peterson Signed-off-by: Camila Belo --- .../src/stages/publish/awsS3.test.ts | 120 ++++-- .../src/stages/publish/awsS3.ts | 41 +- .../stages/publish/azureBlobStorage.test.ts | 105 +++-- .../src/stages/publish/azureBlobStorage.ts | 43 +- .../src/stages/publish/googleStorage.test.ts | 373 +++++++++--------- .../src/stages/publish/googleStorage.ts | 9 +- plugins/techdocs-backend/config.d.ts | 5 + 7 files changed, 416 insertions(+), 280 deletions(-) diff --git a/packages/techdocs-common/src/stages/publish/awsS3.test.ts b/packages/techdocs-common/src/stages/publish/awsS3.test.ts index d25b54c529..715786e3f9 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.test.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.test.ts @@ -39,7 +39,13 @@ const getEntityRootDir = (entity: Entity) => { const logger = getVoidLogger(); -const createPublisherFromConfig = (bucketName: string = 'bucketName') => { +const createPublisherFromConfig = ({ + bucketName = 'bucketName', + legacyUseCaseSensitiveTripletPaths = false, +}: { + bucketName?: string; + legacyUseCaseSensitiveTripletPaths?: boolean; +} = {}) => { const mockConfig = new ConfigReader({ techdocs: { requestUrl: 'http://localhost:7000', @@ -53,6 +59,7 @@ const createPublisherFromConfig = (bucketName: string = 'bucketName') => { bucketName, }, }, + legacyUseCaseSensitiveTripletPaths, }, }); @@ -96,42 +103,37 @@ describe('AwsS3Publish', () => { kind: 'single', } as Entity); + const files = { + 'index.html': '', + '404.html': '', + 'techdocs_metadata.json': JSON.stringify(techdocsMetadata), + assets: { + 'main.css': '', + }, + attachments: { + 'image.png': Buffer.from([2, 3, 5, 3, 5, 3, 5, 9, 1]), + }, + html: { + 'unsafe.html': '', + }, + img: { + 'with spaces.png': 'found it', + 'unsafe.svg': '', + }, + 'some folder': { + 'also with spaces.js': 'found it too', + }, + }; + beforeAll(() => { mockFs({ - [localRootDir]: { - 'index.html': '', - '404.html': '', - assets: { - 'main.css': '', - }, - attachments: { - 'image.png': Buffer.from([2, 3, 5, 3, 5, 3, 5, 9, 1]), - }, - 'techdocs_metadata.json': JSON.stringify(techdocsMetadata), - }, - [storageRootDir]: { - 'index.html': '', - '404.html': '', - 'techdocs_metadata.json': JSON.stringify(techdocsMetadata), - assets: { - 'main.css': '', - }, - attachments: { - 'image.png': Buffer.from([2, 3, 5, 3, 5, 3, 5, 9, 1]), - }, - html: { - 'unsafe.html': '', - }, - img: { - 'with spaces.png': 'found it', - 'unsafe.svg': '', - }, - 'some folder': { - 'also with spaces.js': 'found it too', - }, - }, + [localRootDir]: files, + [storageRootDir]: files, [storageSingleQuoteDir]: { - 'techdocs_metadata.json': `{'site_name': 'backstage', 'site_description': 'site_content', 'etag': 'etag'}`, + 'techdocs_metadata.json': files['techdocs_metadata.json'].replace( + /"/g, + "'", + ), }, }); }); @@ -149,7 +151,9 @@ describe('AwsS3Publish', () => { }); it('should reject incorrect config', async () => { - const publisher = createPublisherFromConfig('errorBucket'); + const publisher = createPublisherFromConfig({ + bucketName: 'errorBucket', + }); expect(await publisher.getReadiness()).toEqual({ isAvailable: false, }); @@ -167,6 +171,18 @@ describe('AwsS3Publish', () => { ).toBeUndefined(); }); + it('should publish a directory as well when legacy casing is used', async () => { + const publisher = createPublisherFromConfig({ + legacyUseCaseSensitiveTripletPaths: true, + }); + expect( + await publisher.publish({ + entity, + directory: localRootDir, + }), + ).toBeUndefined(); + }); + it('should fail to publish a directory', async () => { const wrongPathToGeneratedDirectory = path.join( rootDir, @@ -201,6 +217,13 @@ describe('AwsS3Publish', () => { expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); }); + it('should return true if docs has been generated even if the legacy case is enabled', async () => { + const publisher = createPublisherFromConfig({ + legacyUseCaseSensitiveTripletPaths: true, + }); + expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); + }); + it('should return false if docs has not been generated', async () => { const publisher = createPublisherFromConfig(); expect( @@ -223,6 +246,15 @@ describe('AwsS3Publish', () => { ); }); + it('should return tech docs metadata even if the legacy case is enabled', async () => { + const publisher = createPublisherFromConfig({ + legacyUseCaseSensitiveTripletPaths: true, + }); + expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( + techdocsMetadata, + ); + }); + it('should return tech docs metadata when json encoded with single quotes', async () => { const publisher = createPublisherFromConfig(); expect( @@ -281,6 +313,24 @@ describe('AwsS3Publish', () => { expect(jsResponse.text).toEqual('found it too'); }); + it('should pass expected object path to bucket even if the legacy case is enabled', async () => { + const publisher = createPublisherFromConfig({ + legacyUseCaseSensitiveTripletPaths: true, + }); + app = express().use(publisher.docsRouter()); + // Ensures leading slash is trimmed and encoded path is decoded. + const pngResponse = await request(app).get( + `/${entityTripletPath}/img/with%20spaces.png`, + ); + expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual( + 'found it', + ); + const jsResponse = await request(app).get( + `/${entityTripletPath}/some%20folder/also%20with%20spaces.js`, + ); + expect(jsResponse.text).toEqual('found it too'); + }); + it('should pass text/plain content-type for html', async () => { const htmlResponse = await request(app).get( `/${entityTripletPath}/html/unsafe.html`, diff --git a/packages/techdocs-common/src/stages/publish/awsS3.ts b/packages/techdocs-common/src/stages/publish/awsS3.ts index 26a896cbac..f0e2d9cb6e 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.ts @@ -102,7 +102,17 @@ export class AwsS3Publish implements PublisherBase { ...(s3ForcePathStyle && { s3ForcePathStyle }), }); - return new AwsS3Publish(storageClient, bucketName, logger); + const legacyPathCasing = + config.getOptionalBoolean( + 'techdocs.legacyUseCaseSensitiveTripletPaths', + ) || false; + + return new AwsS3Publish( + storageClient, + bucketName, + legacyPathCasing, + logger, + ); } private static buildCredentials( @@ -139,10 +149,12 @@ export class AwsS3Publish implements PublisherBase { constructor( private readonly storageClient: aws.S3, private readonly bucketName: string, + private readonly legacyPathCasing: boolean, private readonly logger: Logger, ) { this.storageClient = storageClient; this.bucketName = bucketName; + this.legacyPathCasing = legacyPathCasing; this.logger = logger; } @@ -204,9 +216,11 @@ export class AwsS3Publish implements PublisherBase { const entityRootDir = `${ entity.metadata?.namespace ?? ENTITY_DEFAULT_NAMESPACE }/${entity.kind}/${entity.metadata.name}`; - const destination = lowerCaseEntityTripletInStoragePath( - `${entityRootDir}/${relativeFilePathPosix}`, - ); // S3 Bucket file relative path + + const relativeFilePathTriplet = `${entityRootDir}/${relativeFilePathPosix}`; + const destination = this.legacyPathCasing + ? relativeFilePathTriplet + : lowerCaseEntityTripletInStoragePath(relativeFilePathTriplet); // S3 Bucket file relative path // Rate limit the concurrent execution of file uploads to batches of 10 (per publish) const uploadFile = limiter(() => { @@ -239,9 +253,10 @@ export class AwsS3Publish implements PublisherBase { ): Promise { try { return await new Promise(async (resolve, reject) => { - const entityRootDir = lowerCaseEntityTriplet( - `${entityName.namespace}/${entityName.kind}/${entityName.name}`, - ); + const entityTriplet = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; + const entityRootDir = this.legacyPathCasing + ? entityTriplet + : lowerCaseEntityTriplet(entityTriplet); const stream = this.storageClient .getObject({ @@ -282,7 +297,9 @@ export class AwsS3Publish implements PublisherBase { const decodedUri = decodeURI(req.path.replace(/^\//, '')); // filePath example - /default/component/documented-component/index.html - const filePath = lowerCaseEntityTripletInStoragePath(decodedUri); + const filePath = this.legacyPathCasing + ? decodedUri + : lowerCaseEntityTripletInStoragePath(decodedUri); // Files with different extensions (CSS, HTML) need to be served with different headers const fileExtension = path.extname(filePath); @@ -313,9 +330,11 @@ export class AwsS3Publish implements PublisherBase { */ async hasDocsBeenGenerated(entity: Entity): Promise { try { - const entityRootDir = lowerCaseEntityTriplet( - `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`, - ); + const entityTriplet = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + const entityRootDir = this.legacyPathCasing + ? entityTriplet + : lowerCaseEntityTriplet(entityTriplet); + await this.storageClient .headObject({ Bucket: this.bucketName, diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts index 60a70a2e0d..78cc7c0c79 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts @@ -44,7 +44,12 @@ jest.spyOn(logger, 'error').mockReturnValue(logger); const createPublisherFromConfig = ({ accountName = 'accountName', containerName = 'containerName', -}: undefined | { accountName?: string; containerName?: string } = {}) => { + legacyUseCaseSensitiveTripletPaths = false, +}: { + accountName?: string; + containerName?: string; + legacyUseCaseSensitiveTripletPaths?: boolean; +} = {}) => { const mockConfig = new ConfigReader({ techdocs: { requestUrl: 'http://localhost:7000', @@ -58,6 +63,7 @@ const createPublisherFromConfig = ({ containerName, }, }, + legacyUseCaseSensitiveTripletPaths, }, }); @@ -105,32 +111,37 @@ describe('publishing with valid credentials', () => { (logger.error as jest.Mock).mockClear(); }); + const files = { + 'index.html': '', + '404.html': '', + 'techdocs_metadata.json': JSON.stringify(techdocsMetadata), + assets: { + 'main.css': '', + }, + attachments: { + 'image.png': Buffer.from([2, 3, 5, 3, 5, 3, 5, 9, 1]), + }, + html: { + 'unsafe.html': '', + }, + img: { + 'with spaces.png': 'found it', + 'unsafe.svg': '', + }, + 'some folder': { + 'also with spaces.js': 'found it too', + }, + }; + beforeAll(async () => { mockFs({ - [localRootDir]: { - 'index.html': 'file-content', - '404.html': '', - assets: { - 'main.css': '', - }, - 'techdocs_metadata.json': JSON.stringify(techdocsMetadata), - }, - [storageRootDir]: { - 'index.html': '', - 'techdocs_metadata.json': JSON.stringify(techdocsMetadata), - html: { - 'unsafe.html': '', - }, - img: { - 'with spaces.png': 'found it', - 'unsafe.svg': '', - }, - 'some folder': { - 'also with spaces.js': 'found it too', - }, - }, + [localRootDir]: files, + [storageRootDir]: files, [storageSingleQuoteDir]: { - 'techdocs_metadata.json': `{'site_name': 'backstage', 'site_description': 'site_content', 'etag': 'etag'}`, + 'techdocs_metadata.json': files['techdocs_metadata.json'].replace( + /"/g, + "'", + ), }, }); }); @@ -175,6 +186,18 @@ describe('publishing with valid credentials', () => { ).toBeUndefined(); }); + it('should publish a directory as well when legacy casing is used', async () => { + const publisher = createPublisherFromConfig({ + legacyUseCaseSensitiveTripletPaths: true, + }); + expect( + await publisher.publish({ + entity, + directory: localRootDir, + }), + ).toBeUndefined(); + }); + it('should fail to publish a directory', async () => { const wrongPathToGeneratedDirectory = path.join( rootDir, @@ -240,6 +263,13 @@ describe('publishing with valid credentials', () => { expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); }); + it('should return true if docs has been generated even if the legacy case is enabled', async () => { + const publisher = createPublisherFromConfig({ + legacyUseCaseSensitiveTripletPaths: true, + }); + expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); + }); + it('should return false if docs has not been generated', async () => { const publisher = createPublisherFromConfig(); expect( @@ -262,6 +292,15 @@ describe('publishing with valid credentials', () => { ); }); + it('should return tech docs metadata even if the legacy case is enabled', async () => { + const publisher = createPublisherFromConfig({ + legacyUseCaseSensitiveTripletPaths: true, + }); + expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( + techdocsMetadata, + ); + }); + it('should return tech docs metadata when json encoded with single quotes', async () => { const publisher = createPublisherFromConfig(); expect( @@ -324,6 +363,24 @@ describe('publishing with valid credentials', () => { expect(jsResponse.text).toEqual('found it too'); }); + it('should pass expected object path to bucket even if the legacy case is enabled', async () => { + const publisher = createPublisherFromConfig({ + legacyUseCaseSensitiveTripletPaths: true, + }); + app = express().use(publisher.docsRouter()); + // Ensures leading slash is trimmed and encoded path is decoded. + const pngResponse = await request(app).get( + `/${entityTripletPath}/img/with%20spaces.png`, + ); + expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual( + 'found it', + ); + const jsResponse = await request(app).get( + `/${entityTripletPath}/some%20folder/also%20with%20spaces.js`, + ); + expect(jsResponse.text).toEqual('found it too'); + }); + it('should pass text/plain content-type for html', async () => { const htmlResponse = await request(app).get( `/${entityTripletPath}/html/unsafe.html`, diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts index 76f91cff66..cb8367ed9d 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.ts @@ -89,16 +89,28 @@ export class AzureBlobStoragePublish implements PublisherBase { credential, ); - return new AzureBlobStoragePublish(storageClient, containerName, logger); + const legacyPathCasing = + config.getOptionalBoolean( + 'techdocs.legacyUseCaseSensitiveTripletPaths', + ) || false; + + return new AzureBlobStoragePublish( + storageClient, + containerName, + legacyPathCasing, + logger, + ); } constructor( private readonly storageClient: BlobServiceClient, private readonly containerName: string, + private readonly legacyPathCasing: boolean, private readonly logger: Logger, ) { this.storageClient = storageClient; this.containerName = containerName; + this.legacyPathCasing = legacyPathCasing; this.logger = logger; } @@ -167,9 +179,12 @@ export class AzureBlobStoragePublish implements PublisherBase { const entityRootDir = `${ entity.metadata?.namespace ?? ENTITY_DEFAULT_NAMESPACE }/${entity.kind}/${entity.metadata.name}`; - const destination = lowerCaseEntityTripletInStoragePath( - `${entityRootDir}/${relativeFilePathPosix}`, - ); // Azure Blob Storage Container file relative path + + const relativeFilePathTriplet = `${entityRootDir}/${relativeFilePathPosix}`; + const destination = this.legacyPathCasing + ? relativeFilePathTriplet + : lowerCaseEntityTripletInStoragePath(relativeFilePathTriplet); // Azure Blob Storage Container file relative path + return limiter(async () => { const response = await this.storageClient .getContainerClient(this.containerName) @@ -242,9 +257,11 @@ export class AzureBlobStoragePublish implements PublisherBase { async fetchTechDocsMetadata( entityName: EntityName, ): Promise { - const entityRootDir = lowerCaseEntityTriplet( - `${entityName.namespace}/${entityName.kind}/${entityName.name}`, - ); + const entityTriplet = `${entityName.namespace}/${entityName.kind}/${entityName.name}`; + const entityRootDir = this.legacyPathCasing + ? entityTriplet + : lowerCaseEntityTriplet(entityTriplet); + try { const techdocsMetadataJson = await this.download( this.containerName, @@ -273,7 +290,9 @@ export class AzureBlobStoragePublish implements PublisherBase { const decodedUri = decodeURI(req.path.replace(/^\//, '')); // filePath example - /default/Component/documented-component/index.html - const filePath = lowerCaseEntityTripletInStoragePath(decodedUri); + const filePath = this.legacyPathCasing + ? decodedUri + : lowerCaseEntityTripletInStoragePath(decodedUri); // Files with different extensions (CSS, HTML) need to be served with different headers const fileExtension = platformPath.extname(filePath); @@ -301,9 +320,11 @@ export class AzureBlobStoragePublish implements PublisherBase { * can be used to verify if there are any pre-generated docs available to serve. */ hasDocsBeenGenerated(entity: Entity): Promise { - const entityRootDir = lowerCaseEntityTriplet( - `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`, - ); + const entityTriplet = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + const entityRootDir = this.legacyPathCasing + ? entityTriplet + : lowerCaseEntityTriplet(entityTriplet); + return this.storageClient .getContainerClient(this.containerName) .getBlockBlobClient(`${entityRootDir}/index.html`) diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts index a1c9d70fd3..ce08e2f2ce 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts @@ -15,11 +15,7 @@ */ import { getVoidLogger } from '@backstage/backend-common'; -import { - Entity, - ENTITY_DEFAULT_NAMESPACE, - EntityName, -} from '@backstage/catalog-model'; +import { Entity, ENTITY_DEFAULT_NAMESPACE } from '@backstage/catalog-model'; import { ConfigReader } from '@backstage/config'; import express from 'express'; import request from 'supertest'; @@ -27,30 +23,9 @@ import mockFs from 'mock-fs'; import os from 'os'; import path from 'path'; import { GoogleGCSPublish } from './googleStorage'; -import { PublisherBase, TechDocsMetadata } from './types'; // NOTE: /packages/techdocs-common/__mocks__ is being used to mock Google Cloud Storage client library -const createMockEntity = (annotations = {}): Entity => { - return { - apiVersion: 'version', - kind: 'TestKind', - metadata: { - name: 'test-component-name', - namespace: 'test-namespace', - annotations: { - ...annotations, - }, - }, - }; -}; - -const createMockEntityName = (): EntityName => ({ - kind: 'TestKind', - name: 'test-component-name', - namespace: 'test-namespace', -}); - const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; const getEntityRootDir = (entity: Entity) => { @@ -65,10 +40,13 @@ const getEntityRootDir = (entity: Entity) => { const logger = getVoidLogger(); jest.spyOn(logger, 'info').mockReturnValue(logger); -let publisher: PublisherBase; - -beforeEach(async () => { - mockFs.restore(); +const createPublisherFromConfig = ({ + bucketName = 'bucketName', + legacyUseCaseSensitiveTripletPaths = false, +}: { + bucketName?: string; + legacyUseCaseSensitiveTripletPaths?: boolean; +} = {}) => { const mockConfig = new ConfigReader({ techdocs: { requestUrl: 'http://localhost:7000', @@ -76,76 +54,127 @@ beforeEach(async () => { type: 'googleGcs', googleGcs: { credentials: '{}', - bucketName: 'bucketName', + bucketName, }, }, + legacyUseCaseSensitiveTripletPaths, }, }); - publisher = await GoogleGCSPublish.fromConfig(mockConfig, logger); -}); + return GoogleGCSPublish.fromConfig(mockConfig, logger); +}; describe('GoogleGCSPublish', () => { + const entity = { + apiVersion: 'version', + kind: 'Component', + metadata: { + name: 'backstage', + namespace: 'default', + annotations: {}, + }, + }; + + const entityName = { + kind: 'Component', + name: 'backstage', + namespace: 'default', + }; + + const techdocsMetadata = { + site_name: 'backstage', + site_description: 'site_content', + etag: 'etag', + }; + + const localRootDir = getEntityRootDir(entity); + const storageRootDir = getEntityRootDir({ + ...entity, + kind: entity.kind.toLowerCase(), + }); + const storageSingleQuoteDir = getEntityRootDir({ + metadata: { + namespace: 'storage', + name: 'quote', + }, + kind: 'single', + } as Entity); + + const files = { + 'index.html': '', + '404.html': '', + assets: { + 'main.css': '', + }, + 'techdocs_metadata.json': JSON.stringify(techdocsMetadata), + html: { + 'unsafe.html': '', + }, + img: { + 'with spaces.png': 'found it', + 'unsafe.svg': '', + }, + 'some folder': { + 'also with spaces.js': 'found it too', + }, + }; + + beforeAll(() => { + mockFs({ + [localRootDir]: files, + [storageRootDir]: files, + [storageSingleQuoteDir]: { + 'techdocs_metadata.json': files['techdocs_metadata.json'].replace( + /"/g, + "'", + ), + }, + }); + }); + + afterAll(() => { + mockFs.restore(); + }); + describe('getReadiness', () => { it('should validate correct config', async () => { + const publisher = createPublisherFromConfig(); expect(await publisher.getReadiness()).toEqual({ isAvailable: true, }); }); it('should reject incorrect config', async () => { - const mockConfig = new ConfigReader({ - techdocs: { - requestUrl: 'http://localhost:7000', - publisher: { - type: 'googleGcs', - googleGcs: { - credentials: '{}', - bucketName: 'errorBucket', - }, - }, - }, + const publisher = createPublisherFromConfig({ + bucketName: 'errorBucket', }); - - const errorPublisher = GoogleGCSPublish.fromConfig(mockConfig, logger); - - expect(await errorPublisher.getReadiness()).toEqual({ + expect(await publisher.getReadiness()).toEqual({ isAvailable: false, }); }); }); describe('publish', () => { - beforeEach(() => { - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - - mockFs({ - [entityRootDir]: { - 'index.html': '', - '404.html': '', - assets: { - 'main.css': '', - }, - }, - }); - }); - - afterEach(() => { - mockFs.restore(); - }); - it('should publish a directory', async () => { - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir(entity); - + const publisher = createPublisherFromConfig(); expect( await publisher.publish({ entity, - directory: entityRootDir, + directory: localRootDir, + }), + ).toBeUndefined(); + }); + + it('should publish a directory as well when legacy casing is used', async () => { + const publisher = createPublisherFromConfig({ + legacyUseCaseSensitiveTripletPaths: true, + }); + expect( + await publisher.publish({ + entity, + directory: localRootDir, }), ).toBeUndefined(); - mockFs.restore(); }); it('should fail to publish a directory', async () => { @@ -157,14 +186,7 @@ describe('GoogleGCSPublish', () => { 'generatedDirectory', ); - const entity = createMockEntity(); - - await expect( - publisher.publish({ - entity, - directory: wrongPathToGeneratedDirectory, - }), - ).rejects.toThrowError(); + const publisher = createPublisherFromConfig(); const fails = publisher.publish({ entity, @@ -178,173 +200,136 @@ describe('GoogleGCSPublish', () => { `Unable to upload file(s) to Google Cloud Storage. Error: Failed to read template directory: ENOENT, no such file or directory`, ), }); + await expect(fails).rejects.toMatchObject({ message: expect.stringContaining(wrongPathToGeneratedDirectory), }); - - mockFs.restore(); }); }); describe('hasDocsBeenGenerated', () => { it('should return true if docs has been generated', async () => { - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir({ - ...entity, - kind: entity.kind.toLowerCase(), - }); - - mockFs({ - [entityRootDir]: { - 'index.html': 'file-content', - }, - }); - + const publisher = createPublisherFromConfig(); + expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); + }); + + it('should return true if docs has been generated even if the legacy case is enabled', async () => { + const publisher = createPublisherFromConfig({ + legacyUseCaseSensitiveTripletPaths: true, + }); expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); - mockFs.restore(); }); it('should return false if docs has not been generated', async () => { - const entity = createMockEntity(); - - expect(await publisher.hasDocsBeenGenerated(entity)).toBe(false); + const publisher = createPublisherFromConfig(); + expect( + await publisher.hasDocsBeenGenerated({ + kind: 'entity', + metadata: { + namespace: 'invalid', + name: 'triplet', + }, + } as Entity), + ).toBe(false); }); }); describe('fetchTechDocsMetadata', () => { - beforeEach(() => { - mockFs.restore(); + it('should return tech docs metadata', async () => { + const publisher = createPublisherFromConfig(); + expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( + techdocsMetadata, + ); }); - it('should return tech docs metadata', async () => { - const entityNameMock = createMockEntityName(); - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir({ - ...entity, - kind: entity.kind.toLowerCase(), + it('should return tech docs metadata even if the legacy case is enabled', async () => { + const publisher = createPublisherFromConfig({ + legacyUseCaseSensitiveTripletPaths: true, }); - - mockFs({ - [entityRootDir]: { - 'techdocs_metadata.json': - '{"site_name": "backstage", "site_description": "site_content", "etag": "etag"}', - }, - }); - - const expectedMetadata: TechDocsMetadata = { - site_name: 'backstage', - site_description: 'site_content', - etag: 'etag', - }; - expect( - await publisher.fetchTechDocsMetadata(entityNameMock), - ).toStrictEqual(expectedMetadata); + expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( + techdocsMetadata, + ); }); it('should return tech docs metadata when json encoded with single quotes', async () => { - const entityNameMock = createMockEntityName(); - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir({ - ...entity, - kind: entity.kind.toLowerCase(), - }); - - mockFs({ - [entityRootDir]: { - 'techdocs_metadata.json': - "{'site_name': 'backstage', 'site_description': 'site_content', 'etag': 'etag'}", - }, - }); - - const expectedMetadata: TechDocsMetadata = { - site_name: 'backstage', - site_description: 'site_content', - etag: 'etag', - }; + const publisher = createPublisherFromConfig(); expect( - await publisher.fetchTechDocsMetadata(entityNameMock), - ).toStrictEqual(expectedMetadata); - mockFs.restore(); + await publisher.fetchTechDocsMetadata({ + namespace: 'storage', + kind: 'single', + name: 'quote', + }), + ).toStrictEqual(techdocsMetadata); }); it('should return an error if the techdocs_metadata.json file is not present', async () => { - const entityNameMock = createMockEntityName(); - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir({ - ...entity, - kind: entity.kind.toLowerCase(), - }); + const publisher = createPublisherFromConfig(); - const fails = publisher.fetchTechDocsMetadata(entityNameMock); + const invalidEntityName = { + namespace: 'invalid', + kind: 'triplet', + name: 'path', + }; + + const techDocsMetadaFilePath = path.join( + rootDir, + ...Object.values(invalidEntityName), + 'techdocs_metadata.json', + ); + + const fails = publisher.fetchTechDocsMetadata(invalidEntityName); await expect(fails).rejects.toMatchObject({ - message: `The file ${path.join( - entityRootDir, - 'techdocs_metadata.json', - )} does not exist !`, + message: `The file ${techDocsMetadaFilePath} does not exist !`, }); }); }); describe('docsRouter', () => { + const entityTripletPath = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; + let app: express.Express; - const entity = createMockEntity(); - const entityRootDir = getEntityRootDir({ - ...entity, - kind: entity.kind.toLowerCase(), - }); beforeEach(() => { + const publisher = createPublisherFromConfig(); app = express().use(publisher.docsRouter()); - - mockFs.restore(); - mockFs({ - [entityRootDir]: { - html: { - 'unsafe.html': '', - }, - img: { - 'with spaces.png': 'found it', - 'unsafe.svg': '', - }, - 'some folder': { - 'also with spaces.js': 'found it too', - }, - }, - }); - }); - - afterEach(() => { - mockFs.restore(); }); it('should pass expected object path to bucket', async () => { - const { - kind, - metadata: { namespace, name }, - } = entity; - // Ensures leading slash is trimmed and encoded path is decoded. const pngResponse = await request(app).get( - `/${namespace}/${kind}/${name}/img/with%20spaces.png`, + `/${entityTripletPath}/img/with%20spaces.png`, ); expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual( 'found it', ); const jsResponse = await request(app).get( - `/${namespace}/${kind}/${name}/some%20folder/also%20with%20spaces.js`, + `/${entityTripletPath}/some%20folder/also%20with%20spaces.js`, + ); + expect(jsResponse.text).toEqual('found it too'); + }); + + it('should pass expected object path to bucket even if the legacy case is enabled', async () => { + const publisher = createPublisherFromConfig({ + legacyUseCaseSensitiveTripletPaths: true, + }); + app = express().use(publisher.docsRouter()); + // Ensures leading slash is trimmed and encoded path is decoded. + const pngResponse = await request(app).get( + `/${entityTripletPath}/img/with%20spaces.png`, + ); + expect(Buffer.from(pngResponse.body).toString('utf8')).toEqual( + 'found it', + ); + const jsResponse = await request(app).get( + `/${entityTripletPath}/some%20folder/also%20with%20spaces.js`, ); expect(jsResponse.text).toEqual('found it too'); }); it('should pass text/plain content-type for html', async () => { - const { - kind, - metadata: { namespace, name }, - } = entity; - const htmlResponse = await request(app).get( - `/${namespace}/${kind}/${name}/html/unsafe.html`, + `/${entityTripletPath}/html/unsafe.html`, ); expect(htmlResponse.text).toEqual(''); expect(htmlResponse.header).toMatchObject({ @@ -352,7 +337,7 @@ describe('GoogleGCSPublish', () => { }); const svgResponse = await request(app).get( - `/${namespace}/${kind}/${name}/img/unsafe.svg`, + `/${entityTripletPath}/img/unsafe.svg`, ); expect(svgResponse.text).toEqual(''); expect(svgResponse.header).toMatchObject({ diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.ts b/packages/techdocs-common/src/stages/publish/googleStorage.ts index 3aa7b09b74..9db9b6fb49 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.ts @@ -99,8 +99,8 @@ export class GoogleGCSPublish implements PublisherBase { ) { this.storageClient = storageClient; this.bucketName = bucketName; - this.logger = logger; this.legacyPathCasing = legacyPathCasing; + this.logger = logger; } /** @@ -160,11 +160,10 @@ export class GoogleGCSPublish implements PublisherBase { entity.metadata?.namespace ?? ENTITY_DEFAULT_NAMESPACE }/${entity.kind}/${entity.metadata.name}`; + const relativeFilePathTriplet = `${entityRootDir}/${relativeFilePathPosix}`; const destination = this.legacyPathCasing - ? lowerCaseEntityTripletInStoragePath( - `${entityRootDir}/${relativeFilePathPosix}`, - ) - : `${entityRootDir}/${relativeFilePathPosix}`; // GCS Bucket file relative path + ? relativeFilePathTriplet + : lowerCaseEntityTripletInStoragePath(relativeFilePathTriplet); // GCS Bucket file relative path // Rate limit the concurrent execution of file uploads to batches of 10 (per publish) const uploadFile = limiter(() => diff --git a/plugins/techdocs-backend/config.d.ts b/plugins/techdocs-backend/config.d.ts index 7f2fdf8b26..537daa815f 100644 --- a/plugins/techdocs-backend/config.d.ts +++ b/plugins/techdocs-backend/config.d.ts @@ -246,5 +246,10 @@ export interface Config { * @deprecated */ storageUrl?: string; + + /** + * (Optional) + */ + legacyUseCaseSensitiveTripletPaths?: boolean; }; } From 759f15a14a7db08f048679cd42d7a1e2b502c7d1 Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Sat, 7 Aug 2021 21:53:09 +0200 Subject: [PATCH 013/343] test(techdocs-common): refactor storage files mock Signed-off-by: Camila Belo --- .../__mocks__/@azure/storage-blob.ts | 53 ++------ .../__mocks__/@google-cloud/storage.ts | 58 +++------ packages/techdocs-common/__mocks__/aws-sdk.ts | 60 +++------ packages/techdocs-common/src/setupTests.ts | 61 +++++++++ .../src/stages/publish/awsS3.test.ts | 84 +++++------- .../stages/publish/azureBlobStorage.test.ts | 122 +++++++----------- .../src/stages/publish/googleStorage.test.ts | 86 ++++++------ 7 files changed, 229 insertions(+), 295 deletions(-) create mode 100644 packages/techdocs-common/src/setupTests.ts diff --git a/packages/techdocs-common/__mocks__/@azure/storage-blob.ts b/packages/techdocs-common/__mocks__/@azure/storage-blob.ts index 73c5873832..80c60acce7 100644 --- a/packages/techdocs-common/__mocks__/@azure/storage-blob.ts +++ b/packages/techdocs-common/__mocks__/@azure/storage-blob.ts @@ -18,29 +18,8 @@ import type { ContainerGetPropertiesResponse, } from '@azure/storage-blob'; import { EventEmitter } from 'events'; -import fs from 'fs-extra'; -import os from 'os'; -import path from 'path'; -const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; - -/** - * @param sourceFile Relative path to entity root dir. Contains either / or \ as file separator - * depending upon the OS. - */ -const checkFileExists = async (sourceFile: string): Promise => { - // sourceFile will always have / as file separator irrespective of OS since Azure expects /. - // Normalize sourceFile to OS specific path before checking if file exists. - const relativeFilePath = sourceFile.split(path.posix.sep).join(path.sep); - const filePath = path.join(rootDir, sourceFile); - - try { - await fs.access(filePath, fs.constants.F_OK); - return true; - } catch (err) { - return false; - } -}; +const storage = new (global as any).StorageFilesMock(); export class BlockBlobClient { private readonly blobName; @@ -50,9 +29,7 @@ export class BlockBlobClient { } uploadFile(source: string): Promise { - if (!fs.existsSync(source)) { - return Promise.reject(new Error(`The file ${source} does not exist`)); - } + storage.writeFile(this.blobName, source); return Promise.resolve({ _response: { request: { @@ -65,20 +42,19 @@ export class BlockBlobClient { } exists() { - return checkFileExists(this.blobName); + return storage.fileExists(this.blobName); } download() { - const filePath = path.join(rootDir, this.blobName); const emitter = new EventEmitter(); setTimeout(() => { - if (fs.existsSync(filePath)) { - emitter.emit('data', fs.readFileSync(filePath)); + if (storage.fileExists(this.blobName)) { + emitter.emit('data', storage.readFile(this.blobName)); emitter.emit('end'); } else { emitter.emit( 'error', - new Error(`The file ${filePath} does not exist !`), + new Error(`The file ${this.blobName} does not exist!`), ); } }, 0); @@ -89,7 +65,7 @@ export class BlockBlobClient { } class BlockBlobClientFailUpload extends BlockBlobClient { - uploadFile(source: string): Promise { + uploadFile(): Promise { return Promise.resolve({ _response: { request: { @@ -103,12 +79,6 @@ class BlockBlobClientFailUpload extends BlockBlobClient { } export class ContainerClient { - private readonly containerName; - - constructor(containerName: string) { - this.containerName = containerName; - } - getProperties(): Promise { return Promise.resolve({ _response: { @@ -153,18 +123,19 @@ export class BlobServiceClient { private readonly credential; constructor(url: string, credential?: StorageSharedKeyCredential) { + storage.emptyFiles(); this.url = url; this.credential = credential; } getContainerClient(containerName: string) { if (containerName === 'bad_container') { - return new ContainerClientFailGetProperties(containerName); + return new ContainerClientFailGetProperties(); } - if (this.credential.accountName === 'failupload') { - return new ContainerClientFailUpload(containerName); + if (this.credential.accountName === 'bad_account_credentials') { + return new ContainerClientFailUpload(); } - return new ContainerClient(containerName); + return new ContainerClient(); } } diff --git a/packages/techdocs-common/__mocks__/@google-cloud/storage.ts b/packages/techdocs-common/__mocks__/@google-cloud/storage.ts index 18fe62af14..efc2454d08 100644 --- a/packages/techdocs-common/__mocks__/@google-cloud/storage.ts +++ b/packages/techdocs-common/__mocks__/@google-cloud/storage.ts @@ -14,42 +14,19 @@ * limitations under the License. */ import { Readable } from 'stream'; -import fs from 'fs-extra'; -import os from 'os'; -import path from 'path'; -type storageOptions = { - keyFilename?: string; -}; - -const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; -/** - * @param sourceFile Absolute path. Contains either / or \ as file separator depending upon the OS. - */ -const checkFileExists = async (sourceFile: string): Promise => { - // sourceFile will always have / as file separator irrespective of OS since GCS expects /. - // Normalize sourceFile to OS specific path before checking if file exists. - const filePath = sourceFile.split(path.posix.sep).join(path.sep); - - try { - await fs.access(filePath, fs.constants.F_OK); - return true; - } catch (err) { - return false; - } -}; +const storage = new (global as any).StorageFilesMock(); class GCSFile { - private readonly localFilePath: string; + private readonly path: string; - constructor(private readonly destinationFilePath: string) { - this.destinationFilePath = destinationFilePath; - this.localFilePath = path.join(rootDir, this.destinationFilePath); + constructor(path: string) { + this.path = path; } exists() { return new Promise(async (resolve, reject) => { - if (await checkFileExists(this.localFilePath)) { + if (storage.fileExists(this.path)) { resolve([true]); } else { reject(); @@ -62,19 +39,20 @@ class GCSFile { readable._read = () => {}; process.nextTick(() => { - if (fs.existsSync(this.localFilePath)) { + if (storage.fileExists(this.path)) { if (readable.eventNames().includes('pipe')) { readable.emit('pipe'); } - readable.emit('data', fs.readFileSync(this.localFilePath)); + readable.emit('data', storage.readFile(this.path)); readable.emit('end'); } else { readable.emit( 'error', - new Error(`The file ${this.localFilePath} does not exist !`), + new Error(`The file ${this.path} does not exist!`), ); } }); + return readable; } } @@ -87,20 +65,16 @@ class Bucket { } async getMetadata() { - if (this.bucketName === 'errorBucket') { + if (this.bucketName === 'bad_bucket_name') { throw Error('Bucket does not exist'); } - return ''; } upload(source: string, { destination }) { - return new Promise(async (resolve, reject) => { - if (await checkFileExists(source)) { - resolve({ source, destination }); - } else { - reject(`Source file ${source} does not exist.`); - } + return new Promise(async resolve => { + storage.writeFile(destination, source); + resolve(null); }); } @@ -110,10 +84,8 @@ class Bucket { } export class Storage { - private readonly keyFilename; - - constructor(options: storageOptions) { - this.keyFilename = options.keyFilename; + constructor() { + storage.emptyFiles(); } bucket(bucketName) { diff --git a/packages/techdocs-common/__mocks__/aws-sdk.ts b/packages/techdocs-common/__mocks__/aws-sdk.ts index 763fad0bda..36c864bfe7 100644 --- a/packages/techdocs-common/__mocks__/aws-sdk.ts +++ b/packages/techdocs-common/__mocks__/aws-sdk.ts @@ -13,39 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import type { S3 as S3Types } from 'aws-sdk'; import { EventEmitter } from 'events'; -import fs from 'fs-extra'; -import os from 'os'; -import path from 'path'; +import { ReadStream } from 'fs'; export { Credentials } from 'aws-sdk'; -const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; - -/** - * @param Key Relative path to entity root dir. Contains either / or \ as file separator - * depending upon the OS. - */ -const checkFileExists = async (Key: string): Promise => { - // Key will always have / as file separator irrespective of OS since S3 expects /. - // Normalize Key to OS specific path before checking if file exists. - const relativeFilePath = Key.split(path.posix.sep).join(path.sep); - const filePath = path.join(rootDir, Key); - - try { - await fs.access(filePath, fs.constants.F_OK); - return true; - } catch (err) { - return false; - } -}; +const storage = new (global as any).StorageFilesMock(); export class S3 { + constructor() { + storage.emptyFiles(); + } + headObject({ Key }: { Key: string }) { return { promise: async () => { - if (!(await checkFileExists(Key))) { + if (!storage.fileExists(Key)) { throw new Error('File does not exist'); } }, @@ -53,20 +36,16 @@ export class S3 { } getObject({ Key }: { Key: string }) { - const filePath = path.join(rootDir, Key); return { - promise: async () => await checkFileExists(filePath), + promise: async () => storage.fileExists(Key), createReadStream: () => { const emitter = new EventEmitter(); process.nextTick(() => { - if (fs.existsSync(filePath)) { - emitter.emit('data', Buffer.from(fs.readFileSync(filePath))); + if (storage.fileExists(Key)) { + emitter.emit('data', Buffer.from(storage.readFile(Key))); emitter.emit('end'); } else { - emitter.emit( - 'error', - new Error(`The file ${filePath} does not exist !`), - ); + emitter.emit('error', new Error(`The file ${Key} does not exist!`)); } }); return emitter; @@ -85,15 +64,18 @@ export class S3 { }; } - upload({ Key }: { Key: string }) { + upload({ Key, Body }: { Key: string; Body: ReadStream }) { return { promise: () => - new Promise(async (resolve, reject) => { - if (!(await checkFileExists(Key))) { - reject(`The file ${Key} does not exist`); - } else { - resolve(''); - } + new Promise(async resolve => { + const chunks = []; + Body.on('data', chunk => { + chunks.push(chunk); + }); + Body.once('end', () => { + storage.writeFile(Key, Buffer.concat(chunks)); + resolve(null); + }); }), }; } diff --git a/packages/techdocs-common/src/setupTests.ts b/packages/techdocs-common/src/setupTests.ts new file mode 100644 index 0000000000..996a8d35dd --- /dev/null +++ b/packages/techdocs-common/src/setupTests.ts @@ -0,0 +1,61 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import os from 'os'; +import path from 'path'; +import fs from 'fs-extra'; + +const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; + +const encoding = 'utf8'; + +export class StorageFilesMock { + private files: Record; + + constructor() { + this.files = {}; + } + + public emptyFiles() { + this.files = {}; + } + + public fileExists(targetPath: string): boolean { + const filePath = path.join(rootDir, targetPath); + const posixPath = filePath.split(path.posix.sep).join(path.sep); + return this.files[posixPath] !== undefined; + } + + public readFile(targetPath: string): Buffer { + const filePath = path.join(rootDir, targetPath); + return Buffer.from(this.files[filePath] ?? '', encoding); + } + + public writeFile(targetPath: string, sourcePath: string): void; + public writeFile(targetPath: string, sourceBuffer: Buffer): void; + public writeFile(targetPath: string, source: string | Buffer): void { + const filePath = path.join(rootDir, targetPath); + if (typeof source === 'string') { + this.files[filePath] = fs.readFileSync(source).toString(encoding); + } else { + this.files[filePath] = source.toString(encoding); + } + } +} + +const _global = global as any; +_global.rootDir = rootDir; +_global.StorageFilesMock = StorageFilesMock; diff --git a/packages/techdocs-common/src/stages/publish/awsS3.test.ts b/packages/techdocs-common/src/stages/publish/awsS3.test.ts index 715786e3f9..d77d643291 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.test.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.test.ts @@ -20,13 +20,13 @@ import { ConfigReader } from '@backstage/config'; import express from 'express'; import request from 'supertest'; import mockFs from 'mock-fs'; -import os from 'os'; import path from 'path'; +import fs from 'fs-extra'; import { AwsS3Publish } from './awsS3'; // NOTE: /packages/techdocs-common/__mocks__ is being used to mock aws-sdk client library -const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; +const rootDir = (global as any).rootDir; const getEntityRootDir = (entity: Entity) => { const { @@ -46,7 +46,7 @@ const createPublisherFromConfig = ({ bucketName?: string; legacyUseCaseSensitiveTripletPaths?: boolean; } = {}) => { - const mockConfig = new ConfigReader({ + const config = new ConfigReader({ techdocs: { requestUrl: 'http://localhost:7000', publisher: { @@ -63,7 +63,7 @@ const createPublisherFromConfig = ({ }, }); - return AwsS3Publish.fromConfig(mockConfig, logger); + return AwsS3Publish.fromConfig(config, logger); }; describe('AwsS3Publish', () => { @@ -72,7 +72,6 @@ describe('AwsS3Publish', () => { kind: 'Component', metadata: { name: 'backstage', - namespace: 'default', annotations: {}, }, @@ -90,18 +89,7 @@ describe('AwsS3Publish', () => { etag: 'etag', }; - const localRootDir = getEntityRootDir(entity); - const storageRootDir = getEntityRootDir({ - ...entity, - kind: entity.kind.toLowerCase(), - }); - const storageSingleQuoteDir = getEntityRootDir({ - metadata: { - namespace: 'storage', - name: 'quote', - }, - kind: 'single', - } as Entity); + const directory = getEntityRootDir(entity); const files = { 'index.html': '', @@ -110,9 +98,6 @@ describe('AwsS3Publish', () => { assets: { 'main.css': '', }, - attachments: { - 'image.png': Buffer.from([2, 3, 5, 3, 5, 3, 5, 9, 1]), - }, html: { 'unsafe.html': '', }, @@ -127,14 +112,7 @@ describe('AwsS3Publish', () => { beforeAll(() => { mockFs({ - [localRootDir]: files, - [storageRootDir]: files, - [storageSingleQuoteDir]: { - 'techdocs_metadata.json': files['techdocs_metadata.json'].replace( - /"/g, - "'", - ), - }, + [directory]: files, }); }); @@ -163,24 +141,14 @@ describe('AwsS3Publish', () => { describe('publish', () => { it('should publish a directory', async () => { const publisher = createPublisherFromConfig(); - expect( - await publisher.publish({ - entity, - directory: localRootDir, - }), - ).toBeUndefined(); + expect(await publisher.publish({ entity, directory })).toBeUndefined(); }); it('should publish a directory as well when legacy casing is used', async () => { const publisher = createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); - expect( - await publisher.publish({ - entity, - directory: localRootDir, - }), - ).toBeUndefined(); + expect(await publisher.publish({ entity, directory })).toBeUndefined(); }); it('should fail to publish a directory', async () => { @@ -214,6 +182,7 @@ describe('AwsS3Publish', () => { describe('hasDocsBeenGenerated', () => { it('should return true if docs has been generated', async () => { const publisher = createPublisherFromConfig(); + await publisher.publish({ entity, directory }); expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); }); @@ -221,6 +190,7 @@ describe('AwsS3Publish', () => { const publisher = createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); + await publisher.publish({ entity, directory }); expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); }); @@ -241,6 +211,7 @@ describe('AwsS3Publish', () => { describe('fetchTechDocsMetadata', () => { it('should return tech docs metadata', async () => { const publisher = createPublisherFromConfig(); + await publisher.publish({ entity, directory }); expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( techdocsMetadata, ); @@ -250,20 +221,32 @@ describe('AwsS3Publish', () => { const publisher = createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); + await publisher.publish({ entity, directory }); expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( techdocsMetadata, ); }); it('should return tech docs metadata when json encoded with single quotes', async () => { + const techdocsMetadataPath = path.join( + directory, + 'techdocs_metadata.json', + ); + const techdocsMetadataContent = files['techdocs_metadata.json']; + + fs.writeFileSync( + techdocsMetadataPath, + techdocsMetadataContent.replace(/"/g, "'"), + ); + const publisher = createPublisherFromConfig(); - expect( - await publisher.fetchTechDocsMetadata({ - namespace: 'storage', - kind: 'single', - name: 'quote', - }), - ).toStrictEqual(techdocsMetadata); + await publisher.publish({ entity, directory }); + + expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( + techdocsMetadata, + ); + + fs.writeFileSync(techdocsMetadataPath, techdocsMetadataContent); }); it('should return an error if the techdocs_metadata.json file is not present', async () => { @@ -276,7 +259,6 @@ describe('AwsS3Publish', () => { }; const techDocsMetadaFilePath = path.join( - rootDir, ...Object.values(invalidEntityName), 'techdocs_metadata.json', ); @@ -284,7 +266,7 @@ describe('AwsS3Publish', () => { const fails = publisher.fetchTechDocsMetadata(invalidEntityName); await expect(fails).rejects.toMatchObject({ - message: `TechDocs metadata fetch failed, The file ${techDocsMetadaFilePath} does not exist !`, + message: `TechDocs metadata fetch failed, The file ${techDocsMetadaFilePath} does not exist!`, }); }); }); @@ -294,8 +276,9 @@ describe('AwsS3Publish', () => { let app: express.Express; - beforeEach(() => { + beforeEach(async () => { const publisher = createPublisherFromConfig(); + await publisher.publish({ entity, directory }); app = express().use(publisher.docsRouter()); }); @@ -317,6 +300,7 @@ describe('AwsS3Publish', () => { const publisher = createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); + await publisher.publish({ entity, directory }); app = express().use(publisher.docsRouter()); // Ensures leading slash is trimmed and encoded path is decoded. const pngResponse = await request(app).get( diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts index 78cc7c0c79..dd658440d6 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts @@ -20,13 +20,13 @@ import { ConfigReader } from '@backstage/config'; import express from 'express'; import request from 'supertest'; import mockFs from 'mock-fs'; -import os from 'os'; import path from 'path'; +import fs from 'fs-extra'; import { AzureBlobStoragePublish } from './azureBlobStorage'; // NOTE: /packages/techdocs-common/__mocks__ is being used to mock Azure client library -const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; +const rootDir = (global as any).rootDir; const getEntityRootDir = (entity: Entity) => { const { @@ -38,7 +38,6 @@ const getEntityRootDir = (entity: Entity) => { }; const logger = getVoidLogger(); -jest.spyOn(logger, 'info').mockReturnValue(logger); jest.spyOn(logger, 'error').mockReturnValue(logger); const createPublisherFromConfig = ({ @@ -50,7 +49,7 @@ const createPublisherFromConfig = ({ containerName?: string; legacyUseCaseSensitiveTripletPaths?: boolean; } = {}) => { - const mockConfig = new ConfigReader({ + const config = new ConfigReader({ techdocs: { requestUrl: 'http://localhost:7000', publisher: { @@ -67,10 +66,10 @@ const createPublisherFromConfig = ({ }, }); - return AzureBlobStoragePublish.fromConfig(mockConfig, logger); + return AzureBlobStoragePublish.fromConfig(config, logger); }; -describe('publishing with valid credentials', () => { +describe('AzureBlobStoragePublish', () => { const entity = { apiVersion: '1', kind: 'Component', @@ -93,21 +92,9 @@ describe('publishing with valid credentials', () => { etag: 'etag', }; - const localRootDir = getEntityRootDir(entity); - const storageRootDir = getEntityRootDir({ - ...entity, - kind: entity.kind.toLowerCase(), - }); - const storageSingleQuoteDir = getEntityRootDir({ - metadata: { - namespace: 'storage', - name: 'quote', - }, - kind: 'single', - } as Entity); + const directory = getEntityRootDir(entity); beforeEach(() => { - (logger.info as jest.Mock).mockClear(); (logger.error as jest.Mock).mockClear(); }); @@ -118,9 +105,6 @@ describe('publishing with valid credentials', () => { assets: { 'main.css': '', }, - attachments: { - 'image.png': Buffer.from([2, 3, 5, 3, 5, 3, 5, 9, 1]), - }, html: { 'unsafe.html': '', }, @@ -135,14 +119,7 @@ describe('publishing with valid credentials', () => { beforeAll(async () => { mockFs({ - [localRootDir]: files, - [storageRootDir]: files, - [storageSingleQuoteDir]: { - 'techdocs_metadata.json': files['techdocs_metadata.json'].replace( - /"/g, - "'", - ), - }, + [directory]: files, }); }); @@ -178,24 +155,14 @@ describe('publishing with valid credentials', () => { describe('publish', () => { it('should publish a directory', async () => { const publisher = createPublisherFromConfig(); - expect( - await publisher.publish({ - entity, - directory: localRootDir, - }), - ).toBeUndefined(); + expect(await publisher.publish({ entity, directory })).toBeUndefined(); }); it('should publish a directory as well when legacy casing is used', async () => { const publisher = createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); - expect( - await publisher.publish({ - entity, - directory: localRootDir, - }), - ).toBeUndefined(); + expect(await publisher.publish({ entity, directory })).toBeUndefined(); }); it('should fail to publish a directory', async () => { @@ -229,15 +196,12 @@ describe('publishing with valid credentials', () => { it('reports an error when bad account credentials', async () => { const publisher = createPublisherFromConfig({ - accountName: 'failupload', + accountName: 'bad_account_credentials', }); let error; try { - await publisher.publish({ - entity, - directory: localRootDir, - }); + await publisher.publish({ entity, directory }); } catch (e) { error = e; } @@ -249,7 +213,7 @@ describe('publishing with valid credentials', () => { expect(logger.error).toHaveBeenCalledWith( expect.stringContaining( `Unable to upload file(s) to Azure Blob Storage. Error: Upload failed for ${path.join( - localRootDir, + directory, '404.html', )} with status code 500`, ), @@ -258,15 +222,17 @@ describe('publishing with valid credentials', () => { }); describe('hasDocsBeenGenerated', () => { - it('should return true if docs has been generated', async () => { + it('should check expected file', async () => { const publisher = createPublisherFromConfig(); + await publisher.publish({ entity, directory }); expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); }); - it('should return true if docs has been generated even if the legacy case is enabled', async () => { + it('should check expected file when legacy case flag is passed', async () => { const publisher = createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); + await publisher.publish({ entity, directory }); expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); }); @@ -287,6 +253,7 @@ describe('publishing with valid credentials', () => { describe('fetchTechDocsMetadata', () => { it('should return tech docs metadata', async () => { const publisher = createPublisherFromConfig(); + await publisher.publish({ entity, directory }); expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( techdocsMetadata, ); @@ -296,46 +263,53 @@ describe('publishing with valid credentials', () => { const publisher = createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); + await publisher.publish({ entity, directory }); expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( techdocsMetadata, ); }); it('should return tech docs metadata when json encoded with single quotes', async () => { + const techdocsMetadataPath = path.join( + directory, + 'techdocs_metadata.json', + ); + const techdocsMetadataContent = files['techdocs_metadata.json']; + + fs.writeFileSync( + techdocsMetadataPath, + techdocsMetadataContent.replace(/"/g, "'"), + ); + const publisher = createPublisherFromConfig(); - expect( - await publisher.fetchTechDocsMetadata({ - namespace: 'storage', - kind: 'single', - name: 'quote', - }), - ).toStrictEqual(techdocsMetadata); + await publisher.publish({ entity, directory }); + + expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( + techdocsMetadata, + ); + + fs.writeFileSync(techdocsMetadataPath, techdocsMetadataContent); }); it('should return an error if the techdocs_metadata.json file is not present', async () => { const publisher = createPublisherFromConfig(); + const invalidEntityName = { namespace: 'invalid', kind: 'triplet', name: 'path', }; - let error; - try { - await publisher.fetchTechDocsMetadata(invalidEntityName); - } catch (e) { - error = e; - } - - expect(error).toEqual( - new Error( - `TechDocs metadata fetch failed, The file ${path.join( - rootDir, - ...Object.values(invalidEntityName), - 'techdocs_metadata.json', - )} does not exist !`, - ), + const techDocsMetadaFilePath = path.join( + ...Object.values(invalidEntityName), + 'techdocs_metadata.json', ); + + const fails = publisher.fetchTechDocsMetadata(invalidEntityName); + + await expect(fails).rejects.toMatchObject({ + message: `TechDocs metadata fetch failed, The file ${techDocsMetadaFilePath} does not exist!`, + }); }); }); @@ -344,8 +318,9 @@ describe('publishing with valid credentials', () => { let app: express.Express; - beforeEach(() => { + beforeEach(async () => { const publisher = createPublisherFromConfig(); + await publisher.publish({ entity, directory }); app = express().use(publisher.docsRouter()); }); @@ -367,6 +342,7 @@ describe('publishing with valid credentials', () => { const publisher = createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); + await publisher.publish({ entity, directory }); app = express().use(publisher.docsRouter()); // Ensures leading slash is trimmed and encoded path is decoded. const pngResponse = await request(app).get( diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts index ce08e2f2ce..57fde767ba 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts @@ -20,13 +20,13 @@ import { ConfigReader } from '@backstage/config'; import express from 'express'; import request from 'supertest'; import mockFs from 'mock-fs'; -import os from 'os'; import path from 'path'; +import fs from 'fs-extra'; import { GoogleGCSPublish } from './googleStorage'; // NOTE: /packages/techdocs-common/__mocks__ is being used to mock Google Cloud Storage client library -const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; +const rootDir = (global as any).rootDir; const getEntityRootDir = (entity: Entity) => { const { @@ -38,7 +38,6 @@ const getEntityRootDir = (entity: Entity) => { }; const logger = getVoidLogger(); -jest.spyOn(logger, 'info').mockReturnValue(logger); const createPublisherFromConfig = ({ bucketName = 'bucketName', @@ -47,7 +46,7 @@ const createPublisherFromConfig = ({ bucketName?: string; legacyUseCaseSensitiveTripletPaths?: boolean; } = {}) => { - const mockConfig = new ConfigReader({ + const config = new ConfigReader({ techdocs: { requestUrl: 'http://localhost:7000', publisher: { @@ -61,7 +60,7 @@ const createPublisherFromConfig = ({ }, }); - return GoogleGCSPublish.fromConfig(mockConfig, logger); + return GoogleGCSPublish.fromConfig(config, logger); }; describe('GoogleGCSPublish', () => { @@ -87,18 +86,7 @@ describe('GoogleGCSPublish', () => { etag: 'etag', }; - const localRootDir = getEntityRootDir(entity); - const storageRootDir = getEntityRootDir({ - ...entity, - kind: entity.kind.toLowerCase(), - }); - const storageSingleQuoteDir = getEntityRootDir({ - metadata: { - namespace: 'storage', - name: 'quote', - }, - kind: 'single', - } as Entity); + const directory = getEntityRootDir(entity); const files = { 'index.html': '', @@ -121,14 +109,7 @@ describe('GoogleGCSPublish', () => { beforeAll(() => { mockFs({ - [localRootDir]: files, - [storageRootDir]: files, - [storageSingleQuoteDir]: { - 'techdocs_metadata.json': files['techdocs_metadata.json'].replace( - /"/g, - "'", - ), - }, + [directory]: files, }); }); @@ -146,7 +127,7 @@ describe('GoogleGCSPublish', () => { it('should reject incorrect config', async () => { const publisher = createPublisherFromConfig({ - bucketName: 'errorBucket', + bucketName: 'bad_bucket_name', }); expect(await publisher.getReadiness()).toEqual({ isAvailable: false, @@ -157,24 +138,14 @@ describe('GoogleGCSPublish', () => { describe('publish', () => { it('should publish a directory', async () => { const publisher = createPublisherFromConfig(); - expect( - await publisher.publish({ - entity, - directory: localRootDir, - }), - ).toBeUndefined(); + expect(await publisher.publish({ entity, directory })).toBeUndefined(); }); it('should publish a directory as well when legacy casing is used', async () => { const publisher = createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); - expect( - await publisher.publish({ - entity, - directory: localRootDir, - }), - ).toBeUndefined(); + expect(await publisher.publish({ entity, directory })).toBeUndefined(); }); it('should fail to publish a directory', async () => { @@ -210,6 +181,7 @@ describe('GoogleGCSPublish', () => { describe('hasDocsBeenGenerated', () => { it('should return true if docs has been generated', async () => { const publisher = createPublisherFromConfig(); + await publisher.publish({ entity, directory }); expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); }); @@ -217,6 +189,7 @@ describe('GoogleGCSPublish', () => { const publisher = createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); + await publisher.publish({ entity, directory }); expect(await publisher.hasDocsBeenGenerated(entity)).toBe(true); }); @@ -237,6 +210,7 @@ describe('GoogleGCSPublish', () => { describe('fetchTechDocsMetadata', () => { it('should return tech docs metadata', async () => { const publisher = createPublisherFromConfig(); + await publisher.publish({ entity, directory }); expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( techdocsMetadata, ); @@ -246,20 +220,32 @@ describe('GoogleGCSPublish', () => { const publisher = createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); + await publisher.publish({ entity, directory }); expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( techdocsMetadata, ); }); it('should return tech docs metadata when json encoded with single quotes', async () => { + const techdocsMetadataPath = path.join( + directory, + 'techdocs_metadata.json', + ); + const techdocsMetadataContent = files['techdocs_metadata.json']; + + fs.writeFileSync( + techdocsMetadataPath, + techdocsMetadataContent.replace(/"/g, "'"), + ); + const publisher = createPublisherFromConfig(); - expect( - await publisher.fetchTechDocsMetadata({ - namespace: 'storage', - kind: 'single', - name: 'quote', - }), - ).toStrictEqual(techdocsMetadata); + await publisher.publish({ entity, directory }); + + expect(await publisher.fetchTechDocsMetadata(entityName)).toStrictEqual( + techdocsMetadata, + ); + + fs.writeFileSync(techdocsMetadataPath, techdocsMetadataContent); }); it('should return an error if the techdocs_metadata.json file is not present', async () => { @@ -272,7 +258,6 @@ describe('GoogleGCSPublish', () => { }; const techDocsMetadaFilePath = path.join( - rootDir, ...Object.values(invalidEntityName), 'techdocs_metadata.json', ); @@ -280,7 +265,7 @@ describe('GoogleGCSPublish', () => { const fails = publisher.fetchTechDocsMetadata(invalidEntityName); await expect(fails).rejects.toMatchObject({ - message: `The file ${techDocsMetadaFilePath} does not exist !`, + message: `The file ${techDocsMetadaFilePath} does not exist!`, }); }); }); @@ -288,10 +273,11 @@ describe('GoogleGCSPublish', () => { describe('docsRouter', () => { const entityTripletPath = `${entity.metadata.namespace}/${entity.kind}/${entity.metadata.name}`; - let app: express.Express; + let app: Express.Application; - beforeEach(() => { + beforeEach(async () => { const publisher = createPublisherFromConfig(); + await publisher.publish({ entity, directory }); app = express().use(publisher.docsRouter()); }); @@ -313,7 +299,9 @@ describe('GoogleGCSPublish', () => { const publisher = createPublisherFromConfig({ legacyUseCaseSensitiveTripletPaths: true, }); + await publisher.publish({ entity, directory }); app = express().use(publisher.docsRouter()); + // Ensures leading slash is trimmed and encoded path is decoded. const pngResponse = await request(app).get( `/${entityTripletPath}/img/with%20spaces.png`, From 9181e8c6e029af12619eeeb97b6451f87730ef8d Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Mon, 9 Aug 2021 09:27:34 +0200 Subject: [PATCH 014/343] tests(techdocs-common): add globals and types for mocked files Signed-off-by: Camila Belo --- .../__mocks__/@azure/storage-blob.ts | 4 ++-- .../__mocks__/@google-cloud/storage.ts | 2 +- packages/techdocs-common/__mocks__/aws-sdk.ts | 2 +- .../techdocs-common/__mocks__/globals.d.ts | 22 +++++++++++++++++ packages/techdocs-common/__mocks__/types.d.ts | 24 +++++++++++++++++++ packages/techdocs-common/src/globals.d.ts | 22 +++++++++++++++++ packages/techdocs-common/src/setupTests.ts | 11 ++++----- .../src/stages/publish/awsS3.test.ts | 2 +- .../stages/publish/azureBlobStorage.test.ts | 2 +- .../src/stages/publish/googleStorage.test.ts | 2 +- packages/techdocs-common/src/types.d.ts | 24 +++++++++++++++++++ 11 files changed, 104 insertions(+), 13 deletions(-) create mode 100644 packages/techdocs-common/__mocks__/globals.d.ts create mode 100644 packages/techdocs-common/__mocks__/types.d.ts create mode 100644 packages/techdocs-common/src/globals.d.ts create mode 100644 packages/techdocs-common/src/types.d.ts diff --git a/packages/techdocs-common/__mocks__/@azure/storage-blob.ts b/packages/techdocs-common/__mocks__/@azure/storage-blob.ts index 80c60acce7..b901f5f5a7 100644 --- a/packages/techdocs-common/__mocks__/@azure/storage-blob.ts +++ b/packages/techdocs-common/__mocks__/@azure/storage-blob.ts @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import type { +import { BlobUploadCommonResponse, ContainerGetPropertiesResponse, } from '@azure/storage-blob'; import { EventEmitter } from 'events'; -const storage = new (global as any).StorageFilesMock(); +const storage = global.storageFilesMock; export class BlockBlobClient { private readonly blobName; diff --git a/packages/techdocs-common/__mocks__/@google-cloud/storage.ts b/packages/techdocs-common/__mocks__/@google-cloud/storage.ts index efc2454d08..856f09c4df 100644 --- a/packages/techdocs-common/__mocks__/@google-cloud/storage.ts +++ b/packages/techdocs-common/__mocks__/@google-cloud/storage.ts @@ -15,7 +15,7 @@ */ import { Readable } from 'stream'; -const storage = new (global as any).StorageFilesMock(); +const storage = global.storageFilesMock; class GCSFile { private readonly path: string; diff --git a/packages/techdocs-common/__mocks__/aws-sdk.ts b/packages/techdocs-common/__mocks__/aws-sdk.ts index 36c864bfe7..6251fe8357 100644 --- a/packages/techdocs-common/__mocks__/aws-sdk.ts +++ b/packages/techdocs-common/__mocks__/aws-sdk.ts @@ -18,7 +18,7 @@ import { ReadStream } from 'fs'; export { Credentials } from 'aws-sdk'; -const storage = new (global as any).StorageFilesMock(); +const storage = global.storageFilesMock; export class S3 { constructor() { diff --git a/packages/techdocs-common/__mocks__/globals.d.ts b/packages/techdocs-common/__mocks__/globals.d.ts new file mode 100644 index 0000000000..ad293d2ebe --- /dev/null +++ b/packages/techdocs-common/__mocks__/globals.d.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +declare module NodeJS { + interface Global { + rootDir: string; + storageFilesMock: IStorageFilesMock; + } +} diff --git a/packages/techdocs-common/__mocks__/types.d.ts b/packages/techdocs-common/__mocks__/types.d.ts new file mode 100644 index 0000000000..e085b95851 --- /dev/null +++ b/packages/techdocs-common/__mocks__/types.d.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +interface IStorageFilesMock { + emptyFiles(): void; + fileExists(targetPath: string): boolean; + readFile(targetPath: string): Buffer; + writeFile(targetPath: string, sourcePath: string): void; + writeFile(targetPath: string, sourceBuffer: Buffer): void; + writeFile(targetPath: string, source: string | Buffer): void; +} diff --git a/packages/techdocs-common/src/globals.d.ts b/packages/techdocs-common/src/globals.d.ts new file mode 100644 index 0000000000..ad293d2ebe --- /dev/null +++ b/packages/techdocs-common/src/globals.d.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +declare module NodeJS { + interface Global { + rootDir: string; + storageFilesMock: IStorageFilesMock; + } +} diff --git a/packages/techdocs-common/src/setupTests.ts b/packages/techdocs-common/src/setupTests.ts index 996a8d35dd..9a13e57633 100644 --- a/packages/techdocs-common/src/setupTests.ts +++ b/packages/techdocs-common/src/setupTests.ts @@ -18,18 +18,18 @@ import os from 'os'; import path from 'path'; import fs from 'fs-extra'; -const rootDir = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; +const rootDir: string = os.platform() === 'win32' ? 'C:\\rootDir' : '/rootDir'; const encoding = 'utf8'; -export class StorageFilesMock { +class StorageFilesMock implements IStorageFilesMock { private files: Record; constructor() { this.files = {}; } - public emptyFiles() { + public emptyFiles(): void { this.files = {}; } @@ -56,6 +56,5 @@ export class StorageFilesMock { } } -const _global = global as any; -_global.rootDir = rootDir; -_global.StorageFilesMock = StorageFilesMock; +global.rootDir = rootDir; +global.storageFilesMock = new StorageFilesMock(); diff --git a/packages/techdocs-common/src/stages/publish/awsS3.test.ts b/packages/techdocs-common/src/stages/publish/awsS3.test.ts index d77d643291..62ec80d634 100644 --- a/packages/techdocs-common/src/stages/publish/awsS3.test.ts +++ b/packages/techdocs-common/src/stages/publish/awsS3.test.ts @@ -26,7 +26,7 @@ import { AwsS3Publish } from './awsS3'; // NOTE: /packages/techdocs-common/__mocks__ is being used to mock aws-sdk client library -const rootDir = (global as any).rootDir; +const rootDir = global.rootDir; const getEntityRootDir = (entity: Entity) => { const { diff --git a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts index dd658440d6..c32ec310d0 100644 --- a/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/azureBlobStorage.test.ts @@ -26,7 +26,7 @@ import { AzureBlobStoragePublish } from './azureBlobStorage'; // NOTE: /packages/techdocs-common/__mocks__ is being used to mock Azure client library -const rootDir = (global as any).rootDir; +const rootDir = global.rootDir; const getEntityRootDir = (entity: Entity) => { const { diff --git a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts index 57fde767ba..fe93ef909b 100644 --- a/packages/techdocs-common/src/stages/publish/googleStorage.test.ts +++ b/packages/techdocs-common/src/stages/publish/googleStorage.test.ts @@ -26,7 +26,7 @@ import { GoogleGCSPublish } from './googleStorage'; // NOTE: /packages/techdocs-common/__mocks__ is being used to mock Google Cloud Storage client library -const rootDir = (global as any).rootDir; +const rootDir = global.rootDir; const getEntityRootDir = (entity: Entity) => { const { diff --git a/packages/techdocs-common/src/types.d.ts b/packages/techdocs-common/src/types.d.ts new file mode 100644 index 0000000000..e085b95851 --- /dev/null +++ b/packages/techdocs-common/src/types.d.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +interface IStorageFilesMock { + emptyFiles(): void; + fileExists(targetPath: string): boolean; + readFile(targetPath: string): Buffer; + writeFile(targetPath: string, sourcePath: string): void; + writeFile(targetPath: string, sourceBuffer: Buffer): void; + writeFile(targetPath: string, source: string | Buffer): void; +} From c772d9a84eab20ebb6553b15a07fb9af4f7f3bff Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Mon, 9 Aug 2021 10:23:18 +0200 Subject: [PATCH 015/343] docs(techdocs-common): Add changeset file for next release Signed-off-by: Camila Belo --- .changeset/techdocs-all-done.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/techdocs-all-done.md diff --git a/.changeset/techdocs-all-done.md b/.changeset/techdocs-all-done.md new file mode 100644 index 0000000000..2696b40df7 --- /dev/null +++ b/.changeset/techdocs-all-done.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-techdocs': patch +--- + +This is the third and final step of migrating cloud storage entities to lowercase ([see](https://github.com/backstage/backstage/issues/4367#issuecomment-876461002)). + +Updates GCS, AWS S3 and Azure Blob Storage Provider to lowercase entity triplet paths before reading and writing to it cloud storage. From 2f4f15846d6fe91a2a1dcbaaad0907929d074e36 Mon Sep 17 00:00:00 2001 From: Camila Belo Date: Mon, 9 Aug 2021 11:38:33 +0200 Subject: [PATCH 016/343] docs(techdocs-common): add case sentitive migration steps Co-authored-by: Eric Peterson Signed-off-by: Camila Belo --- .changeset/techdocs-all-done.md | 18 +++++-- docs/features/techdocs/configuration.md | 8 +++ docs/features/techdocs/how-to-guides.md | 70 +++++++++++++++++++++++++ plugins/techdocs-backend/config.d.ts | 7 ++- 4 files changed, 99 insertions(+), 4 deletions(-) diff --git a/.changeset/techdocs-all-done.md b/.changeset/techdocs-all-done.md index 2696b40df7..6d2146e2a5 100644 --- a/.changeset/techdocs-all-done.md +++ b/.changeset/techdocs-all-done.md @@ -1,7 +1,19 @@ --- -'@backstage/plugin-techdocs': patch +'@backstage/plugin-techdocs': minor +'@backstage/plugin-techdocs-backend': minor +'@backstage/techdocs-common': minor --- +TechDocs sites can now be accessed using paths containing entity triplets of +any case (e.g. `/docs/namespace/KIND/name` or `/docs/namespace/kind/name`). + +If you do not use an external storage provider for serving TechDocs, this is a +transparent change and no action is required from you. + +If you _do_ use an external storage provider for serving TechDocs (e.g. Google +Cloud Storage, AWS S3, or Azure Blob Storage), you must run a migration command +against your storage provider before updating to this version. + +[A migration guide is available here](#todo). + This is the third and final step of migrating cloud storage entities to lowercase ([see](https://github.com/backstage/backstage/issues/4367#issuecomment-876461002)). - -Updates GCS, AWS S3 and Azure Blob Storage Provider to lowercase entity triplet paths before reading and writing to it cloud storage. diff --git a/docs/features/techdocs/configuration.md b/docs/features/techdocs/configuration.md index 4bb6857106..46cbc2a8a2 100644 --- a/docs/features/techdocs/configuration.md +++ b/docs/features/techdocs/configuration.md @@ -113,6 +113,14 @@ techdocs: # https://docs.microsoft.com/en-us/azure/storage/common/storage-auth?toc=/azure/storage/blobs/toc.json accountKey: ${TECHDOCS_AZURE_BLOB_STORAGE_ACCOUNT_KEY} + # (Optional and not recommended) Prior to version [0.x.y] of TechDocs, docs + # sites could only be accessed over paths with case-sensitive entity triplets + # e.g. (namespace/Kind/name). If you are upgrading from an older version of + # TechDocs and are unable to perform the necessary migration of files in your + # external storage, you can set this value to `true` to temporarily revert to + # the old, case-sensitive entity triplet behavior. + legacyUseCaseSensitiveTripletPaths: false + # (Optional and Legacy) TechDocs makes API calls to techdocs-backend using this URL. e.g. get docs of an entity, get metadata, etc. # You don't have to specify this anymore. diff --git a/docs/features/techdocs/how-to-guides.md b/docs/features/techdocs/how-to-guides.md index e1533498d7..4962b9b107 100644 --- a/docs/features/techdocs/how-to-guides.md +++ b/docs/features/techdocs/how-to-guides.md @@ -184,3 +184,73 @@ annotation ) as an 'entities' attribute. For a reference to the React structure of the default home page, please refer to `plugins/techdocs/src/home/components/TechDocsCustomHome.tsx`. + +## How to migrate from TechDocs Alpha to Beta + +> This guide only applies to the "recommended" TechDocs deployment method (where +> an external storage provider and external CI/CD is used). If you use the +> "basic" or "out-of-the-box" setup, you can stop here! No action needed. + +The beta version of TechDocs (v0.x.y) made a breaking change to the way TechDocs +content was accessed and stored, allowing pages to be accessed with +case-insensitive entity triplet paths (e.g. `/docs/namespace/kind/name` whereas +in prior versions, they could only be accessed at `/docs/namespace/Kind/name`). +In order to enable this change, documentation has be stored in an external +storage provider using an object key whose entity triplet is lower-cased. + +New installations of TechDocs since the beta version will work fine with no +action, but for those who were running TechDocs prior to this version, a +migration will need to be performed so that all existing content in your storage +bucket matches this lower-case entity triplet expectation. + +1. **Ensure you have the right permissions on your storage provider**: In order + to migrate files in your storage provider, the `techdocs-cli` needs to be + able to read/copy/rename/move/delete files. The exact instructions vary by + storage provider, but check the [using cloud storage][using-cloud-storage] + page for details. + +2. **Run a non-destructive migration of files**: Ensure you have the latest + version of `techdocs-cli` installed. Then run the following command, using + the details relevant for your provider / configuration. This will copy all + files from, e.g. `namespace/Kind/name/index.html` to + `namespace/kind/name/index.html`, without removing the original files. + +```sh +techdocs-cli migrate --publisher-type --storage-name --verbose +``` + +3. **Deploy the updated versions of the TechDocs plugins**: Once the migration + above has been run, you can deploy the beta versions of the TechDocs backend + and frontend plugins to your Backstage instance. + +4. **Verify that your TechDocs sites are still loading/accessible**: Try + accessing a TechDocs site using different entity-triplet case variants, e.g. + `/docs/namespace/KIND/name` or `/docs/namespace/kind/name`. Your TechDocs + site should load regardless of the URL path casing you use. + +5. **Clean up the old objects from storage**: Once you've verified that your + TechDocs site is accessible, you can clean up your storage bucket by + re-running the `migrate` command on the TechDocs CLI, but with an additional + `removeOriginal` flag passed: + +```sh +techdocs-cli migrate --publisher-type --storage-name --removeOriginal --verbose +``` + +6. **Update your CI/CD pipelines to use the beta version of the TechDocs CLI**: + Finally, you can update all of your CI/CD pipelines to use at least v0.x.y of + the TechDocs CLI, ensuring that all sites are published to the new, + lower-cased entity triplet paths going forward. + +If you encounter problems running this migration, please [report the +issue][beta-migrate-bug]. You can temporarily revert to pre-beta storage +expectations with a configuration change: + +```yaml +techdocs: + legacyUseCaseSensitiveTripletPaths: true +``` + +[beta-migrate-bug]: +https://github.com/backstage/backstage/issues/new?assignees=&labels=bug&template=bug_template.md&title=[TechDocs]%20Unable%20to%20run%20beta%20migration +[using-cloud-storage]: ./using-cloud-storage.md diff --git a/plugins/techdocs-backend/config.d.ts b/plugins/techdocs-backend/config.d.ts index 537daa815f..2f1efa0037 100644 --- a/plugins/techdocs-backend/config.d.ts +++ b/plugins/techdocs-backend/config.d.ts @@ -248,7 +248,12 @@ export interface Config { storageUrl?: string; /** - * (Optional) + * (Optional and not recommended) Prior to version [0.x.y] of TechDocs, docs + * sites could only be accessed over paths with case-sensitive entity triplets + * e.g. (namespace/Kind/name). If you are upgrading from an older version of + * TechDocs and are unable to perform the necessary migration of files in your + * external storage, you can set this value to `true` to temporarily revert to + * the old, case-sensitive entity triplet behavior. */ legacyUseCaseSensitiveTripletPaths?: boolean; }; From 3067f0a0c2dcd208f150be59710c20e4aa0840bd Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Mon, 9 Aug 2021 18:37:31 +0200 Subject: [PATCH 017/343] Note OpenStack Swift caveat. Signed-off-by: Eric Peterson --- .changeset/techdocs-all-done.md | 11 ++++++----- docs/features/techdocs/README.md | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.changeset/techdocs-all-done.md b/.changeset/techdocs-all-done.md index 6d2146e2a5..c4c1b5ed83 100644 --- a/.changeset/techdocs-all-done.md +++ b/.changeset/techdocs-all-done.md @@ -10,10 +10,11 @@ any case (e.g. `/docs/namespace/KIND/name` or `/docs/namespace/kind/name`). If you do not use an external storage provider for serving TechDocs, this is a transparent change and no action is required from you. -If you _do_ use an external storage provider for serving TechDocs (e.g. Google -Cloud Storage, AWS S3, or Azure Blob Storage), you must run a migration command -against your storage provider before updating to this version. +If you _do_ use an external storage provider for serving TechDocs (one of\* GCS, +AWS S3, or Azure Blob Storage), you must run a migration command against your +storage provider before updating. -[A migration guide is available here](#todo). +[A migration guide is available here](https://backstage.io/docs/features/techdocs/how-to-guides#how-to-migrate-from-techdocs-alpha-to-beta). -This is the third and final step of migrating cloud storage entities to lowercase ([see](https://github.com/backstage/backstage/issues/4367#issuecomment-876461002)). +- (\*) We're seeking help from the community to bring OpenStack Swift support + [to feature parity](https://github.com/backstage/backstage/issues/6763) with the above. diff --git a/docs/features/techdocs/README.md b/docs/features/techdocs/README.md index 6732b3ab18..b845691ac8 100644 --- a/docs/features/techdocs/README.md +++ b/docs/features/techdocs/README.md @@ -54,7 +54,7 @@ providers are used. | Google Cloud Storage (GCS) | Yes ✅ | | Amazon Web Services (AWS) S3 | Yes ✅ | | Azure Blob Storage | Yes ✅ | -| OpenStack Swift | Yes ✅ | +| OpenStack Swift | Community ✅ | [Reach out to us](#feedback) if you want to request more platforms. From add5ee5d97ac949929d4790665bdeef5ddf40bb3 Mon Sep 17 00:00:00 2001 From: Eric Peterson Date: Tue, 10 Aug 2021 12:30:34 +0200 Subject: [PATCH 018/343] Use lower-case links within techdocs plugin; allow legacy behavior with config. Signed-off-by: Eric Peterson --- plugins/techdocs/config.d.ts | 7 ++ .../src/home/components/DocsCardGrid.test.tsx | 65 +++++++++++++++- .../src/home/components/DocsCardGrid.tsx | 16 +++- .../src/home/components/DocsTable.test.tsx | 76 ++++++++++++++++++- .../src/home/components/DocsTable.tsx | 13 +++- 5 files changed, 168 insertions(+), 9 deletions(-) diff --git a/plugins/techdocs/config.d.ts b/plugins/techdocs/config.d.ts index ebc2aa2fd7..d6bf21b10b 100644 --- a/plugins/techdocs/config.d.ts +++ b/plugins/techdocs/config.d.ts @@ -26,6 +26,13 @@ export interface Config { */ builder: 'local' | 'external'; + /** + * Allows fallback to case-sensitive triplets in case of migration issues. + * @visibility frontend + * @see https://backstage.io/docs/features/techdocs/how-to-guides#how-to-migrate-from-techdocs-alpha-to-beta + */ + legacyUseCaseSensitiveTripletPaths?: boolean; + /** * @example http://localhost:7000/api/techdocs * @visibility frontend diff --git a/plugins/techdocs/src/home/components/DocsCardGrid.test.tsx b/plugins/techdocs/src/home/components/DocsCardGrid.test.tsx index f9831e2fa4..4adada48fa 100644 --- a/plugins/techdocs/src/home/components/DocsCardGrid.test.tsx +++ b/plugins/techdocs/src/home/components/DocsCardGrid.test.tsx @@ -17,11 +17,35 @@ import { wrapInTestApp } from '@backstage/test-utils'; import { render } from '@testing-library/react'; import React from 'react'; +import { configApiRef } from '@backstage/core-plugin-api'; import { DocsCardGrid } from './DocsCardGrid'; +// Hacky way to mock a specific boolean config value. +const getOptionalBooleanMock = jest.fn().mockReturnValue(false); +jest.mock('@backstage/core-plugin-api', () => ({ + ...jest.requireActual('@backstage/core-plugin-api'), + useApi: (apiRef: any) => { + const actualUseApi = jest.requireActual( + '@backstage/core-plugin-api', + ).useApi; + const actualApi = actualUseApi(apiRef); + if (apiRef === configApiRef) { + const configReader = actualApi; + configReader.getOptionalBoolean = getOptionalBooleanMock; + return configReader; + } + + return actualApi; + }, +})); + describe('Entity Docs Card Grid', () => { + beforeEach(() => { + jest.resetAllMocks(); + }); + it('should render all entities passed ot it', async () => { - const { findByText } = render( + const { findByText, findAllByRole } = render( wrapInTestApp( { ); expect(await findByText('testName')).toBeInTheDocument(); expect(await findByText('testName2')).toBeInTheDocument(); + const [button1, button2] = await findAllByRole('button'); + expect(button1.getAttribute('href')).toContain( + '/default/testkind/testname', + ); + expect(button2.getAttribute('href')).toContain( + '/default/testkind2/testname2', + ); + }); + + it('should fall back to case-sensitive links when configured', async () => { + getOptionalBooleanMock.mockReturnValue(true); + + const { findByRole } = render( + wrapInTestApp( + , + ), + ); + + const button = await findByRole('button'); + expect(getOptionalBooleanMock).toHaveBeenCalledWith( + 'techdocs.legacyUseCaseSensitiveTripletPaths', + ); + expect(button.getAttribute('href')).toContain( + '/SomeNamespace/TestKind/testName', + ); }); }); diff --git a/plugins/techdocs/src/home/components/DocsCardGrid.tsx b/plugins/techdocs/src/home/components/DocsCardGrid.tsx index 500debaf2d..8b7e114b09 100644 --- a/plugins/techdocs/src/home/components/DocsCardGrid.tsx +++ b/plugins/techdocs/src/home/components/DocsCardGrid.tsx @@ -18,6 +18,7 @@ import React from 'react'; import { generatePath } from 'react-router-dom'; import { Entity } from '@backstage/catalog-model'; +import { useApi, configApiRef } from '@backstage/core-plugin-api'; import { Card, CardActions, CardContent, CardMedia } from '@material-ui/core'; import { rootDocsRouteRef } from '../../routes'; @@ -32,6 +33,13 @@ export const DocsCardGrid = ({ }: { entities: Entity[] | undefined; }) => { + // Lower-case entity triplets by default, but allow override. + const toLowerMaybe = useApi(configApiRef).getOptionalBoolean( + 'techdocs.legacyUseCaseSensitiveTripletPaths', + ) + ? (str: string) => str + : (str: string) => str.toLocaleLowerCase(); + if (!entities) return null; return ( @@ -46,9 +54,11 @@ export const DocsCardGrid = ({ + + + ); +}; diff --git a/plugins/home/src/components/index.ts b/plugins/home/src/components/index.ts new file mode 100644 index 0000000000..aae0cbe26d --- /dev/null +++ b/plugins/home/src/components/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2020 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { HomepageCompositionRoot } from './HomepageCompositionRoot'; +export { SettingsModal } from './SettingsModal'; diff --git a/plugins/home/src/extensions.tsx b/plugins/home/src/extensions.tsx new file mode 100644 index 0000000000..93fe4ceb4f --- /dev/null +++ b/plugins/home/src/extensions.tsx @@ -0,0 +1,123 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React, { Suspense } from 'react'; +import { IconButton } from '@material-ui/core'; +import SettingsIcon from '@material-ui/icons/Settings'; +import { InfoCard } from '@backstage/core-components'; +import { SettingsModal } from './components'; +import { createReactExtension, useApp } from '@backstage/core-plugin-api'; + +export type ComponentRenderer = { + Renderer?: (props: RendererProps) => JSX.Element; +}; + +type ComponentParts = { + Content: () => JSX.Element; + Actions?: () => JSX.Element; + Settings?: () => JSX.Element; + ContextProvider?: (props: any) => JSX.Element; +}; + +type RendererProps = { title: string } & ComponentParts; + +export function createCardExtension({ + title, + components, +}: { + title: string; + components: () => Promise; +}) { + return createReactExtension({ + component: { + lazy: () => + components().then(({ Content, Actions, Settings, ContextProvider }) => { + const CardExtension = ({ + Renderer, + title: overrideTitle, + ...childProps + }: ComponentRenderer & { title?: string } & T) => { + const app = useApp(); + const { Progress } = app.getComponents(); + const [settingsOpen, setSettingsOpen] = React.useState(false); + + if (Renderer) { + return ( + }> + + + ); + } + + const cardProps = { + title: overrideTitle ?? title, + ...(Settings + ? { + action: ( + setSettingsOpen(true)}> + Settings + + ), + } + : {}), + ...(Actions + ? { + actions: , + } + : {}), + }; + + const innerContent = ( + + {Settings && ( + setSettingsOpen(false)} + > + + + )} + + + ); + + return ( + }> + {ContextProvider ? ( + + {innerContent} + + ) : ( + innerContent + )} + + ); + }; + return CardExtension; + }), + }, + }); +} diff --git a/plugins/home/src/homePageComponents/RandomJoke/Actions.tsx b/plugins/home/src/homePageComponents/RandomJoke/Actions.tsx new file mode 100644 index 0000000000..cebb0400ab --- /dev/null +++ b/plugins/home/src/homePageComponents/RandomJoke/Actions.tsx @@ -0,0 +1,29 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; + +import { Button } from '@material-ui/core'; +import { useRandomJoke } from './Context'; + +export const Actions = () => { + const { rerollJoke } = useRandomJoke(); + return ( + + ); +}; diff --git a/plugins/home/src/homePageComponents/RandomJoke/Content.tsx b/plugins/home/src/homePageComponents/RandomJoke/Content.tsx new file mode 100644 index 0000000000..231e32356a --- /dev/null +++ b/plugins/home/src/homePageComponents/RandomJoke/Content.tsx @@ -0,0 +1,31 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React from 'react'; +import { useRandomJoke } from './Context'; + +export const Content = () => { + const { joke, loading } = useRandomJoke(); + + if (loading) return

Loading...

; + + return ( +
+

{joke.setup}

+

{joke.punchline}

+
+ ); +}; diff --git a/plugins/home/src/homePageComponents/RandomJoke/Context.tsx b/plugins/home/src/homePageComponents/RandomJoke/Context.tsx new file mode 100644 index 0000000000..5c2790d8a5 --- /dev/null +++ b/plugins/home/src/homePageComponents/RandomJoke/Context.tsx @@ -0,0 +1,99 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import React, { createContext } from 'react'; + +export type JokeType = 'any' | 'programming'; + +type Joke = { + setup: string; + punchline: string; +}; + +type RandomJokeContextValue = { + loading: boolean; + joke: Joke; + type: JokeType; + rerollJoke: Function; + handleChangeType: Function; +}; + +const Context = createContext(undefined); + +const getNewJoke = (type: string): Promise => + fetch( + `https://official-joke-api.appspot.com/jokes${ + type !== 'any' ? `/${type}` : '' + }/random`, + ) + .then(res => res.json()) + .then(data => (Array.isArray(data) ? data[0] : data)); + +export const ContextProvider = ({ + children, + defaultCategory, +}: { + children: JSX.Element; + defaultCategory?: JokeType; +}) => { + const [loading, setLoading] = React.useState(true); + const [joke, setJoke] = React.useState({ + setup: '', + punchline: '', + }); + const [type, setType] = React.useState( + defaultCategory || ('programming' as JokeType), + ); + + const rerollJoke = React.useCallback(() => { + setLoading(true); + getNewJoke(type).then(newJoke => setJoke(newJoke)); + }, [type]); + + const handleChangeType = (newType: JokeType) => { + setType(newType); + }; + + React.useEffect(() => { + setLoading(false); + }, [joke]); + + React.useEffect(() => { + rerollJoke(); + }, [rerollJoke]); + + const value: RandomJokeContextValue = { + loading, + joke, + type, + rerollJoke, + handleChangeType, + }; + + return {children}; +}; + +export const useRandomJoke = () => { + const value = React.useContext(Context); + + if (value === undefined) { + throw new Error('useRandomJoke must be used within a RandomJokeProvider'); + } + + return value; +}; + +export default Context; diff --git a/plugins/home/src/homePageComponents/RandomJoke/Settings.tsx b/plugins/home/src/homePageComponents/RandomJoke/Settings.tsx new file mode 100644 index 0000000000..ee5de69dc6 --- /dev/null +++ b/plugins/home/src/homePageComponents/RandomJoke/Settings.tsx @@ -0,0 +1,48 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + FormControl, + FormLabel, + RadioGroup, + FormControlLabel, + Radio, +} from '@material-ui/core'; +import React from 'react'; +import { useRandomJoke, JokeType } from './Context'; + +export const Settings = () => { + const { type, handleChangeType } = useRandomJoke(); + const JOKE_TYPES: JokeType[] = ['any' as JokeType, 'programming' as JokeType]; + return ( + + Joke Type + handleChangeType(e.target.value)} + > + {JOKE_TYPES.map(t => ( + } + label={`${t.slice(0, 1).toUpperCase()}${t.slice(1)}`} + /> + ))} + + + ); +}; diff --git a/plugins/home/src/homePageComponents/RandomJoke/index.ts b/plugins/home/src/homePageComponents/RandomJoke/index.ts new file mode 100644 index 0000000000..aafbc17a2e --- /dev/null +++ b/plugins/home/src/homePageComponents/RandomJoke/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { Actions } from './Actions'; +export { Content } from './Content'; +export { Settings } from './Settings'; +export { ContextProvider } from './Context'; diff --git a/plugins/home/src/index.ts b/plugins/home/src/index.ts new file mode 100644 index 0000000000..812e1b2495 --- /dev/null +++ b/plugins/home/src/index.ts @@ -0,0 +1,26 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { + homePlugin, + HomepageCompositionRoot, + RandomJokeHomePageComponent, + ComponentAccordion, + ComponentTabs, + ComponentTab, +} from './plugin'; +export { SettingsModal } from './components'; +export { createCardExtension } from './extensions'; diff --git a/plugins/home/src/plugin.test.ts b/plugins/home/src/plugin.test.ts new file mode 100644 index 0000000000..920a9076f4 --- /dev/null +++ b/plugins/home/src/plugin.test.ts @@ -0,0 +1,22 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { homePlugin } from './plugin'; + +describe('home', () => { + it('should export plugin', () => { + expect(homePlugin).toBeDefined(); + }); +}); diff --git a/plugins/home/src/plugin.ts b/plugins/home/src/plugin.ts new file mode 100644 index 0000000000..5600d2d315 --- /dev/null +++ b/plugins/home/src/plugin.ts @@ -0,0 +1,68 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { + createComponentExtension, + createPlugin, + createRoutableExtension, +} from '@backstage/core-plugin-api'; +import { createCardExtension } from './extensions'; + +import { rootRouteRef } from './routes'; + +export const homePlugin = createPlugin({ + id: 'home', + routes: { + root: rootRouteRef, + }, +}); + +export const HomepageCompositionRoot = homePlugin.provide( + createRoutableExtension({ + component: () => + import('./components').then(m => m.HomepageCompositionRoot), + mountPoint: rootRouteRef, + }), +); + +export const ComponentAccordion = homePlugin.provide( + createComponentExtension({ + component: { + lazy: () => + import('./componentRenderers').then(m => m.ComponentAccordion), + }, + }), +); +export const ComponentTabs = homePlugin.provide( + createComponentExtension({ + component: { + lazy: () => import('./componentRenderers').then(m => m.ComponentTabs), + }, + }), +); +export const ComponentTab = homePlugin.provide( + createComponentExtension({ + component: { + lazy: () => import('./componentRenderers').then(m => m.ComponentTab), + }, + }), +); + +export const RandomJokeHomePageComponent = homePlugin.provide( + createCardExtension<{ defaultCategory?: 'any' | 'programming' }>({ + title: 'Random Joke', + components: () => import('./homePageComponents/RandomJoke'), + }), +); diff --git a/plugins/home/src/routes.ts b/plugins/home/src/routes.ts new file mode 100644 index 0000000000..2c641d8433 --- /dev/null +++ b/plugins/home/src/routes.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { createRouteRef } from '@backstage/core-plugin-api'; + +export const rootRouteRef = createRouteRef({ + title: 'home', +}); diff --git a/plugins/home/src/setupTests.ts b/plugins/home/src/setupTests.ts new file mode 100644 index 0000000000..fc6dbd98f8 --- /dev/null +++ b/plugins/home/src/setupTests.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import '@testing-library/jest-dom'; +import 'cross-fetch/polyfill'; From ba01a6bcdfc918d53c2494482bb44a70f6cb4aa2 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Tue, 17 Aug 2021 14:59:07 +0200 Subject: [PATCH 121/343] Parse date correctly, drop math.Floor Signed-off-by: Johan Haals --- .../DefaultProcessingDatabase.test.ts | 19 ++++++++++++++++--- plugins/catalog-backend/src/next/refresh.ts | 2 +- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts index 09647b980c..df9c08ad6c 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts @@ -66,6 +66,21 @@ describe('Default Processing Database', () => { await db('refresh_state').insert(ref); }; + const parseDate = (date: string | Date): DateTime => { + const parsedDate = + typeof date === 'string' + ? DateTime.fromSQL(date, { zone: 'UTC' }) + : DateTime.fromJSDate(date); + + if (!parsedDate.isValid) { + throw new Error( + `Failed to parse date, reason: ${parsedDate.invalidReason}, explanation: ${parsedDate.invalidExplanation}`, + ); + } + + return parsedDate; + }; + describe('addUprocessedEntities', () => { function mockEntity(name: string, type: string): Entity { return { @@ -995,9 +1010,7 @@ describe('Default Processing Database', () => { const result = await knex('refresh_state') .where('entity_ref', 'location:default/new-root') .select(); - const nextUpdate = DateTime.fromSQL(result[0].next_update_at, { - zone: 'utc', - }); + const nextUpdate = parseDate(result[0].next_update_at); expect(nextUpdate.diff(now, 'seconds').seconds).toBeGreaterThanOrEqual( 100, ); diff --git a/plugins/catalog-backend/src/next/refresh.ts b/plugins/catalog-backend/src/next/refresh.ts index caec7e9548..aae992f391 100644 --- a/plugins/catalog-backend/src/next/refresh.ts +++ b/plugins/catalog-backend/src/next/refresh.ts @@ -29,6 +29,6 @@ export function createRandomRefreshInterval(options: { }): RefreshIntervalFunction { const { minSeconds, maxSeconds } = options; return () => { - return Math.floor(Math.random() * (maxSeconds - minSeconds) + minSeconds); + return Math.random() * (maxSeconds - minSeconds) + minSeconds; }; } From 1136ac88b604946f522a6e08333f5e271580384b Mon Sep 17 00:00:00 2001 From: Roy Jacobs Date: Tue, 17 Aug 2021 15:03:51 +0200 Subject: [PATCH 122/343] First part of GitLabDiscoveryProcessor (no target parsing yet) Signed-off-by: Roy Jacobs --- .../GitLabDiscoveryProcessor.test.ts | 252 ++++++++++++++++++ .../processors/GitLabDiscoveryProcessor.ts | 129 +++++++++ .../src/ingestion/processors/gitlab/client.ts | 94 +++++++ .../src/ingestion/processors/gitlab/index.ts | 18 ++ .../src/ingestion/processors/gitlab/types.ts | 23 ++ .../src/ingestion/processors/index.ts | 1 + .../src/next/NextCatalogBuilder.ts | 2 + .../src/service/CatalogBuilder.ts | 2 + 8 files changed, 521 insertions(+) create mode 100644 plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts create mode 100644 plugins/catalog-backend/src/ingestion/processors/gitlab/types.ts diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts new file mode 100644 index 0000000000..5eb5c51869 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts @@ -0,0 +1,252 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConfigReader } from '@backstage/config'; +import { getVoidLogger } from '@backstage/backend-common'; +import { LocationSpec } from '@backstage/catalog-model'; +import { + GitLabDiscoveryProcessor, +} from './GitLabDiscoveryProcessor'; +import { setupServer } from 'msw/node'; +import { rest } from 'msw'; +import { GitLabProject } from "./gitlab"; + +const server = setupServer(); + +function setupFakeGitLab( + callback: (request: { + page: number; + }) => { data: GitLabProject[]; nextPage?: number }, +) { + server.use( + rest.get('https://gitlab.fake/api/v4/projects', (req, res, ctx) => { + if (req.headers.get('private-token') !== 'test-token') { + return res(ctx.status(401), ctx.json({})); + } + const page = req.url.searchParams.get('page'); + const response = callback({ + page: parseInt(page!, 10), + }); + + // Filter the fake results based on the `last_activity_after` parameter + const last_activity_after = req.url.searchParams.get( + 'last_activity_after', + ); + const filteredData = response.data.filter( + v => + !last_activity_after || + Date.parse(v.last_activity_at) >= Date.parse(last_activity_after), + ); + + return res( + ctx.set('x-next-page', response.nextPage?.toString() ?? ''), + ctx.json(filteredData), + ); + }), + ); +} + +function getConfig(): any { + return { + backend: { + cache: { store: 'memory' }, + }, + integrations: { + gitlab: [ + { + host: 'gitlab.fake', + apiBaseUrl: 'https://gitlab.fake/api/v4', + token: 'test-token', + }, + ], + }, + }; +} + +function getProcessor(config?: any): GitLabDiscoveryProcessor { + return GitLabDiscoveryProcessor.fromConfig( + new ConfigReader(config || getConfig()), + { + logger: getVoidLogger(), + }, + ); +} + +describe('GitlabDiscoveryProcessor', () => { + beforeAll(() => { + server.listen(); + jest.useFakeTimers('modern'); + jest.setSystemTime(new Date('2001-01-01T12:34:56Z')); + }); + afterEach(() => server.resetHandlers()); + afterAll(() => { + server.close(); + jest.useRealTimers(); + }); + + const location: LocationSpec = { + type: 'gitlab-discovery', + target: 'https://gitlab.fake/north-star', + }; + + describe('handles repositories', () => { + it('pages through all repositories', async () => { + const processor = getProcessor(); + setupFakeGitLab(request => { + switch (request.page) { + case 1: + return { + data: [ + { + id: 1, + archived: false, + default_branch: 'main', + last_activity_at: '2021-08-05T11:03:05.774Z', + web_url: 'https://gitlab.fake/1', + }, + ], + nextPage: 2, + }; + case 2: + return { + data: [ + { + id: 2, + archived: false, + default_branch: 'master', + last_activity_at: '2021-08-05T11:03:05.774Z', + web_url: 'https://gitlab.fake/2', + }, + { + id: 3, + archived: true, // ARCHIVED + default_branch: 'master', + last_activity_at: '2021-08-05T11:03:05.774Z', + web_url: 'https://gitlab.fake/3', + }, + ], + }; + default: + throw new Error('Invalid request'); + } + }); + + const result: any[] = []; + await processor.readLocation(location, false, e => { + result.push(e); + }); + expect(result).toEqual([ + { + type: 'location', + location: { + type: 'url', + target: 'https://gitlab.fake/1/-/blob/main/catalog-info.yaml', + }, + optional: true, + }, + { + type: 'location', + location: { + type: 'url', + target: 'https://gitlab.fake/2/-/blob/master/catalog-info.yaml', + }, + optional: true, + }, + ]); + }); + + it('uses the previous scan timestamp to filter', async () => { + const processor = getProcessor(); + setupFakeGitLab(request => { + switch (request.page) { + case 1: + return { + data: [ + { + id: 1, + archived: false, + default_branch: 'main', + last_activity_at: '2000-01-01T00:00:00Z', + web_url: 'https://gitlab.fake/1', + }, + { + id: 2, + archived: false, + default_branch: 'main', + last_activity_at: '2002-01-01T00:00:00Z', + web_url: 'https://gitlab.fake/2', + }, + ], + }; + default: + throw new Error('Invalid request'); + } + }); + + const result: any[] = []; + + // First scan should find all repos, since no last activity was cached + await processor.readLocation(location, false, e => { + result.push(e); + }); + expect(result).toHaveLength(2); + + // Second scan should have used the mocked Date to set the last scanned time to 2001 + // This should result in only the second repo being scanned, since that has a timestamp of 2002 + const result2: any[] = []; + await processor.readLocation(location, false, e => { + result2.push(e); + }); + expect(result2).toHaveLength(1); + }); + }); + + describe('handles failure', () => { + it('invalid token', async () => { + // Setup an empty fake gitlab, since we don't care about actual results + setupFakeGitLab(_ => { + return { + data: [], + }; + }); + + const config = getConfig(); + config.integrations.gitlab[0].token = 'invalid'; + await expect( + getProcessor(config).readLocation(location, false, _ => {}), + ).rejects.toThrow(/Unauthorized/); + }); + + it('missing integration', async () => { + const config = getConfig(); + delete config.integrations; + await expect( + getProcessor(config).readLocation(location, false, _ => {}), + ).rejects.toThrow(/no GitLab integration/); + }); + + it('location type', async () => { + const incorrectLocation: LocationSpec = { + type: 'something-that-is-not-gitlab-discovery', + target: 'https://gitlab.fake/north-star', + }; + + await expect( + getProcessor().readLocation(incorrectLocation, false, _ => {}), + ).resolves.toBeFalsy(); + }); + }); +}); diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts new file mode 100644 index 0000000000..b3cce6e711 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts @@ -0,0 +1,129 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { LocationSpec } from '@backstage/catalog-model'; +import { Config } from '@backstage/config'; +import { + ScmIntegrations, +} from '@backstage/integration'; +import { Logger } from 'winston'; +import * as results from './results'; +import { CatalogProcessor, CatalogProcessorEmit } from './types'; +import { GitLabClient, GitLabProject, paginated } from "./gitlab"; +import { CacheClient, CacheManager, PluginCacheManager } from "@backstage/backend-common"; + +/** + * Extracts repositories out of a GitLab org. + */ +export class GitLabDiscoveryProcessor implements CatalogProcessor { + private readonly integrations: ScmIntegrations; + private readonly logger: Logger; + private readonly cache: CacheClient; + + static fromConfig(config: Config, options: { logger: Logger }) { + const integrations = ScmIntegrations.fromConfig(config); + const pluginCache = CacheManager.fromConfig(config).forPlugin( + 'gitlab-discovery', + ); + + return new GitLabDiscoveryProcessor({ + ...options, + integrations, + pluginCache, + }); + } + + private constructor(options: { + integrations: ScmIntegrations; + pluginCache: PluginCacheManager; + logger: Logger; + }) { + this.integrations = options.integrations; + this.cache = options.pluginCache.getClient(); + this.logger = options.logger; + } + + async readLocation( + location: LocationSpec, + _optional: boolean, + emit: CatalogProcessorEmit, + ): Promise { + if (location.type !== 'gitlab-discovery') { + return false; + } + + const integration = this.integrations.gitlab.byUrl(location.target); + if (!integration) { + throw new Error( + `There is no GitLab integration that matches ${location.target}. Please add a configuration entry for it under integrations.gitlab`, + ); + } + + const client = new GitLabClient({ + config: integration.config, + logger: this.logger, + }); + const startTimestamp = Date.now(); + this.logger.info(`Reading GitLab projects from ${location.target}`); + + const projects = paginated(options => client.listProjects(options), { + last_activity_after: await this.updateLastActivity(), + page: 1, + }); + + const result: Result = { + scanned: 0, + matches: [], + }; + for await (const project of projects) { + result.scanned++; + if (!project.archived) { + result.matches.push(project); + } + } + + for (const project of result.matches) { + emit( + results.location( + { + type: 'url', + // The format expected by the GitLabUrlReader: https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath + target: `${project.web_url}/-/blob/${project.default_branch}/catalog-info.yaml`, + }, + true, + ), + ); + } + + const duration = ((Date.now() - startTimestamp) / 1000).toFixed(1); + this.logger.debug( + `Read ${result.scanned} GitLab repositories in ${duration} seconds`, + ); + + return true; + } + + async updateLastActivity(): Promise { + const lastActivity = await this.cache.get('last-activity'); + await this.cache.set('last-activity', new Date().toISOString()); + return lastActivity as string | undefined; + } +} + +type Result = { + scanned: number; + matches: GitLabProject[]; +}; diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts new file mode 100644 index 0000000000..fb7763dd1c --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts @@ -0,0 +1,94 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fetch from 'cross-fetch'; +import { + getGitLabRequestOptions, + GitLabIntegrationConfig, +} from '@backstage/integration'; +import { Logger } from 'winston'; + +export class GitLabClient { + private readonly config: GitLabIntegrationConfig; + private readonly logger: Logger; + + constructor(options: { config: GitLabIntegrationConfig; logger: Logger }) { + this.config = options.config; + this.logger = options.logger; + } + + async listProjects(options?: ListOptions): Promise> { + return this.pagedRequest(`${this.config.apiBaseUrl}/projects`, options); + } + + private async pagedRequest( + endpoint: string, + options?: ListOptions, + ): Promise> { + const request = new URL(endpoint); + for (const key in options) { + if (options[key]) { + request.searchParams.append(key, options[key]!.toString()); + } + } + + this.logger.debug(`Fetching: ${request.toString()}`); + const response = await fetch( + request.toString(), + getGitLabRequestOptions(this.config), + ); + if (!response.ok) { + throw new Error( + `Unexpected response when fetching ${request.toString()}. Expected 200 but got ${ + response.status + } - ${response.statusText}`, + ); + } + return response.json().then(items => { + const nextPage = response.headers.get('x-next-page'); + + return { + items, + nextPage: nextPage ? Number(nextPage) : null, + } as PagedResponse; + }); + } +} + +export type ListOptions = { + [key: string]: string | number | undefined; + per_page?: number | undefined; + page?: number | undefined; +}; + +export type PagedResponse = { + items: T[]; + nextPage?: number; +}; + +export async function* paginated( + request: (options: ListOptions) => Promise>, + options: ListOptions, +) { + let res; + do { + res = await request(options); + options.page = res.nextPage; + for (const item of res.items) { + yield item; + } + } while (res.nextPage); +} diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts b/plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts new file mode 100644 index 0000000000..a6c6419d21 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { GitLabClient, paginated } from './client'; +export type { GitLabProject } from "./types"; diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/types.ts b/plugins/catalog-backend/src/ingestion/processors/gitlab/types.ts new file mode 100644 index 0000000000..a66411ddc8 --- /dev/null +++ b/plugins/catalog-backend/src/ingestion/processors/gitlab/types.ts @@ -0,0 +1,23 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export type GitLabProject = { + id: number; + default_branch: string; + archived: boolean; + last_activity_at: string; + web_url: string; +}; diff --git a/plugins/catalog-backend/src/ingestion/processors/index.ts b/plugins/catalog-backend/src/ingestion/processors/index.ts index 6a44ea0a8f..fe4d374829 100644 --- a/plugins/catalog-backend/src/ingestion/processors/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/index.ts @@ -26,6 +26,7 @@ export { FileReaderProcessor } from './FileReaderProcessor'; export { GithubDiscoveryProcessor } from './GithubDiscoveryProcessor'; export { GithubOrgReaderProcessor } from './GithubOrgReaderProcessor'; export { GithubMultiOrgReaderProcessor } from './GithubMultiOrgReaderProcessor'; +export { GitLabDiscoveryProcessor } from './GitLabDiscoveryProcessor'; export { LocationEntityProcessor } from './LocationEntityProcessor'; export { PlaceholderProcessor } from './PlaceholderProcessor'; export type { PlaceholderResolver } from './PlaceholderProcessor'; diff --git a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts index fb12077508..88fcc27055 100644 --- a/plugins/catalog-backend/src/next/NextCatalogBuilder.ts +++ b/plugins/catalog-backend/src/next/NextCatalogBuilder.ts @@ -50,6 +50,7 @@ import { FileReaderProcessor, GithubDiscoveryProcessor, GithubOrgReaderProcessor, + GitLabDiscoveryProcessor, PlaceholderProcessor, PlaceholderResolver, UrlReaderProcessor, @@ -372,6 +373,7 @@ export class NextCatalogBuilder { BitbucketDiscoveryProcessor.fromConfig(config, { logger }), GithubDiscoveryProcessor.fromConfig(config, { logger }), GithubOrgReaderProcessor.fromConfig(config, { logger }), + GitLabDiscoveryProcessor.fromConfig(config, { logger }), new UrlReaderProcessor({ reader, logger }), CodeOwnersProcessor.fromConfig(config, { logger, reader }), new AnnotateLocationEntityProcessor({ integrations }), diff --git a/plugins/catalog-backend/src/service/CatalogBuilder.ts b/plugins/catalog-backend/src/service/CatalogBuilder.ts index 76b4af7450..70c1890fbf 100644 --- a/plugins/catalog-backend/src/service/CatalogBuilder.ts +++ b/plugins/catalog-backend/src/service/CatalogBuilder.ts @@ -46,6 +46,7 @@ import { FileReaderProcessor, GithubDiscoveryProcessor, GithubOrgReaderProcessor, + GitLabDiscoveryProcessor, HigherOrderOperation, HigherOrderOperations, LocationEntityProcessor, @@ -316,6 +317,7 @@ export class CatalogBuilder { BitbucketDiscoveryProcessor.fromConfig(config, { logger }), GithubDiscoveryProcessor.fromConfig(config, { logger }), GithubOrgReaderProcessor.fromConfig(config, { logger }), + GitLabDiscoveryProcessor.fromConfig(config, { logger }), new UrlReaderProcessor({ reader, logger }), CodeOwnersProcessor.fromConfig(config, { logger, reader }), new LocationEntityProcessor({ integrations }), From a487f61564c3ccdc6938f8990fb1313b367dd827 Mon Sep 17 00:00:00 2001 From: Roy Jacobs Date: Tue, 17 Aug 2021 16:24:07 +0200 Subject: [PATCH 123/343] Parse URLs and allow filtering based on group/subgroup Signed-off-by: Roy Jacobs --- .../GitLabDiscoveryProcessor.test.ts | 160 +++++++++++++++--- .../processors/GitLabDiscoveryProcessor.ts | 71 ++++++-- .../src/ingestion/processors/gitlab/client.ts | 15 +- 3 files changed, 210 insertions(+), 36 deletions(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts index 5eb5c51869..a0e44d7648 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.test.ts @@ -17,28 +17,47 @@ import { ConfigReader } from '@backstage/config'; import { getVoidLogger } from '@backstage/backend-common'; import { LocationSpec } from '@backstage/catalog-model'; -import { - GitLabDiscoveryProcessor, -} from './GitLabDiscoveryProcessor'; +import { GitLabDiscoveryProcessor, parseUrl } from './GitLabDiscoveryProcessor'; import { setupServer } from 'msw/node'; import { rest } from 'msw'; -import { GitLabProject } from "./gitlab"; +import { GitLabProject } from './gitlab'; const server = setupServer(); -function setupFakeGitLab( - callback: (request: { - page: number; - }) => { data: GitLabProject[]; nextPage?: number }, +const PROJECTS_URL = 'https://gitlab.fake/api/v4/projects'; +const GROUP_PROJECTS_URL = + 'https://gitlab.fake/api/v4/groups/group%2Fsubgroup/projects'; + +const PROJECT_LOCATION: LocationSpec = { + type: 'gitlab-discovery', + target: 'https://gitlab.fake/blob/*/catalog-info.yaml', +}; +const PROJECT_LOCATION_MASTER_BRANCH: LocationSpec = { + type: 'gitlab-discovery', + target: 'https://gitlab.fake/blob/master/catalog-info.yaml', +}; +const GROUP_LOCATION: LocationSpec = { + type: 'gitlab-discovery', + target: 'https://gitlab.fake/group/subgroup/blob/*/catalog-info.yaml', +}; + +function setupFakeServer( + url: string, + callback: (request: { page: number; include_subgroups: boolean }) => { + data: GitLabProject[]; + nextPage?: number; + }, ) { server.use( - rest.get('https://gitlab.fake/api/v4/projects', (req, res, ctx) => { + rest.get(url, (req, res, ctx) => { if (req.headers.get('private-token') !== 'test-token') { return res(ctx.status(401), ctx.json({})); } const page = req.url.searchParams.get('page'); + const include_subgroups = req.url.searchParams.get('include_subgroups'); const response = callback({ page: parseInt(page!, 10), + include_subgroups: include_subgroups === 'true', }); // Filter the fake results based on the `last_activity_after` parameter @@ -97,15 +116,43 @@ describe('GitlabDiscoveryProcessor', () => { jest.useRealTimers(); }); - const location: LocationSpec = { - type: 'gitlab-discovery', - target: 'https://gitlab.fake/north-star', - }; + describe('parseUrl', () => { + it('parses well formed URLs', () => { + expect( + parseUrl('https://gitlab.com/group/subgroup/blob/master/catalog.yaml'), + ).toEqual({ + group: 'group/subgroup', + host: 'gitlab.com', + branch: 'master', + catalogPath: 'catalog.yaml', + }); + expect( + parseUrl('https://gitlab.com/blob/*/subfolder/catalog.yaml'), + ).toEqual({ + group: undefined, + host: 'gitlab.com', + branch: '*', + catalogPath: 'subfolder/catalog.yaml', + }); + }); + + it('throws on incorrectly formed URLs', () => { + expect(() => parseUrl('https://gitlab.com')).toThrow(); + expect(() => parseUrl('https://gitlab.com//')).toThrow(); + expect(() => parseUrl('https://gitlab.com/foo')).toThrow(); + expect(() => parseUrl('https://gitlab.com//foo')).toThrow(); + expect(() => parseUrl('https://gitlab.com/org/teams')).toThrow(); + expect(() => parseUrl('https://gitlab.com/org//teams')).toThrow(); + expect(() => + parseUrl('https://gitlab.com/org//teams/blob/catalog.yaml'), + ).toThrow(); + }); + }); describe('handles repositories', () => { it('pages through all repositories', async () => { const processor = getProcessor(); - setupFakeGitLab(request => { + setupFakeServer(PROJECTS_URL, request => { switch (request.page) { case 1: return { @@ -145,7 +192,7 @@ describe('GitlabDiscoveryProcessor', () => { }); const result: any[] = []; - await processor.readLocation(location, false, e => { + await processor.readLocation(PROJECT_LOCATION, false, e => { result.push(e); }); expect(result).toEqual([ @@ -168,9 +215,78 @@ describe('GitlabDiscoveryProcessor', () => { ]); }); + it('can force a branch name', async () => { + const processor = getProcessor(); + setupFakeServer(PROJECTS_URL, request => { + switch (request.page) { + case 1: + return { + data: [ + { + id: 1, + archived: false, + default_branch: 'main', + last_activity_at: '2021-08-05T11:03:05.774Z', + web_url: 'https://gitlab.fake/1', + }, + ], + }; + default: + throw new Error('Invalid request'); + } + }); + + const result: any[] = []; + await processor.readLocation(PROJECT_LOCATION_MASTER_BRANCH, false, e => { + result.push(e); + }); + expect(result).toEqual([ + { + type: 'location', + location: { + type: 'url', + target: 'https://gitlab.fake/1/-/blob/master/catalog-info.yaml', + }, + optional: true, + }, + ]); + }); + + it('can filter based on group', async () => { + const processor = getProcessor(); + setupFakeServer(GROUP_PROJECTS_URL, request => { + if (!request.include_subgroups) { + throw new Error('include_subgroups should be set'); + } + switch (request.page) { + case 1: + return { + data: [ + { + id: 1, + archived: false, + default_branch: 'main', + last_activity_at: '2021-08-05T11:03:05.774Z', + web_url: 'https://gitlab.fake/1', + }, + ], + }; + default: + throw new Error('Invalid request'); + } + }); + + const result: any[] = []; + await processor.readLocation(GROUP_LOCATION, false, e => { + result.push(e); + }); + // If everything was set up correctly, we should have received the fake repo specified above + expect(result).toHaveLength(1); + }); + it('uses the previous scan timestamp to filter', async () => { const processor = getProcessor(); - setupFakeGitLab(request => { + setupFakeServer(PROJECTS_URL, request => { switch (request.page) { case 1: return { @@ -199,7 +315,7 @@ describe('GitlabDiscoveryProcessor', () => { const result: any[] = []; // First scan should find all repos, since no last activity was cached - await processor.readLocation(location, false, e => { + await processor.readLocation(PROJECT_LOCATION, false, e => { result.push(e); }); expect(result).toHaveLength(2); @@ -207,7 +323,7 @@ describe('GitlabDiscoveryProcessor', () => { // Second scan should have used the mocked Date to set the last scanned time to 2001 // This should result in only the second repo being scanned, since that has a timestamp of 2002 const result2: any[] = []; - await processor.readLocation(location, false, e => { + await processor.readLocation(PROJECT_LOCATION, false, e => { result2.push(e); }); expect(result2).toHaveLength(1); @@ -217,7 +333,7 @@ describe('GitlabDiscoveryProcessor', () => { describe('handles failure', () => { it('invalid token', async () => { // Setup an empty fake gitlab, since we don't care about actual results - setupFakeGitLab(_ => { + setupFakeServer(PROJECTS_URL, _ => { return { data: [], }; @@ -226,7 +342,7 @@ describe('GitlabDiscoveryProcessor', () => { const config = getConfig(); config.integrations.gitlab[0].token = 'invalid'; await expect( - getProcessor(config).readLocation(location, false, _ => {}), + getProcessor(config).readLocation(PROJECT_LOCATION, false, _ => {}), ).rejects.toThrow(/Unauthorized/); }); @@ -234,14 +350,14 @@ describe('GitlabDiscoveryProcessor', () => { const config = getConfig(); delete config.integrations; await expect( - getProcessor(config).readLocation(location, false, _ => {}), + getProcessor(config).readLocation(PROJECT_LOCATION, false, _ => {}), ).rejects.toThrow(/no GitLab integration/); }); it('location type', async () => { const incorrectLocation: LocationSpec = { type: 'something-that-is-not-gitlab-discovery', - target: 'https://gitlab.fake/north-star', + target: 'https://gitlab.fake/oh-dear', }; await expect( diff --git a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts index b3cce6e711..5603a1aa7f 100644 --- a/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts +++ b/plugins/catalog-backend/src/ingestion/processors/GitLabDiscoveryProcessor.ts @@ -16,17 +16,19 @@ import { LocationSpec } from '@backstage/catalog-model'; import { Config } from '@backstage/config'; -import { - ScmIntegrations, -} from '@backstage/integration'; +import { ScmIntegrations } from '@backstage/integration'; import { Logger } from 'winston'; import * as results from './results'; import { CatalogProcessor, CatalogProcessorEmit } from './types'; -import { GitLabClient, GitLabProject, paginated } from "./gitlab"; -import { CacheClient, CacheManager, PluginCacheManager } from "@backstage/backend-common"; +import { GitLabClient, GitLabProject, paginated } from './gitlab'; +import { + CacheClient, + CacheManager, + PluginCacheManager, +} from '@backstage/backend-common'; /** - * Extracts repositories out of a GitLab org. + * Extracts repositories out of an GitLab instance. */ export class GitLabDiscoveryProcessor implements CatalogProcessor { private readonly integrations: ScmIntegrations; @@ -35,9 +37,8 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { static fromConfig(config: Config, options: { logger: Logger }) { const integrations = ScmIntegrations.fromConfig(config); - const pluginCache = CacheManager.fromConfig(config).forPlugin( - 'gitlab-discovery', - ); + const pluginCache = + CacheManager.fromConfig(config).forPlugin('gitlab-discovery'); return new GitLabDiscoveryProcessor({ ...options, @@ -65,10 +66,12 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { return false; } - const integration = this.integrations.gitlab.byUrl(location.target); + const { group, host, branch, catalogPath } = parseUrl(location.target); + + const integration = this.integrations.gitlab.byUrl(`https://${host}`); if (!integration) { throw new Error( - `There is no GitLab integration that matches ${location.target}. Please add a configuration entry for it under integrations.gitlab`, + `There is no GitLab integration that matches ${host}. Please add a configuration entry for it under integrations.gitlab`, ); } @@ -80,6 +83,7 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { this.logger.info(`Reading GitLab projects from ${location.target}`); const projects = paginated(options => client.listProjects(options), { + group, last_activity_after: await this.updateLastActivity(), page: 1, }); @@ -96,12 +100,19 @@ export class GitLabDiscoveryProcessor implements CatalogProcessor { } for (const project of result.matches) { + const project_branch = branch === '*' ? project.default_branch : branch; + emit( results.location( { type: 'url', - // The format expected by the GitLabUrlReader: https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath - target: `${project.web_url}/-/blob/${project.default_branch}/catalog-info.yaml`, + // The format expected by the GitLabUrlReader: + // https://gitlab.com/groupA/teams/teamA/subgroupA/repoA/-/blob/branch/filepath + // + // This unfortunately will trigger another API call in `getGitLabFileFetchUrl` to get the project ID. + // The alternative is using the `buildRawUrl` function, which does not support subgroups, so providing a raw + // URL here won't work either. + target: `${project.web_url}/-/blob/${project_branch}/${catalogPath}`, }, true, ), @@ -127,3 +138,37 @@ type Result = { scanned: number; matches: GitLabProject[]; }; + +/* + * Helpers + */ + +export function parseUrl(urlString: string): { + group?: string; + host: string; + branch: string; + catalogPath: string; +} { + const url = new URL(urlString); + const path = url.pathname.substr(1).split('/'); + + // (/group/subgroup)/blob/branch|*/filepath + const blobIndex = path.findIndex(p => p === 'blob'); + if (blobIndex !== -1 && path.length > blobIndex + 2) { + const group = + blobIndex > 0 ? path.slice(0, blobIndex).join('/') : undefined; + + return { + group, + host: url.host, + branch: decodeURIComponent(path[blobIndex + 1]), + catalogPath: decodeURIComponent(path.slice(blobIndex + 2).join('/')), + }; + } + + throw new Error(`Failed to parse ${urlString}`); +} + +export function escapeRegExp(str: string): RegExp { + return new RegExp(`^${str.replace(/\*/g, '.*')}$`); +} diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts index fb7763dd1c..8781e071f7 100644 --- a/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts +++ b/plugins/catalog-backend/src/ingestion/processors/gitlab/client.ts @@ -31,6 +31,18 @@ export class GitLabClient { } async listProjects(options?: ListOptions): Promise> { + if (options?.group) { + return this.pagedRequest( + `${this.config.apiBaseUrl}/groups/${encodeURIComponent( + options?.group, + )}/projects`, + { + ...options, + include_subgroups: true, + }, + ); + } + return this.pagedRequest(`${this.config.apiBaseUrl}/projects`, options); } @@ -69,7 +81,8 @@ export class GitLabClient { } export type ListOptions = { - [key: string]: string | number | undefined; + [key: string]: string | number | boolean | undefined; + group?: string; per_page?: number | undefined; page?: number | undefined; }; From 29f6c7541a4f10dd3844924c7ed97a586be05359 Mon Sep 17 00:00:00 2001 From: Roy Jacobs Date: Tue, 17 Aug 2021 16:28:36 +0200 Subject: [PATCH 124/343] Add documentation for GitLab discovery Signed-off-by: Roy Jacobs --- docs/integrations/gitlab/discovery.md | 36 +++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 docs/integrations/gitlab/discovery.md diff --git a/docs/integrations/gitlab/discovery.md b/docs/integrations/gitlab/discovery.md new file mode 100644 index 0000000000..ade575b74d --- /dev/null +++ b/docs/integrations/gitlab/discovery.md @@ -0,0 +1,36 @@ +--- +id: discovery +title: GitLab Discovery +sidebar_label: Discovery +# prettier-ignore +description: Automatically discovering catalog entities from repositories in GitLab +--- + +The GitLab integration has a special discovery processor for discovering catalog +entities from GitLab. The processor will crawl the GitLab instance and register +entities matching the configured path. This can be useful as an alternative to +static locations or manually adding things to the catalog. + +To use the discovery processor, you'll need a GitLab integration +[set up](locations.md) with a `token`. Then you can add a location target to the +catalog configuration: + +```yaml +catalog: + locations: + - type: gitlab-discovery + target: https://gitlab.com/group/subgroup/blob/main/catalog-info.yaml +``` + +Note the `gitlab-discovery` type, as this is not a regular `url` processor. + +The target is composed of three parts: + +- The base URL, `https://gitlab.com` in this case +- The group path, `group/subgroup` in this case. This is optional: If you omit + this path the processor will scan the entire GitLab instance instead. +- The path within each repository to find the catalog YAML file. This will + usually be `/blob/main/catalog-info.yaml`, `/blob/master/catalog-info.yaml` or + a similar variation for catalog files stored in the root directory of each + repository. If you want to use the repository's default branch use the `*` + wildcard, e.g.: `/blob/*/catalog-info.yaml` From 96785dce38ba99b317677decb24585a9e5c22bef Mon Sep 17 00:00:00 2001 From: Roy Jacobs Date: Tue, 17 Aug 2021 16:31:57 +0200 Subject: [PATCH 125/343] Add changeset for GitLab discovery Signed-off-by: Roy Jacobs --- .changeset/lucky-gifts-help.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/lucky-gifts-help.md diff --git a/.changeset/lucky-gifts-help.md b/.changeset/lucky-gifts-help.md new file mode 100644 index 0000000000..1bc100d824 --- /dev/null +++ b/.changeset/lucky-gifts-help.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': minor +--- + +Added GitLabDiscoveryProcessor, which allows catalog discovery from a GitLab instance From 017f2f6c7e2a2bbf37f48776da976b53e9cca1e3 Mon Sep 17 00:00:00 2001 From: Roy Jacobs Date: Tue, 17 Aug 2021 16:44:33 +0200 Subject: [PATCH 126/343] Forgot to run 'Prettier' Signed-off-by: Roy Jacobs --- .../catalog-backend/src/ingestion/processors/gitlab/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts b/plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts index a6c6419d21..1df3d2cb84 100644 --- a/plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts +++ b/plugins/catalog-backend/src/ingestion/processors/gitlab/index.ts @@ -15,4 +15,4 @@ */ export { GitLabClient, paginated } from './client'; -export type { GitLabProject } from "./types"; +export type { GitLabProject } from './types'; From a6ae6115d3b23e6688fcc7081d25a7ce8b8a2ae3 Mon Sep 17 00:00:00 2001 From: Roy Jacobs Date: Tue, 17 Aug 2021 17:08:21 +0200 Subject: [PATCH 127/343] Update api-report.md Signed-off-by: Roy Jacobs --- plugins/catalog-backend/api-report.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/plugins/catalog-backend/api-report.md b/plugins/catalog-backend/api-report.md index 7286855d4e..0c7b08d94c 100644 --- a/plugins/catalog-backend/api-report.md +++ b/plugins/catalog-backend/api-report.md @@ -771,6 +771,27 @@ export class GithubOrgReaderProcessor implements CatalogProcessor { ): Promise; } +// Warning: (ae-missing-release-tag) "GitLabDiscoveryProcessor" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public +export class GitLabDiscoveryProcessor implements CatalogProcessor { + // (undocumented) + static fromConfig( + config: Config, + options: { + logger: Logger_2; + }, + ): GitLabDiscoveryProcessor; + // (undocumented) + readLocation( + location: LocationSpec, + _optional: boolean, + emit: CatalogProcessorEmit, + ): Promise; + // (undocumented) + updateLastActivity(): Promise; +} + // Warning: (ae-missing-release-tag) "HigherOrderOperation" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) From 879a98ca3b36970450f61e33d6c49feafb0e6cdb Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 17 Aug 2021 17:22:57 +0200 Subject: [PATCH 128/343] auth-backend: fix origin filter tests on windows Signed-off-by: Patrik Oldsberg --- plugins/auth-backend/src/service/router.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/auth-backend/src/service/router.test.ts b/plugins/auth-backend/src/service/router.test.ts index ca8a14f4e6..f0f5134848 100644 --- a/plugins/auth-backend/src/service/router.test.ts +++ b/plugins/auth-backend/src/service/router.test.ts @@ -28,19 +28,19 @@ describe('Auth origin filtering', () => { const config = defaultConfig(); config.getOptionalString = getOptionalString; it('Will explode, invalid origin', () => { - const origin = 'https://test\\.example.net'; + const origin = 'https://test.example.net'; expect(createOriginFilter(config)(origin)).toBeFalsy(); }); it('Will explode, invalid origin domain', () => { - const origin = 'https://test-1234\\.examplee.net'; + const origin = 'https://test-1234.examplee.net'; expect(createOriginFilter(config)(origin)).toBeFalsy(); }); it("Won't explode, valid origin with numbers", () => { - const origin = 'https://test-1234\\.example.net'; + const origin = 'https://test-1234.example.net'; expect(createOriginFilter(config)(origin)).toBeTruthy(); }); it("Won't explode, valid origin with chars and numbers", () => { - const origin = 'https://test-test1234\\.example.net'; + const origin = 'https://test-test1234.example.net'; expect(createOriginFilter(config)(origin)).toBeTruthy(); }); }); From 24d0e1ea1253f8f828741d2405cddc589d9d02f6 Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Tue, 17 Aug 2021 17:32:21 +0200 Subject: [PATCH 129/343] Set field id in scaffolder Signed-off-by: Oliver Sand --- .changeset/fifty-taxis-deny.md | 5 +++++ .../src/components/fields/EntityPicker/EntityPicker.tsx | 4 +++- .../components/fields/TextValuePicker/TextValuePicker.tsx | 8 ++++++-- 3 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 .changeset/fifty-taxis-deny.md diff --git a/.changeset/fifty-taxis-deny.md b/.changeset/fifty-taxis-deny.md new file mode 100644 index 0000000000..63100635d1 --- /dev/null +++ b/.changeset/fifty-taxis-deny.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-scaffolder': patch +--- + +Set `id` in ``. diff --git a/plugins/scaffolder/src/components/fields/EntityPicker/EntityPicker.tsx b/plugins/scaffolder/src/components/fields/EntityPicker/EntityPicker.tsx index 04693a94cd..fca40d822f 100644 --- a/plugins/scaffolder/src/components/fields/EntityPicker/EntityPicker.tsx +++ b/plugins/scaffolder/src/components/fields/EntityPicker/EntityPicker.tsx @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { useApi } from '@backstage/core-plugin-api'; import { catalogApiRef, formatEntityRefTitle, @@ -23,7 +24,6 @@ import Autocomplete from '@material-ui/lab/Autocomplete'; import { FieldProps } from '@rjsf/core'; import React from 'react'; import { useAsync } from 'react-use'; -import { useApi } from '@backstage/core-plugin-api'; export const EntityPicker = ({ onChange, @@ -32,6 +32,7 @@ export const EntityPicker = ({ uiSchema, rawErrors, formData, + idSchema, }: FieldProps) => { const allowedKinds = uiSchema['ui:options']?.allowedKinds as string[]; const defaultKind = uiSchema['ui:options']?.defaultKind as string | undefined; @@ -58,6 +59,7 @@ export const EntityPicker = ({ error={rawErrors?.length > 0 && !formData} > ) => ( Date: Tue, 17 Aug 2021 18:13:33 +0200 Subject: [PATCH 130/343] Update API report Signed-off-by: Oliver Sand --- plugins/scaffolder/api-report.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/scaffolder/api-report.md b/plugins/scaffolder/api-report.md index 4df2a0808c..b005610f04 100644 --- a/plugins/scaffolder/api-report.md +++ b/plugins/scaffolder/api-report.md @@ -66,6 +66,7 @@ export const EntityPicker: ({ uiSchema, rawErrors, formData, + idSchema, }: FieldProps) => JSX.Element; // Warning: (ae-missing-release-tag) "EntityPickerFieldExtension" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) @@ -229,6 +230,8 @@ export const TextValuePicker: ({ rawErrors, formData, uiSchema: { 'ui:autofocus': autoFocus }, + idSchema, + placeholder, }: FieldProps) => JSX.Element; // (No @packageDocumentation comment for this package) From 77cdc5a842f85cfd661c4d2f6233d6a5e6230acf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fredrik=20Adel=C3=B6w?= Date: Tue, 17 Aug 2021 19:40:06 +0200 Subject: [PATCH 131/343] Support passing a UserTransformer through the processor to the reader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fredrik Adelöw --- .changeset/shy-pets-join.md | 5 +++ .../api-report.md | 3 ++ .../src/microsoftGraph/read.test.ts | 42 +++++++++++++++++++ .../src/microsoftGraph/read.ts | 2 + .../MicrosoftGraphOrgReaderProcessor.ts | 11 ++++- 5 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 .changeset/shy-pets-join.md diff --git a/.changeset/shy-pets-join.md b/.changeset/shy-pets-join.md new file mode 100644 index 0000000000..ba39dfd4f4 --- /dev/null +++ b/.changeset/shy-pets-join.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-msgraph': patch +--- + +Pass along a `UserTransformer` to the read step diff --git a/plugins/catalog-backend-module-msgraph/api-report.md b/plugins/catalog-backend-module-msgraph/api-report.md index 91f8180e94..f614bdca36 100644 --- a/plugins/catalog-backend-module-msgraph/api-report.md +++ b/plugins/catalog-backend-module-msgraph/api-report.md @@ -111,6 +111,7 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { constructor(options: { providers: MicrosoftGraphProviderConfig[]; logger: Logger_2; + userTransformer?: UserTransformer; groupTransformer?: GroupTransformer; }); // (undocumented) @@ -118,6 +119,7 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { config: Config, options: { logger: Logger_2; + userTransformer?: UserTransformer; groupTransformer?: GroupTransformer; }, ): MicrosoftGraphOrgReaderProcessor; @@ -170,6 +172,7 @@ export function readMicrosoftGraphOrg( options: { userFilter?: string; groupFilter?: string; + userTransformer?: UserTransformer; groupTransformer?: GroupTransformer; logger: Logger_2; }, diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts index 68f7e216ea..82279b76c6 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.test.ts @@ -114,6 +114,48 @@ describe('read microsoft graph', () => { expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1); expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120); }); + + it('should read users with custom transformer', async () => { + async function* getExampleUsers() { + yield { + id: 'userid', + displayName: 'User Name', + mail: 'user.name@example.com', + }; + } + + client.getUsers.mockImplementation(getExampleUsers); + client.getUserPhotoWithSizeLimit.mockResolvedValue( + 'data:image/jpeg;base64,...', + ); + + const { users } = await readMicrosoftGraphUsers(client, { + userFilter: 'accountEnabled eq true', + transformer: async () => ({ + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { name: 'x' }, + spec: { memberOf: [] }, + }), + logger: getVoidLogger(), + }); + + expect(users).toEqual([ + { + apiVersion: 'backstage.io/v1alpha1', + kind: 'User', + metadata: { name: 'x' }, + spec: { memberOf: [] }, + }, + ]); + + expect(client.getUsers).toBeCalledTimes(1); + expect(client.getUsers).toBeCalledWith({ + filter: 'accountEnabled eq true', + }); + expect(client.getUserPhotoWithSizeLimit).toBeCalledTimes(1); + expect(client.getUserPhotoWithSizeLimit).toBeCalledWith('userid', 120); + }); }); describe('readMicrosoftGraphOrganization', () => { diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts index 62513b0e10..6fbd2cb0de 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts @@ -382,12 +382,14 @@ export async function readMicrosoftGraphOrg( options: { userFilter?: string; groupFilter?: string; + userTransformer?: UserTransformer; groupTransformer?: GroupTransformer; logger: Logger; }, ): Promise<{ users: UserEntity[]; groups: GroupEntity[] }> { const { users } = await readMicrosoftGraphUsers(client, { userFilter: options.userFilter, + transformer: options.userTransformer, logger: options.logger, }); const { groups, rootGroup, groupMember, groupMemberOf } = diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts index 88240db618..1495bce93c 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts @@ -28,6 +28,7 @@ import { MicrosoftGraphProviderConfig, readMicrosoftGraphConfig, readMicrosoftGraphOrg, + UserTransformer, } from '../microsoftGraph'; /** @@ -36,11 +37,16 @@ import { export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { private readonly providers: MicrosoftGraphProviderConfig[]; private readonly logger: Logger; + private readonly userTransformer?: UserTransformer; private readonly groupTransformer?: GroupTransformer; static fromConfig( config: Config, - options: { logger: Logger; groupTransformer?: GroupTransformer }, + options: { + logger: Logger; + userTransformer?: UserTransformer; + groupTransformer?: GroupTransformer; + }, ) { const c = config.getOptionalConfig('catalog.processors.microsoftGraphOrg'); return new MicrosoftGraphOrgReaderProcessor({ @@ -52,10 +58,12 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { constructor(options: { providers: MicrosoftGraphProviderConfig[]; logger: Logger; + userTransformer?: UserTransformer; groupTransformer?: GroupTransformer; }) { this.providers = options.providers; this.logger = options.logger; + this.userTransformer = options.userTransformer; this.groupTransformer = options.groupTransformer; } @@ -89,6 +97,7 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { { userFilter: provider.userFilter, groupFilter: provider.groupFilter, + userTransformer: this.userTransformer, groupTransformer: this.groupTransformer, logger: this.logger, }, From 9f9b808f963ecd1ec572ef00515bd58173d1aaf3 Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Tue, 17 Aug 2021 23:28:24 +0200 Subject: [PATCH 132/343] Review comments Signed-off-by: Oliver Sand --- .../src/components/fields/EntityPicker/EntityPicker.tsx | 2 +- .../src/components/fields/TextValuePicker/TextValuePicker.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/scaffolder/src/components/fields/EntityPicker/EntityPicker.tsx b/plugins/scaffolder/src/components/fields/EntityPicker/EntityPicker.tsx index fca40d822f..a8ef524a13 100644 --- a/plugins/scaffolder/src/components/fields/EntityPicker/EntityPicker.tsx +++ b/plugins/scaffolder/src/components/fields/EntityPicker/EntityPicker.tsx @@ -59,7 +59,7 @@ export const EntityPicker = ({ error={rawErrors?.length > 0 && !formData} > ) => ( Date: Tue, 17 Aug 2021 17:19:45 -0400 Subject: [PATCH 133/343] chore(UnregisterEntityDialog): migrate to catalog-react Signed-off-by: Phil Kuang --- .changeset/seven-glasses-sit.md | 6 ++++++ plugins/catalog-react/api-report.md | 11 +++++++++++ .../UnregisterEntityDialog.test.tsx | 2 +- .../UnregisterEntityDialog.tsx | 4 ++-- .../components/UnregisterEntityDialog/index.ts | 17 +++++++++++++++++ .../useUnregisterEntityDialogState.test.tsx | 3 ++- .../useUnregisterEntityDialogState.ts | 2 +- plugins/catalog-react/src/components/index.ts | 1 + .../components/EntityLayout/EntityLayout.tsx | 2 +- .../EntityPageLayout/EntityPageLayout.tsx | 2 +- 10 files changed, 43 insertions(+), 7 deletions(-) create mode 100644 .changeset/seven-glasses-sit.md rename plugins/{catalog => catalog-react}/src/components/UnregisterEntityDialog/UnregisterEntityDialog.test.tsx (99%) rename plugins/{catalog => catalog-react}/src/components/UnregisterEntityDialog/UnregisterEntityDialog.tsx (98%) create mode 100644 plugins/catalog-react/src/components/UnregisterEntityDialog/index.ts rename plugins/{catalog => catalog-react}/src/components/UnregisterEntityDialog/useUnregisterEntityDialogState.test.tsx (98%) rename plugins/{catalog => catalog-react}/src/components/UnregisterEntityDialog/useUnregisterEntityDialogState.ts (98%) diff --git a/.changeset/seven-glasses-sit.md b/.changeset/seven-glasses-sit.md new file mode 100644 index 0000000000..349c972641 --- /dev/null +++ b/.changeset/seven-glasses-sit.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-catalog': patch +'@backstage/plugin-catalog-react': patch +--- + +Migrate and export `UnregisterEntityDialog` component from `catalog-react` package diff --git a/plugins/catalog-react/api-report.md b/plugins/catalog-react/api-report.md index 57bb5360c2..c5f3226874 100644 --- a/plugins/catalog-react/api-report.md +++ b/plugins/catalog-react/api-report.md @@ -715,6 +715,17 @@ export function reduceEntityFilters( // @public (undocumented) export const rootRoute: RouteRef; +// Warning: (ae-forgotten-export) The symbol "Props" needs to be exported by the entry point index.d.ts +// Warning: (ae-missing-release-tag) "UnregisterEntityDialog" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export const UnregisterEntityDialog: ({ + open, + onConfirm, + onClose, + entity, +}: Props_3) => JSX.Element; + // Warning: (ae-missing-release-tag) "useEntity" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public diff --git a/plugins/catalog/src/components/UnregisterEntityDialog/UnregisterEntityDialog.test.tsx b/plugins/catalog-react/src/components/UnregisterEntityDialog/UnregisterEntityDialog.test.tsx similarity index 99% rename from plugins/catalog/src/components/UnregisterEntityDialog/UnregisterEntityDialog.test.tsx rename to plugins/catalog-react/src/components/UnregisterEntityDialog/UnregisterEntityDialog.test.tsx index 55ee42f253..339323c1bf 100644 --- a/plugins/catalog/src/components/UnregisterEntityDialog/UnregisterEntityDialog.test.tsx +++ b/plugins/catalog-react/src/components/UnregisterEntityDialog/UnregisterEntityDialog.test.tsx @@ -21,7 +21,7 @@ import React from 'react'; import { UnregisterEntityDialog } from './UnregisterEntityDialog'; import { ORIGIN_LOCATION_ANNOTATION } from '@backstage/catalog-model'; import { CatalogClient } from '@backstage/catalog-client'; -import { catalogApiRef } from '@backstage/plugin-catalog-react'; +import { catalogApiRef } from '../../api'; import { screen, waitFor } from '@testing-library/react'; import { renderInTestApp } from '@backstage/test-utils'; import * as state from './useUnregisterEntityDialogState'; diff --git a/plugins/catalog/src/components/UnregisterEntityDialog/UnregisterEntityDialog.tsx b/plugins/catalog-react/src/components/UnregisterEntityDialog/UnregisterEntityDialog.tsx similarity index 98% rename from plugins/catalog/src/components/UnregisterEntityDialog/UnregisterEntityDialog.tsx rename to plugins/catalog-react/src/components/UnregisterEntityDialog/UnregisterEntityDialog.tsx index abec90bad1..1f1f01ff48 100644 --- a/plugins/catalog/src/components/UnregisterEntityDialog/UnregisterEntityDialog.tsx +++ b/plugins/catalog-react/src/components/UnregisterEntityDialog/UnregisterEntityDialog.tsx @@ -1,5 +1,5 @@ /* - * Copyright 2020 The Backstage Authors + * Copyright 2021 The Backstage Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ */ import { Entity } from '@backstage/catalog-model'; -import { EntityRefLink } from '@backstage/plugin-catalog-react'; +import { EntityRefLink } from '../EntityRefLink'; import { Box, Button, diff --git a/plugins/catalog-react/src/components/UnregisterEntityDialog/index.ts b/plugins/catalog-react/src/components/UnregisterEntityDialog/index.ts new file mode 100644 index 0000000000..8fc750e706 --- /dev/null +++ b/plugins/catalog-react/src/components/UnregisterEntityDialog/index.ts @@ -0,0 +1,17 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { UnregisterEntityDialog } from './UnregisterEntityDialog'; diff --git a/plugins/catalog/src/components/UnregisterEntityDialog/useUnregisterEntityDialogState.test.tsx b/plugins/catalog-react/src/components/UnregisterEntityDialog/useUnregisterEntityDialogState.test.tsx similarity index 98% rename from plugins/catalog/src/components/UnregisterEntityDialog/useUnregisterEntityDialogState.test.tsx rename to plugins/catalog-react/src/components/UnregisterEntityDialog/useUnregisterEntityDialogState.test.tsx index 6cc2f4b23b..8b4f436aed 100644 --- a/plugins/catalog/src/components/UnregisterEntityDialog/useUnregisterEntityDialogState.test.tsx +++ b/plugins/catalog-react/src/components/UnregisterEntityDialog/useUnregisterEntityDialogState.test.tsx @@ -19,7 +19,8 @@ import { Location, ORIGIN_LOCATION_ANNOTATION, } from '@backstage/catalog-model'; -import { CatalogApi, catalogApiRef } from '@backstage/plugin-catalog-react'; +import { CatalogApi } from '@backstage/catalog-client'; +import { catalogApiRef } from '../../api'; import { act, renderHook, diff --git a/plugins/catalog/src/components/UnregisterEntityDialog/useUnregisterEntityDialogState.ts b/plugins/catalog-react/src/components/UnregisterEntityDialog/useUnregisterEntityDialogState.ts similarity index 98% rename from plugins/catalog/src/components/UnregisterEntityDialog/useUnregisterEntityDialogState.ts rename to plugins/catalog-react/src/components/UnregisterEntityDialog/useUnregisterEntityDialogState.ts index 9b2e68cc5d..ae0a323d36 100644 --- a/plugins/catalog/src/components/UnregisterEntityDialog/useUnregisterEntityDialogState.ts +++ b/plugins/catalog-react/src/components/UnregisterEntityDialog/useUnregisterEntityDialogState.ts @@ -20,7 +20,7 @@ import { getEntityName, ORIGIN_LOCATION_ANNOTATION, } from '@backstage/catalog-model'; -import { catalogApiRef } from '@backstage/plugin-catalog-react'; +import { catalogApiRef } from '../../api'; import { useCallback } from 'react'; import { useAsync } from 'react-use'; import { useApi } from '@backstage/core-plugin-api'; diff --git a/plugins/catalog-react/src/components/index.ts b/plugins/catalog-react/src/components/index.ts index 2664af8ef9..434514c4ed 100644 --- a/plugins/catalog-react/src/components/index.ts +++ b/plugins/catalog-react/src/components/index.ts @@ -23,4 +23,5 @@ export * from './EntityTable'; export * from './EntityTagPicker'; export * from './EntityTypePicker'; export * from './FavoriteEntity'; +export * from './UnregisterEntityDialog'; export * from './UserListPicker'; diff --git a/plugins/catalog/src/components/EntityLayout/EntityLayout.tsx b/plugins/catalog/src/components/EntityLayout/EntityLayout.tsx index eabcbe4993..4eec0d5341 100644 --- a/plugins/catalog/src/components/EntityLayout/EntityLayout.tsx +++ b/plugins/catalog/src/components/EntityLayout/EntityLayout.tsx @@ -39,6 +39,7 @@ import { EntityRefLinks, FavoriteEntity, getEntityRelations, + UnregisterEntityDialog, useEntityCompoundName, } from '@backstage/plugin-catalog-react'; import { Box, TabProps } from '@material-ui/core'; @@ -46,7 +47,6 @@ import { Alert } from '@material-ui/lab'; import React, { useContext, useState } from 'react'; import { useNavigate } from 'react-router'; import { EntityContextMenu } from '../EntityContextMenu/EntityContextMenu'; -import { UnregisterEntityDialog } from '../UnregisterEntityDialog/UnregisterEntityDialog'; type SubRoute = { path: string; diff --git a/plugins/catalog/src/components/EntityPageLayout/EntityPageLayout.tsx b/plugins/catalog/src/components/EntityPageLayout/EntityPageLayout.tsx index 088a9f2c53..c371d6245e 100644 --- a/plugins/catalog/src/components/EntityPageLayout/EntityPageLayout.tsx +++ b/plugins/catalog/src/components/EntityPageLayout/EntityPageLayout.tsx @@ -23,13 +23,13 @@ import { EntityRefLinks, FavoriteEntity, getEntityRelations, + UnregisterEntityDialog, useEntityCompoundName, } from '@backstage/plugin-catalog-react'; import { Box } from '@material-ui/core'; import React, { useContext, useState } from 'react'; import { useNavigate } from 'react-router'; import { EntityContextMenu } from '../EntityContextMenu/EntityContextMenu'; -import { UnregisterEntityDialog } from '../UnregisterEntityDialog/UnregisterEntityDialog'; import { Tabbed } from './Tabbed'; import { From 603094c8369bce8e1943f1d5618a33aaebdd5ebf Mon Sep 17 00:00:00 2001 From: Sathish Kumar Date: Tue, 17 Aug 2021 14:55:47 -0500 Subject: [PATCH 134/343] Adding Backstage Plugin for Gitlab Signed-off-by: Sathish Kumar --- microsite/data/gitlab.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 microsite/data/gitlab.yaml diff --git a/microsite/data/gitlab.yaml b/microsite/data/gitlab.yaml new file mode 100644 index 0000000000..d88f001c07 --- /dev/null +++ b/microsite/data/gitlab.yaml @@ -0,0 +1,12 @@ +--- +title: Gitlab +author: Loblaw +authorUrl: https://github.com/loblaw-sre/backstage-plugin-gitlab +category: CI/CD +description: View Gitlab pipelines, merge requests, languages and contributors. +documentation: https://github.com/loblaw-sre/backstage-plugin-gitlab +iconUrl: https://about.gitlab.com/images/icons/logos/slp-logo.svg +npmPackageName: '@loblaw/backstage-plugin-gitlab' +tags: + - ci + - cd From 83700022a0654facb622b2a41ec6e52d73368dbd Mon Sep 17 00:00:00 2001 From: Sathish Kumar Date: Tue, 17 Aug 2021 19:27:37 -0500 Subject: [PATCH 135/343] Updated the file path for the plugin Signed-off-by: Sathish Kumar --- microsite/data/{ => plugins}/gitlab.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename microsite/data/{ => plugins}/gitlab.yaml (100%) diff --git a/microsite/data/gitlab.yaml b/microsite/data/plugins/gitlab.yaml similarity index 100% rename from microsite/data/gitlab.yaml rename to microsite/data/plugins/gitlab.yaml From 550101a2849e091c4d302209c0e6301f473e60de Mon Sep 17 00:00:00 2001 From: Tim Hansen Date: Tue, 17 Aug 2021 19:03:29 -0600 Subject: [PATCH 136/343] GitLab logo, name tweak Signed-off-by: Tim Hansen --- microsite/data/plugins/gitlab.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/microsite/data/plugins/gitlab.yaml b/microsite/data/plugins/gitlab.yaml index d88f001c07..7a5493c7f4 100644 --- a/microsite/data/plugins/gitlab.yaml +++ b/microsite/data/plugins/gitlab.yaml @@ -1,11 +1,11 @@ --- -title: Gitlab +title: GitLab author: Loblaw authorUrl: https://github.com/loblaw-sre/backstage-plugin-gitlab category: CI/CD -description: View Gitlab pipelines, merge requests, languages and contributors. +description: View GitLab pipelines, merge requests, languages and contributors. documentation: https://github.com/loblaw-sre/backstage-plugin-gitlab -iconUrl: https://about.gitlab.com/images/icons/logos/slp-logo.svg +iconUrl: https://about.gitlab.com/images/press/logo/png/gitlab-icon-rgb.png npmPackageName: '@loblaw/backstage-plugin-gitlab' tags: - ci From cd44f1d602bf58f32771c190a22d22da9445a6c6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Aug 2021 04:07:43 +0000 Subject: [PATCH 137/343] chore(deps): bump dompurify from 2.3.0 to 2.3.1 Bumps [dompurify](https://github.com/cure53/DOMPurify) from 2.3.0 to 2.3.1. - [Release notes](https://github.com/cure53/DOMPurify/releases) - [Commits](https://github.com/cure53/DOMPurify/compare/2.3.0...2.3.1) --- updated-dependencies: - dependency-name: dompurify dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- yarn.lock | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/yarn.lock b/yarn.lock index 4d15795a5f..a201aab185 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12425,15 +12425,10 @@ domhandler@^4.0.0: dependencies: domelementtype "^2.1.0" -dompurify@^2.0.12, dompurify@^2.1.1, dompurify@^2.2.8: - version "2.2.8" - resolved "https://registry.npmjs.org/dompurify/-/dompurify-2.2.8.tgz#ce88e395f6d00b6dc53f80d6b2a6fdf5446873c6" - integrity sha512-9H0UL59EkDLgY3dUFjLV6IEUaHm5qp3mxSqWw7Yyx4Zhk2Jn2cmLe+CNPP3xy13zl8Bqg+0NehQzkdMoVhGRww== - -dompurify@^2.2.9: - version "2.3.0" - resolved "https://registry.npmjs.org/dompurify/-/dompurify-2.3.0.tgz#07bb39515e491588e5756b1d3e8375b5964814e2" - integrity sha512-VV5C6Kr53YVHGOBKO/F86OYX6/iLTw2yVSI721gKetxpHCK/V5TaLEf9ODjRgl1KLSWRMY6cUhAbv/c+IUnwQw== +dompurify@^2.0.12, dompurify@^2.1.1, dompurify@^2.2.8, dompurify@^2.2.9: + version "2.3.1" + resolved "https://registry.npmjs.org/dompurify/-/dompurify-2.3.1.tgz#a47059ca21fd1212d3c8f71fdea6943b8bfbdf6a" + integrity sha512-xGWt+NHAQS+4tpgbOAI08yxW0Pr256Gu/FNE2frZVTbgrBUn8M7tz7/ktS/LZ2MHeGqz6topj0/xY+y8R5FBFw== domutils@1.5.1: version "1.5.1" From 15b41477bd3c5d511720e44e2c1779214ea45dc3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Aug 2021 04:12:04 +0000 Subject: [PATCH 138/343] chore(deps): bump graphiql from 1.4.0 to 1.4.2 Bumps [graphiql](https://github.com/graphql/graphiql) from 1.4.0 to 1.4.2. - [Release notes](https://github.com/graphql/graphiql/releases) - [Changelog](https://github.com/graphql/graphiql/blob/main/CHANGELOG.md) - [Commits](https://github.com/graphql/graphiql/commits/graphiql@1.4.2) --- updated-dependencies: - dependency-name: graphiql dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- yarn.lock | 44 +++++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/yarn.lock b/yarn.lock index 4d15795a5f..682a83563c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3073,14 +3073,15 @@ stream-events "^1.0.1" xdg-basedir "^4.0.0" -"@graphiql/toolkit@^0.1.0": - version "0.1.1" - resolved "https://registry.npmjs.org/@graphiql/toolkit/-/toolkit-0.1.1.tgz#a7da3ba460ceae27bcdc8f03831ca4f88f90f3d7" - integrity sha512-cvsuaPkOA6/TZOdqEdvzqr7i+or2STTpSsteyDkrUXrwftRnH9ZfiUwnHPyf0AC2cKMwpP/Dny/UTS6CLC8ZNQ== +"@graphiql/toolkit@^0.2.0": + version "0.2.2" + resolved "https://registry.npmjs.org/@graphiql/toolkit/-/toolkit-0.2.2.tgz#193d570afcf686c9ee61c92054c1782b9f3c1255" + integrity sha512-kDgYhqnS4p4LqSo1KvLd3tbX8Hhdj0ZrgQuGsosjjEnahiPYmmylxUL1p9lj6348OsypcTlCncGpEjeb9S3TiQ== dependencies: - "@n1ru4l/push-pull-async-iterable-iterator" "^2.0.1" - graphql-ws "^4.1.0" - meros "^1.1.2" + "@n1ru4l/push-pull-async-iterable-iterator" "^2.1.4" + graphql-ws "^4.3.2" + meros "^1.1.4" + optionalDependencies: subscriptions-transport-ws "^0.9.18" "@graphql-codegen/cli@^1.21.3": @@ -4740,10 +4741,10 @@ strict-event-emitter "^0.2.0" xmldom "^0.6.0" -"@n1ru4l/push-pull-async-iterable-iterator@^2.0.1": - version "2.1.2" - resolved "https://registry.npmjs.org/@n1ru4l/push-pull-async-iterable-iterator/-/push-pull-async-iterable-iterator-2.1.2.tgz#e486bf86c4c29e78601694a26f31c2dec0c08d9b" - integrity sha512-KwZGeX2XK7Xj9ksWwei5923QnqIGoEuLlh3O46OW9vc8hQxjzmMTKCgJMVZ5ne5xaWFQYDT2dMpbUhq6hEOhxA== +"@n1ru4l/push-pull-async-iterable-iterator@^2.1.4": + version "2.1.4" + resolved "https://registry.npmjs.org/@n1ru4l/push-pull-async-iterable-iterator/-/push-pull-async-iterable-iterator-2.1.4.tgz#a90225474352f9f159bff979905f707b9c6bcf04" + integrity sha512-qLIvoOUJ+zritv+BlzcBMePKNjKQzH9Rb2i9W98YXxf/M62Lye8qH0peyiU8yJ1tL0kfulWi31BoK10E6BKJeA== "@nodelib/fs.scandir@2.1.3": version "2.1.3" @@ -12528,7 +12529,7 @@ downshift@^6.0.15: prop-types "^15.7.2" react-is "^17.0.2" -dset@^3.0.0: +dset@^3.1.0: version "3.1.0" resolved "https://registry.npmjs.org/dset/-/dset-3.1.0.tgz#23feb6df93816ea452566308b1374d6e869b0d7b" integrity sha512-7xTQ5DzyE59Nn+7ZgXDXjKAGSGmXZHqttMVVz1r4QNfmGpyj+cm2YtI3II0c/+4zS4a9yq2mBhgdeq2QnpcYlw== @@ -14893,15 +14894,15 @@ grapheme-splitter@^1.0.4: integrity sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ== graphiql@^1.0.0-alpha.10: - version "1.4.0" - resolved "https://registry.npmjs.org/graphiql/-/graphiql-1.4.0.tgz#8b17c988720a9da5ea3ebf7ff9d86939b462655e" - integrity sha512-E/Xzfu3YnifdINrK6dKHD1G0qnWkYYK2gHQ//vApUadMb4I4lZQ399ZKt5nqM5kzrATf5FDUTuSpnkaeSoARkQ== + version "1.4.2" + resolved "https://registry.npmjs.org/graphiql/-/graphiql-1.4.2.tgz#a1dc1a4d8d35f60c90d6d8a9eb62a99756e9fd9b" + integrity sha512-TQDuuU/ZqTWV1yQDpVEiKskg0IYA+Wck37DYrrFzLlpgZWRbWiyab1PyHKiRep7J540CgScBg6C/gGCymKyO3g== dependencies: - "@graphiql/toolkit" "^0.1.0" + "@graphiql/toolkit" "^0.2.0" codemirror "^5.54.0" codemirror-graphql "^1.0.0" copy-to-clipboard "^3.2.0" - dset "^3.0.0" + dset "^3.1.0" entities "^2.0.0" graphql-language-service "^3.1.2" markdown-it "^10.0.0" @@ -15030,7 +15031,12 @@ graphql-type-json@^0.3.2: resolved "https://registry.npmjs.org/graphql-type-json/-/graphql-type-json-0.3.2.tgz#f53a851dbfe07bd1c8157d24150064baab41e115" integrity sha512-J+vjof74oMlCWXSvt0DOf2APEdZOCdubEvGDUAlqH//VBYcOYsGgRW7Xzorr44LvkjiuvecWc8fChxuZZbChtg== -graphql-ws@^4.1.0, graphql-ws@^4.4.1: +graphql-ws@^4.3.2: + version "4.9.0" + resolved "https://registry.npmjs.org/graphql-ws/-/graphql-ws-4.9.0.tgz#5cfd8bb490b35e86583d8322f5d5d099c26e365c" + integrity sha512-sHkK9+lUm20/BGawNEWNtVAeJzhZeBg21VmvmLoT5NdGVeZWv5PdIhkcayQIAgjSyyQ17WMKmbDijIPG2On+Ag== + +graphql-ws@^4.4.1: version "4.7.0" resolved "https://registry.npmjs.org/graphql-ws/-/graphql-ws-4.7.0.tgz#b323fbf35a3736eed85dac24c0054d6d10c93e62" integrity sha512-Md8SsmC9ZlsogFPd3Ot8HbIAAqsHh8Xoq7j4AmcIat1Bh6k91tjVyQvA0Au1/BolXSYq+RDvib6rATU2Hcf1Xw== @@ -19183,7 +19189,7 @@ merge@^2.1.0: resolved "https://registry.npmjs.org/merge/-/merge-2.1.1.tgz#59ef4bf7e0b3e879186436e8481c06a6c162ca98" integrity sha512-jz+Cfrg9GWOZbQAnDQ4hlVnQky+341Yk5ru8bZSe6sIDTCIg8n9i/u7hSQGSVOF3C7lH6mGtqjkiT9G4wFLL0w== -meros@1.1.4, meros@^1.1.2: +meros@1.1.4, meros@^1.1.4: version "1.1.4" resolved "https://registry.npmjs.org/meros/-/meros-1.1.4.tgz#c17994d3133db8b23807f62bec7f0cb276cfd948" integrity sha512-E9ZXfK9iQfG9s73ars9qvvvbSIkJZF5yOo9j4tcwM5tN8mUKfj/EKN5PzOr3ZH0y5wL7dLAHw3RVEfpQV9Q7VQ== From be498d22feb04cf97ed2a29c4a6a36eccfcd062d Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Wed, 18 Aug 2021 09:05:14 +0200 Subject: [PATCH 139/343] Pass along a `OrganizationTransformer` to the read step Follow up to #6855 Signed-off-by: Oliver Sand --- .changeset/violet-fishes-look.md | 5 +++++ .../api-report.md | 3 +++ .../src/microsoftGraph/read.ts | 18 +++++++++++++----- .../MicrosoftGraphOrgReaderProcessor.ts | 6 ++++++ 4 files changed, 27 insertions(+), 5 deletions(-) create mode 100644 .changeset/violet-fishes-look.md diff --git a/.changeset/violet-fishes-look.md b/.changeset/violet-fishes-look.md new file mode 100644 index 0000000000..f526638a59 --- /dev/null +++ b/.changeset/violet-fishes-look.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend-module-msgraph': patch +--- + +Pass along a `OrganizationTransformer` to the read step diff --git a/plugins/catalog-backend-module-msgraph/api-report.md b/plugins/catalog-backend-module-msgraph/api-report.md index f614bdca36..7a10ad2995 100644 --- a/plugins/catalog-backend-module-msgraph/api-report.md +++ b/plugins/catalog-backend-module-msgraph/api-report.md @@ -113,6 +113,7 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { logger: Logger_2; userTransformer?: UserTransformer; groupTransformer?: GroupTransformer; + organizationTransformer?: OrganizationTransformer; }); // (undocumented) static fromConfig( @@ -121,6 +122,7 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { logger: Logger_2; userTransformer?: UserTransformer; groupTransformer?: GroupTransformer; + organizationTransformer?: OrganizationTransformer; }, ): MicrosoftGraphOrgReaderProcessor; // (undocumented) @@ -174,6 +176,7 @@ export function readMicrosoftGraphOrg( groupFilter?: string; userTransformer?: UserTransformer; groupTransformer?: GroupTransformer; + organizationTransformer?: OrganizationTransformer; logger: Logger_2; }, ): Promise<{ diff --git a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts index 6fbd2cb0de..096cf8a479 100644 --- a/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts +++ b/plugins/catalog-backend-module-msgraph/src/microsoftGraph/read.ts @@ -20,6 +20,7 @@ import { } from '@backstage/catalog-model'; import * as MicrosoftGraph from '@microsoft/microsoft-graph-types'; import limiterFactory from 'p-limit'; +import { Logger } from 'winston'; import { MicrosoftGraphClient } from './client'; import { MICROSOFT_GRAPH_GROUP_ID_ANNOTATION, @@ -33,7 +34,6 @@ import { OrganizationTransformer, UserTransformer, } from './types'; -import { Logger } from 'winston'; export async function defaultUserTransformer( user: MicrosoftGraph.User, @@ -212,7 +212,11 @@ export async function defaultGroupTransformer( export async function readMicrosoftGraphGroups( client: MicrosoftGraphClient, tenantId: string, - options?: { groupFilter?: string; transformer?: GroupTransformer }, + options?: { + groupFilter?: string; + groupTransformer?: GroupTransformer; + organizationTransformer?: OrganizationTransformer; + }, ): Promise<{ groups: GroupEntity[]; // With all relations empty rootGroup: GroupEntity | undefined; // With all relations empty @@ -224,13 +228,15 @@ export async function readMicrosoftGraphGroups( const groupMemberOf: Map> = new Map(); const limiter = limiterFactory(10); - const { rootGroup } = await readMicrosoftGraphOrganization(client, tenantId); + const { rootGroup } = await readMicrosoftGraphOrganization(client, tenantId, { + transformer: options?.organizationTransformer, + }); if (rootGroup) { groupMember.set(rootGroup.metadata.name, new Set()); groups.push(rootGroup); } - const transformer = options?.transformer ?? defaultGroupTransformer; + const transformer = options?.groupTransformer ?? defaultGroupTransformer; const promises: Promise[] = []; for await (const group of client.getGroups({ @@ -384,6 +390,7 @@ export async function readMicrosoftGraphOrg( groupFilter?: string; userTransformer?: UserTransformer; groupTransformer?: GroupTransformer; + organizationTransformer?: OrganizationTransformer; logger: Logger; }, ): Promise<{ users: UserEntity[]; groups: GroupEntity[] }> { @@ -395,7 +402,8 @@ export async function readMicrosoftGraphOrg( const { groups, rootGroup, groupMember, groupMemberOf } = await readMicrosoftGraphGroups(client, tenantId, { groupFilter: options?.groupFilter, - transformer: options?.groupTransformer, + groupTransformer: options?.groupTransformer, + organizationTransformer: options?.organizationTransformer, }); resolveRelations(rootGroup, groups, users, groupMember, groupMemberOf); diff --git a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts index 1495bce93c..161cc8799f 100644 --- a/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts +++ b/plugins/catalog-backend-module-msgraph/src/processors/MicrosoftGraphOrgReaderProcessor.ts @@ -26,6 +26,7 @@ import { GroupTransformer, MicrosoftGraphClient, MicrosoftGraphProviderConfig, + OrganizationTransformer, readMicrosoftGraphConfig, readMicrosoftGraphOrg, UserTransformer, @@ -39,6 +40,7 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { private readonly logger: Logger; private readonly userTransformer?: UserTransformer; private readonly groupTransformer?: GroupTransformer; + private readonly organizationTransformer?: OrganizationTransformer; static fromConfig( config: Config, @@ -46,6 +48,7 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { logger: Logger; userTransformer?: UserTransformer; groupTransformer?: GroupTransformer; + organizationTransformer?: OrganizationTransformer; }, ) { const c = config.getOptionalConfig('catalog.processors.microsoftGraphOrg'); @@ -60,11 +63,13 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { logger: Logger; userTransformer?: UserTransformer; groupTransformer?: GroupTransformer; + organizationTransformer?: OrganizationTransformer; }) { this.providers = options.providers; this.logger = options.logger; this.userTransformer = options.userTransformer; this.groupTransformer = options.groupTransformer; + this.organizationTransformer = options.organizationTransformer; } async readLocation( @@ -99,6 +104,7 @@ export class MicrosoftGraphOrgReaderProcessor implements CatalogProcessor { groupFilter: provider.groupFilter, userTransformer: this.userTransformer, groupTransformer: this.groupTransformer, + organizationTransformer: this.organizationTransformer, logger: this.logger, }, ); From db83840a65cb93a9c6061ef83dfcd4dcdb911798 Mon Sep 17 00:00:00 2001 From: Johan Haals Date: Wed, 18 Aug 2021 11:06:01 +0200 Subject: [PATCH 140/343] Update .changeset/early-ducks-stare.md Signed-off-by: Johan Haals Co-authored-by: Patrik Oldsberg --- .changeset/early-ducks-stare.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/early-ducks-stare.md b/.changeset/early-ducks-stare.md index 58b1525e25..1fd44f2755 100644 --- a/.changeset/early-ducks-stare.md +++ b/.changeset/early-ducks-stare.md @@ -1,5 +1,5 @@ --- -'@backstage/plugin-catalog-backend': minor +'@backstage/plugin-catalog-backend': patch --- Updates the `DefaultProcessingDatabase` to accept a refresh interval function instead of a fixed refresh interval in seconds which used to default to 100s. The catalog now ships with a default refresh interval function that schedules entities for refresh every 100-150 seconds, this should From ee99798da7aa2c257737af1c3c95dceaa5eb3c4b Mon Sep 17 00:00:00 2001 From: Oliver Sand Date: Wed, 18 Aug 2021 11:19:25 +0200 Subject: [PATCH 141/343] Correct version requirements on postgres from 11 to 12 (#6863) Signed-off-by: Oliver Sand --- .changeset/quiet-hornets-yawn.md | 6 ++++++ docs/features/search/search-engines.md | 2 +- plugins/search-backend-module-pg/README.md | 2 +- .../src/database/DatabaseDocumentStore.ts | 8 ++++---- 4 files changed, 12 insertions(+), 6 deletions(-) create mode 100644 .changeset/quiet-hornets-yawn.md diff --git a/.changeset/quiet-hornets-yawn.md b/.changeset/quiet-hornets-yawn.md new file mode 100644 index 0000000000..35a0a7706d --- /dev/null +++ b/.changeset/quiet-hornets-yawn.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-search-backend-module-pg': patch +--- + +Correct version requirements on postgres from 11 to 12. Postgres 12 is required +due the use of generated columns. diff --git a/docs/features/search/search-engines.md b/docs/features/search/search-engines.md index 69a223b29e..a0302ce279 100644 --- a/docs/features/search/search-engines.md +++ b/docs/features/search/search-engines.md @@ -43,7 +43,7 @@ provides decent results and performs well with ten thousands of indexed documents. The connection to postgres is established via the database manager also used by other plugins. -> **Important**: The search plugin requires at least Postgres 11! +> **Important**: The search plugin requires at least Postgres 12! To use the `PgSearchEngine`, make sure that you have a Postgres database configured and make the following changes to your backend: diff --git a/plugins/search-backend-module-pg/README.md b/plugins/search-backend-module-pg/README.md index fb9dbccc4a..d3f98132aa 100644 --- a/plugins/search-backend-module-pg/README.md +++ b/plugins/search-backend-module-pg/README.md @@ -8,7 +8,7 @@ well with ten thousands of indexed documents. The connection to postgres is established via the database manager also used by other plugins. -> **Important**: The search plugin requires at least Postgres 11! +> **Important**: The search plugin requires at least Postgres 12! ## Getting started diff --git a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts index ecaa3e2f0f..77f15746a6 100644 --- a/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts +++ b/plugins/search-backend-module-pg/src/database/DatabaseDocumentStore.ts @@ -34,18 +34,18 @@ export class DatabaseDocumentStore implements DatabaseStore { try { const majorVersion = await queryPostgresMajorVersion(knex); - if (majorVersion < 11) { + if (majorVersion < 12) { // We are using some features (like generated columns) that aren't // available in older postgres versions. throw new Error( - `The PgSearchEngine requires at least postgres version 11 (but is running on ${majorVersion})`, + `The PgSearchEngine requires at least postgres version 12 (but is running on ${majorVersion})`, ); } } catch { // Actually both mysql and sqlite have a full text search, too. We could // implement them separately or add them here. throw new Error( - 'The PgSearchEngine is only supported when using a postgres database (>=11.x)', + 'The PgSearchEngine is only supported when using a postgres database (>=12.x)', ); } @@ -59,7 +59,7 @@ export class DatabaseDocumentStore implements DatabaseStore { try { const majorVersion = await queryPostgresMajorVersion(knex); - return majorVersion >= 11; + return majorVersion >= 12; } catch { return false; } From 500abf47ebd30b0f7af80a5b18109c5f3ea162ff Mon Sep 17 00:00:00 2001 From: Himanshu Mishra Date: Wed, 18 Aug 2021 13:35:52 +0200 Subject: [PATCH 142/343] marketplace: add firehyrant plugin Signed-off-by: Himanshu Mishra --- microsite/data/plugins/firehyrant.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 microsite/data/plugins/firehyrant.yaml diff --git a/microsite/data/plugins/firehyrant.yaml b/microsite/data/plugins/firehyrant.yaml new file mode 100644 index 0000000000..5faad64d37 --- /dev/null +++ b/microsite/data/plugins/firehyrant.yaml @@ -0,0 +1,9 @@ +--- +title: FireHydrant +author: FireHyrant +authorUrl: https://firehydrant.io/ +category: Incident Management +description: The FireHydrant plugin brings incident management to Backstage, and it displays service incidents information such as active incidents and incident analytics. +documentation: https://github.com/backstage/backstage/blob/master/plugins/firehydrant/README.md +iconUrl: https://raw.githubusercontent.com/backstage/backstage/master/plugins/firehydrant/doc/firehydrant_logo.png +npmPackageName: '@backstage/plugin-firehyrant' From d041655a7d2b60977033e41acccabac0fe696c01 Mon Sep 17 00:00:00 2001 From: Vincenzo Scamporlino Date: Wed, 18 Aug 2021 09:13:27 +0200 Subject: [PATCH 143/343] copy-text-button: use Clipboard API Co-authored-by: Himanshu Mishra Signed-off-by: Vincenzo Scamporlino --- .changeset/nervous-mails-peel.md | 5 ++ .../CodeSnippet/CodeSnippet.test.tsx | 17 ----- .../components/CodeSnippet/CodeSnippet.tsx | 2 +- .../CopyTextButton/CopyTextButton.test.tsx | 12 +++- .../CopyTextButton/CopyTextButton.tsx | 24 ++----- .../PlainApiDefinitionWidget.test.tsx | 5 +- .../PlainApiDefinitionWidget.test.tsx.snap | 62 +++++++++++++++++++ .../components/EntityBadgesDialog.test.tsx | 2 +- 8 files changed, 86 insertions(+), 43 deletions(-) create mode 100644 .changeset/nervous-mails-peel.md create mode 100644 plugins/api-docs/src/components/PlainApiDefinitionWidget/__snapshots__/PlainApiDefinitionWidget.test.tsx.snap diff --git a/.changeset/nervous-mails-peel.md b/.changeset/nervous-mails-peel.md new file mode 100644 index 0000000000..530fa9023a --- /dev/null +++ b/.changeset/nervous-mails-peel.md @@ -0,0 +1,5 @@ +--- +'@backstage/core-components': patch +--- + +Fix accessibility issue in ``. The component doesn't render anymore an hidden `textarea` containing the text to be copied. diff --git a/packages/core-components/src/components/CodeSnippet/CodeSnippet.test.tsx b/packages/core-components/src/components/CodeSnippet/CodeSnippet.test.tsx index 298a8d5780..a4ffd3462b 100644 --- a/packages/core-components/src/components/CodeSnippet/CodeSnippet.test.tsx +++ b/packages/core-components/src/components/CodeSnippet/CodeSnippet.test.tsx @@ -15,8 +15,6 @@ */ import React from 'react'; -import { fireEvent } from '@testing-library/react'; -import { act } from 'react-dom/test-utils'; import { renderInTestApp } from '@backstage/test-utils'; import { CodeSnippet } from './CodeSnippet'; @@ -56,19 +54,4 @@ describe('', () => { expect(getByText('2')).toBeInTheDocument(); expect(getByText('3')).toBeInTheDocument(); }); - - it('copy code using button', async () => { - jest.useFakeTimers(); - document.execCommand = jest.fn(); - const { getByTitle } = await renderInTestApp( - , - ); - const button = getByTitle('Text copied to clipboard'); - fireEvent.click(button); - act(() => { - jest.runAllTimers(); - }); - expect(document.execCommand).toHaveBeenCalled(); - jest.useRealTimers(); - }); }); diff --git a/packages/core-components/src/components/CodeSnippet/CodeSnippet.tsx b/packages/core-components/src/components/CodeSnippet/CodeSnippet.tsx index a3badaad07..613dff80a9 100644 --- a/packages/core-components/src/components/CodeSnippet/CodeSnippet.tsx +++ b/packages/core-components/src/components/CodeSnippet/CodeSnippet.tsx @@ -42,7 +42,7 @@ export const CodeSnippet = ({ const mode = theme.palette.type === 'dark' ? dark : docco; const highlightColor = theme.palette.type === 'dark' ? '#256bf3' : '#e6ffed'; return ( -
+
{ const PopperJS = jest.requireActual('popper.js'); @@ -51,14 +55,16 @@ const apiRegistry = ApiRegistry.from([ ], ]); +jest.mock('copy-to-clipboard', () => jest.fn()); + describe('', () => { it('renders without exploding', async () => { - const { getByDisplayValue } = await renderInTestApp( + const { getByTestId } = await renderInTestApp( , ); - getByDisplayValue('mockText'); + expect(getByTestId('copy-button')).toBeInTheDocument(); }); it('displays tooltip on click', async () => { @@ -74,7 +80,7 @@ describe('', () => { act(() => { jest.runAllTimers(); }); - expect(document.execCommand).toHaveBeenCalled(); + expect(copy).toHaveBeenCalledWith('mockText'); rendered.getByText('mockTooltip'); jest.useRealTimers(); }); diff --git a/packages/core-components/src/components/CopyTextButton/CopyTextButton.tsx b/packages/core-components/src/components/CopyTextButton/CopyTextButton.tsx index b5bc516d1a..b7120f6ad3 100644 --- a/packages/core-components/src/components/CopyTextButton/CopyTextButton.tsx +++ b/packages/core-components/src/components/CopyTextButton/CopyTextButton.tsx @@ -14,11 +14,11 @@ * limitations under the License. */ -import { errorApiRef, useApi } from '@backstage/core-plugin-api'; import { IconButton, Tooltip } from '@material-ui/core'; import CopyIcon from '@material-ui/icons/FileCopy'; import PropTypes from 'prop-types'; -import React, { MouseEventHandler, useRef, useState } from 'react'; +import React, { MouseEventHandler, useState } from 'react'; +import { useCopyToClipboard } from 'react-use'; /** * Copy text button with visual feedback in the form of @@ -51,31 +51,17 @@ export const CopyTextButton = (props: Props) => { ...defaultProps, ...props, }; - const errorApi = useApi(errorApiRef); - const inputRef = useRef(null); const [open, setOpen] = useState(false); + const [, copyToClipboard] = useCopyToClipboard(); const handleCopyClick: MouseEventHandler = e => { e.stopPropagation(); setOpen(true); - - try { - if (inputRef.current) { - inputRef.current.select(); - document.execCommand('copy'); - } - } catch (error) { - errorApi.post(error); - } + copyToClipboard(text); }; return ( <> -