diff --git a/.changeset/extragavent-fast-fly.md b/.changeset/extragavent-fast-fly.md index 68f5d7a848..ffa2e6e859 100644 --- a/.changeset/extragavent-fast-fly.md +++ b/.changeset/extragavent-fast-fly.md @@ -7,4 +7,4 @@ Fixing badges-backend plugin to get a token from the TokenManager instead of par Implementing an obfuscation feature to protect an open badges endpoint from being enumerated. The feature is disabled by default and the change is compatible with the previous version. -**BREAKING**: createRouter now require that a tokenManager, logger and identityApi are passed in as options. +**BREAKING**: createRouter now require that a tokenManager, logger, identityApi and database, are passed in as options. diff --git a/packages/backend/src/plugins/badges.ts b/packages/backend/src/plugins/badges.ts index fa50731809..6b40d1e5fb 100644 --- a/packages/backend/src/plugins/badges.ts +++ b/packages/backend/src/plugins/badges.ts @@ -20,10 +20,15 @@ import { } from '@backstage/plugin-badges-backend'; import { Router } from 'express'; import { PluginEnvironment } from '../types'; +import { DatabaseBadgesStore } from '@backstage/plugin-badges-backend'; export default async function createPlugin( env: PluginEnvironment, ): Promise { + const db = await DatabaseBadgesStore.create({ + database: env.database, + }); + return await createRouter({ config: env.config, discovery: env.discovery, @@ -31,5 +36,6 @@ export default async function createPlugin( tokenManager: env.tokenManager, logger: env.logger, identity: env.identity, + db: db, }); } diff --git a/plugins/badges-backend/README.md b/plugins/badges-backend/README.md index 6444a5f828..4fcef981f1 100644 --- a/plugins/badges-backend/README.md +++ b/plugins/badges-backend/README.md @@ -32,12 +32,18 @@ import { PluginEnvironment } from '../types'; export default async function createPlugin( env: PluginEnvironment, ): Promise { + const db = await DatabaseBadgesStore.create({ + database: env.database, + }); + return await createRouter({ config: env.config, discovery: env.discovery, badgeFactories: createDefaultBadgeFactories(), tokenManager: env.tokenManager, logger: env.logger, + identity: env.identity, + db: db, }); } ``` @@ -123,16 +129,18 @@ To enable the obfuscation you need to activate the `obfuscation` feature in the ```yaml app: badges: - obfuscate: 'true' + obfuscate: true ``` +> Note that you cannot use env vars to set the `obfuscate` value. It must be a boolean value and env vars are always strings. + Also you need to provide the [salt]() in the `app-config.yaml`: ```yaml custom: badges-backend: salt: # required - cacheTimeToLive: 10 # minutes (optional) + cacheTimeToLive: 60 # minutes (optional) ``` Any string can be used as a salt, but it's recommended to use a long random string to increase entropy. You can generate a random string using the following command: @@ -148,7 +156,7 @@ openssl rand -hex 32 The badges backend api exposes two main endpoints for entity badges. The `/badges` prefix is arbitrary, and the default for the example backend. -### If obfuscation is disabled (default or apps.badges.obfuscate: "false") +### If obfuscation is disabled (default or apps.badges.obfuscate: false) - `/badges/entity/:namespace/:kind/:name/badge-specs` List all defined badges for a particular entity, in json format. See @@ -159,7 +167,7 @@ The badges backend api exposes two main endpoints for entity badges. The an SVG image. If the `accept` request header prefers `application/json` the badge spec as JSON will be returned instead of the image. -### If obfuscation is enabled (apps.badges.obfuscate: "true") +### If obfuscation is enabled (apps.badges.obfuscate: true) - `/badges/entity/:namespace/:kind/:name/obfuscated` Get the obfuscated entity hash from name, namespace, kind. diff --git a/plugins/badges-backend/api-report.md b/plugins/badges-backend/api-report.md index f43acf395f..6f252cf27d 100644 --- a/plugins/badges-backend/api-report.md +++ b/plugins/badges-backend/api-report.md @@ -7,6 +7,7 @@ import { CatalogApi } from '@backstage/catalog-client'; import { Config } from '@backstage/config'; import { Entity } from '@backstage/catalog-model'; import express from 'express'; +import { GetEntitiesResponse } from '@backstage/catalog-client'; import { IdentityApi } from '@backstage/plugin-auth-node'; import { Logger } from 'winston'; import { PluginEndpointDiscovery } from '@backstage/backend-common'; @@ -81,6 +82,51 @@ export type BadgeSpec = { markdown: string; }; +// @public +export interface BadgesStore { + // (undocumented) + countAllBadges(): Promise; + // (undocumented) + createAllBadges( + entities: GetEntitiesResponse, + salt: string | undefined, + ): Promise; + // (undocumented) + deleteObsoleteHashes( + entities: GetEntitiesResponse, + salt: string | undefined, + ): Promise; + // (undocumented) + getAllBadges(): Promise< + { + name: string; + namespace: string; + kind: string; + hash: string; + }[] + >; + // (undocumented) + getBadgeFromHash(hash: string): Promise< + | { + name: string; + namespace: string; + kind: string; + } + | undefined + >; + // (undocumented) + getHashFromEntityMetadata( + name: string, + namespace: string, + kind: string, + ): Promise< + | { + hash: string; + } + | undefined + >; +} + // @public (undocumented) export type BadgeStyle = (typeof BADGE_STYLES)[number]; @@ -114,6 +160,8 @@ export interface RouterOptions { // (undocumented) config: Config; // (undocumented) + db: BadgesStore; + // (undocumented) discovery: PluginEndpointDiscovery; // (undocumented) identity: IdentityApi; diff --git a/plugins/badges-backend/migrations/20230404_init.js b/plugins/badges-backend/migrations/20230404_init.js new file mode 100644 index 0000000000..d9a395d09a --- /dev/null +++ b/plugins/badges-backend/migrations/20230404_init.js @@ -0,0 +1,36 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +exports.up = async function up(knex) { + await knex.schema.createTable('badges', table => { + table.string('kind').notNullable(); + table.string('namespace').notNullable(); + table.string('name').notNullable(); + table + .string('hash') + .unique() + .comment( + 'Hash is calculated from the SHA256 of badge kind, namespace, name, and applications salt', + ) + .notNullable(); + table.index(['hash'], 'badges_hash_index'); + table.primary(['hash']); + }); +}; + +exports.down = async function down(knex) { + await knex.schema.dropTable('badges'); +}; diff --git a/plugins/badges-backend/package.json b/plugins/badges-backend/package.json index 29c824fd70..4b05ffb2e4 100644 --- a/plugins/badges-backend/package.json +++ b/plugins/badges-backend/package.json @@ -44,23 +44,21 @@ "cors": "^2.8.5", "express": "^4.17.1", "express-promise-router": "^4.1.0", + "knex": "^2.4.2", "lodash": "^4.17.21", "supertest": "^6.3.3", "winston": "^3.2.1", "yn": "^4.0.0" }, "devDependencies": { + "@backstage/backend-test-utils": "workspace:^", "@backstage/catalog-client": "workspace:^", "@backstage/cli": "workspace:^", "@backstage/core-app-api": "workspace:^", "@backstage/dev-utils": "workspace:^", "@backstage/test-utils": "workspace:^", - "@testing-library/jest-dom": "^5.10.1", - "@testing-library/react": "^12.1.3", - "@testing-library/user-event": "^14.0.0", "@types/node": "*", - "cross-fetch": "^3.1.5", - "msw": "^0.49.0" + "cross-fetch": "^3.1.5" }, "files": [ "dist" diff --git a/plugins/badges-backend/src/database/badgesStore.ts b/plugins/badges-backend/src/database/badgesStore.ts new file mode 100644 index 0000000000..fa7bfa3f40 --- /dev/null +++ b/plugins/badges-backend/src/database/badgesStore.ts @@ -0,0 +1,182 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + PluginDatabaseManager, + resolvePackagePath, +} from '@backstage/backend-common'; +import { Knex } from 'knex'; +import crypto from 'crypto'; +import { GetEntitiesResponse } from '@backstage/catalog-client'; + +/** + * internal + * @public + */ +export interface BadgesStore { + createAllBadges( + entities: GetEntitiesResponse, + salt: string | undefined, + ): Promise; + + getBadgeFromHash( + hash: string, + ): Promise<{ name: string; namespace: string; kind: string } | undefined>; + + getHashFromEntityMetadata( + name: string, + namespace: string, + kind: string, + ): Promise<{ hash: string } | undefined>; + + deleteObsoleteHashes( + entities: GetEntitiesResponse, + salt: string | undefined, + ): Promise; + + countAllBadges(): Promise; + + getAllBadges(): Promise< + { name: string; namespace: string; kind: string; hash: string }[] + >; +} + +const migrationsDir = resolvePackagePath( + '@backstage/plugin-badges-backend', // Package name + 'migrations', // Migrations directory +); + +/** + * DatabaseBadgesStore + * @internal + */ +export class DatabaseBadgesStore implements BadgesStore { + private constructor(private readonly db: Knex) {} + + static async create({ + database, + skipMigrations, + }: { + database: PluginDatabaseManager; + skipMigrations?: boolean; + }): Promise { + const client = await database.getClient(); + + if (!database.migrations?.skip && !skipMigrations) { + await client.migrate.latest({ + directory: migrationsDir, + }); + } + + return new DatabaseBadgesStore(client); + } + + async createAllBadges( + entities: GetEntitiesResponse, + salt: string, + ): Promise { + for (const entity of entities.items) { + const name = entity.metadata.name.toLocaleLowerCase(); + const namespace = + entity.metadata.namespace?.toLocaleLowerCase() ?? 'default'; + const kind = entity.kind.toLocaleLowerCase(); + const entityHash = crypto + .createHash('sha256') + .update(`${kind}:${namespace}:${name}:${salt}`) + .digest('hex'); + + await this.db('badges') + .insert({ + hash: entityHash, + namespace: namespace, + name: name, + kind: kind, + }) + .onConflict() + .ignore(); + } + } + + async getAllBadges(): Promise< + { name: string; namespace: string; kind: string; hash: string }[] + > { + const result = await this.db('badges').select('*').orderBy('name', 'asc'); + return result; + } + + async getBadgeFromHash( + hash: string, + ): Promise<{ name: string; namespace: string; kind: string } | undefined> { + const result = await this.db('badges') + .select('namespace', 'name', 'kind') + .where({ hash: hash }) + .first(); + + return result; + } + + async getHashFromEntityMetadata( + name: string, + namespace: string, + kind: string, + ): Promise<{ hash: string } | undefined> { + const result = await this.db('badges') + .select('hash') + .where({ name: name, namespace: namespace, kind: kind }) + .first(); + + return result; + } + + async deleteObsoleteHashes( + entities: GetEntitiesResponse, + salt: string, + ): Promise { + const entityInCatalog: string[] = []; + const entityInBadgeDatabase: string[] = []; + let entityToDelete: string[] = []; + + for (const entity of entities.items) { + const name = entity.metadata.name.toLowerCase(); + const namespace = entity.metadata.namespace?.toLowerCase() ?? 'default'; + const kind = entity.kind.toLowerCase(); + const entityHash = crypto + .createHash('sha256') + .update(`${kind}:${namespace}:${name}:${salt}`) + .digest('hex'); + + entityInCatalog.push(entityHash); + } + + for (const entity of await this.getAllBadges()) { + entityInBadgeDatabase.push(entity.hash); + } + + entityToDelete = entityInBadgeDatabase.filter( + entity => !entityInCatalog.includes(entity), + ); + + for (const entity of entityToDelete) { + await this.db('badges').where({ hash: entity }).del(); + } + } + + async countAllBadges(): Promise { + const result = await this.db('badges').countDistinct('hash as count'); + const count: number = +result[0].count; + return count; + } +} diff --git a/plugins/badges-backend/src/index.ts b/plugins/badges-backend/src/index.ts index a45714cb81..35a061c516 100644 --- a/plugins/badges-backend/src/index.ts +++ b/plugins/badges-backend/src/index.ts @@ -24,3 +24,4 @@ export * from './badges'; export * from './lib'; export * from './service/router'; export * from './types'; +export * from './database/badgesStore'; diff --git a/plugins/badges-backend/src/service/router.ts b/plugins/badges-backend/src/service/router.ts index 5944975a9b..db568eb0d0 100644 --- a/plugins/badges-backend/src/service/router.ts +++ b/plugins/badges-backend/src/service/router.ts @@ -26,11 +26,13 @@ import { Config } from '@backstage/config'; import { NotFoundError } from '@backstage/errors'; import { BadgeBuilder, DefaultBadgeBuilder } from '../lib/BadgeBuilder'; import { BadgeContext, BadgeFactories } from '../types'; -import { isEmpty, isNil } from 'lodash'; +import { isNil } from 'lodash'; import crypto from 'crypto'; import { Logger } from 'winston'; import { IdentityApi } from '@backstage/plugin-auth-node'; import { getBearerTokenFromAuthorizationHeader } from '@backstage/plugin-auth-node'; +import type { BadgesStore } from '../database/badgesStore'; + /** @public */ export interface RouterOptions { badgeBuilder?: BadgeBuilder; @@ -41,6 +43,7 @@ export interface RouterOptions { tokenManager: TokenManager; logger: Logger; identity: IdentityApi; + db: BadgesStore; } /** @public */ @@ -54,38 +57,46 @@ export async function createRouter( new DefaultBadgeBuilder(options.badgeFactories || {}); const router = Router(); - const tokenManager = options.tokenManager; - let lookupTable: Map< - string, - { - name: string; - namespace: string; - kind: string; - creationDate: number; - } - > = new Map(); + const { db, config, logger, tokenManager, discovery, identity } = options; + const baseUrl = await discovery.getExternalBaseUrl('badges'); + const salt = config.getOptionalString('custom.badges-backend.salt'); + const cacheTimeToLive = + config.getOptionalNumber('badgeDatabaseRefreshCacheTimeToLive') ?? 3600; + + let lastDatabaseRefresh = 0; // Check if the users have enabled the obfuscation of the entity name - const obfuscate: boolean = - options.config.getOptionalBoolean('app.badges.obfuscate') ?? false; + if (config.getOptionalBoolean('app.badges.obfuscate')) { + logger.info('Badges obfuscation is enabled'); - if (obfuscate) { - options.logger.info('Badges obfuscation is enabled'); + if (isNil(salt)) { + throw new Error( + 'Badges obfuscation is enabled but no salt has been provided', + ); + } // Use the generated hash instead of the triplet namespace/kind/name router.get('/entity/:entityHash/badge-specs', async (req, res) => { const { entityHash } = req.params; - if (isLookupTableToRefresh(lookupTable)) { - lookupTable = await generateHashLookupTable(); + // Chech if the database needs to be refreshed + if (await isBadgeDatabaseRefreshNeeded(lastDatabaseRefresh)) { + lastDatabaseRefresh = await refreshBadgeDatabase(); + } + + // Retrieve the badge info from the database + const badgeInfos = await db.getBadgeFromHash(entityHash); + + if (isNil(badgeInfos)) { + throw new NotFoundError( + `No badge found for entity hash "${entityHash}"`, + ); } - // Try to match the queried hash with a key in the lookup table - const entityInfo = await getEntityInfoFromLookupTable(entityHash); // If a mapping is found, map name, namespace and kind - const name = entityInfo.name; - const namespace = entityInfo.namespace; - const kind = entityInfo.kind; + const name = badgeInfos.name; + const namespace = badgeInfos.namespace; + const kind = badgeInfos.kind; const token = await tokenManager.getToken(); // Query the catalog with the name, namespace, kind to get the entity informations @@ -107,13 +118,8 @@ export async function createRouter( const specs = []; for (const badgeInfo of await badgeBuilder.getBadges()) { const context: BadgeContext = { - badgeUrl: await getBadgeObfuscatedUrl( - namespace, - kind, - name, - badgeInfo.id, - ), - config: options.config, + badgeUrl: await getBadgeObfuscatedUrl(entityHash, badgeInfo.id), + config: config, entity, }; @@ -129,19 +135,25 @@ export async function createRouter( // Use the generated hash instead of the triplet namespace/kind/name router.get('/entity/:entityHash/:badgeId', async (req, res) => { - const { entityHash, badgeId } = req.params; - - if (isLookupTableToRefresh(lookupTable)) { - lookupTable = await generateHashLookupTable(); + if (await isBadgeDatabaseRefreshNeeded(lastDatabaseRefresh)) { + lastDatabaseRefresh = await refreshBadgeDatabase(); } - // Try to match the queried hash with a key in the lookup table - const entityInfo = await getEntityInfoFromLookupTable(entityHash); + const { entityHash, badgeId } = req.params; + + // Retrieve the badge info from the database + const badgeInfo = await db.getBadgeFromHash(entityHash); + + if (isNil(badgeInfo)) { + throw new NotFoundError( + `No badge found for entity hash "${entityHash}"`, + ); + } // If a mapping is found, map name, namespace and kind - const name = entityInfo.name; - const namespace = entityInfo.namespace; - const kind = entityInfo.kind; + const name = badgeInfo.name; + const namespace = badgeInfo.namespace; + const kind = badgeInfo.kind; const token = await tokenManager.getToken(); const entity = await catalog.getEntityByRef( { @@ -168,8 +180,8 @@ export async function createRouter( const badgeOptions = { badgeInfo: { id: badgeId }, context: { - badgeUrl: await getBadgeObfuscatedUrl(namespace, kind, name, badgeId), - config: options.config, + badgeUrl: await getBadgeObfuscatedUrl(entityHash, badgeId), + config: config, entity, }, }; @@ -203,22 +215,42 @@ export async function createRouter( } try { - req.user = options.identity.getIdentity({ request: req }); + req.user = identity.getIdentity({ request: req }); next(); } catch (error) { - options.tokenManager.authenticate(token.toString()); + tokenManager.authenticate(token.toString()); next(error); } }, async (req, res) => { + if (await isBadgeDatabaseRefreshNeeded(lastDatabaseRefresh)) { + lastDatabaseRefresh = await refreshBadgeDatabase(); + } + const { namespace, kind, name } = req.params; - const salt = options.config.getString('custom.badges-backend.salt'); - const hash = crypto + const storedEntityHash: { hash: string } | undefined = + await db.getHashFromEntityMetadata(name, namespace, kind); + + if (isNil(storedEntityHash)) { + throw new NotFoundError( + `No hash found for entity "${namespace}/${kind}/${name}"`, + ); + } + + // Compare a live calculated hash with the stored one to avoid returning an obsolete entityHash (in case of a salt change) + + const liveHash = crypto .createHash('sha256') .update(`${kind}:${namespace}:${name}:${salt}`) .digest('hex'); - return res.status(200).json({ hash }); + if (storedEntityHash.hash !== liveHash) { + throw new NotFoundError( + "Stored Hash and live Hash don't match, did you change the salt? Try to refresh the badge database table and try again", + ); + } + + return res.status(200).json(storedEntityHash); }, ); @@ -248,7 +280,7 @@ export async function createRouter( for (const badgeInfo of await badgeBuilder.getBadges()) { const context: BadgeContext = { badgeUrl: await getBadgeUrl(namespace, kind, name, badgeInfo.id), - config: options.config, + config: config, entity, }; @@ -288,7 +320,7 @@ export async function createRouter( badgeInfo: { id: badgeId }, context: { badgeUrl: await getBadgeUrl(namespace, kind, name, badgeId), - config: options.config, + config: config, entity, }, }; @@ -314,117 +346,11 @@ export async function createRouter( return router; } - // This function generate a table that maps the hash to the namespace/kind/name triplet - async function generateHashLookupTable() { - const logger = options.logger.child({ service: 'badges-backend' }); - const generationStartTimestamp = Date.now(); - logger.info('Start Generating lookup table'); - const token = await options.tokenManager.getToken(); - - // The salt is used to increase the hash entropy - const salt = options.config.getString('custom.badges-backend.salt'); - - // Get all the entities in the catalog of kind "Component" - const entitiesList = await catalog.getEntities( - { - filter: { - kind: 'Component', - }, - }, - token, - ); - if (isEmpty(entitiesList)) { - throw new NotFoundError(`No entities found`); - } - - // For each entity, generate the hash with the triplet kind:namespace:name then add the salt. Finally add it to the lookup table - entitiesList.items.map(async entity => { - const name = entity.metadata.name.toLocaleLowerCase(); - const creationDate: number = Date.now(); - const namespace = - entity.metadata.namespace?.toLocaleLowerCase() ?? 'default'; - const kind = entity.kind.toLocaleLowerCase(); - const hash = crypto - .createHash('sha256') - .update(`${kind}:${namespace}:${name}:${salt}`) - .digest('hex'); - - lookupTable.set(hash, { name, namespace, kind, creationDate }); - - return lookupTable; - }); - - // Monitor the lookup generation time and log it - logger.info( - `Finished generating lookup table in ${ - Date.now() - generationStartTimestamp - } ms`, - ); - // Monitor the lookup table size in byte - logger.info(`Lookup table size: ${lookupTable.size} entries`); - - return lookupTable; - } - - // This function check if the lookup table is empty or if it is expired (based on the cacheTimeToLive config) and need to be refreshed - function isLookupTableToRefresh( - table: Map< - string, - { - name: string; - namespace: string; - kind: string; - creationDate: number; - } - >, - ): boolean { - const lookupTableLength = table.size; - - if (lookupTableLength === 0) { - return true; - } - - const now = Date.now(); - const cacheTimeToLive = - options.config.getOptionalNumber( - 'custom.badges-backend.cacheTimeToLive', - ) ?? 10 * 60 * 1000; - - const valuesArray = Array.from(table.values()); - const lastEntry = valuesArray[valuesArray.length - 1]; - const lastCreationDate = lastEntry.creationDate; - - return lastCreationDate + cacheTimeToLive < now; - } - - // This function return the namespace/kind/name triplet from the lookup table based on the hash - async function getEntityInfoFromLookupTable( - entityHash: string, - ): Promise<{ name: string; namespace: string; kind: string }> { - if (lookupTable.get(entityHash) === undefined) { - throw new NotFoundError(`No entity with hash "${entityHash}"`); - } - - const name = lookupTable.get(entityHash)!.name; - const namespace = lookupTable.get(entityHash)!.namespace; - const kind = lookupTable.get(entityHash)!.kind; - - return { name, namespace, kind }; - } - // This function return the obfuscated badge url based on the namespace/kind/name triplet async function getBadgeObfuscatedUrl( - namespace: string, - kind: string, - name: string, + hash: string, badgeId: string, ): Promise { - const baseUrl = await options.discovery.getExternalBaseUrl('badges'); - const salt = options.config.getString('custom.badges-backend.salt'); - const hash = crypto - .createHash('sha256') - .update(`${kind}:${namespace}:${name}:${salt}`) - .digest('hex'); return `${baseUrl}/entity/${hash}/${badgeId}`; } @@ -435,7 +361,47 @@ export async function createRouter( name: string, badgeId: string, ): Promise { - const baseUrl = await options.discovery.getExternalBaseUrl('badges'); return `${baseUrl}/entity/${namespace}/${kind}/${name}/badge/${badgeId}`; } + + async function isBadgeDatabaseRefreshNeeded( + lastDatabaseRefreshTimestamp: number, + ): Promise { + if ((await db.countAllBadges()) === 0) { + logger.info('Badge database is empty, refreshing it'); + return true; + } + + if ( + lastDatabaseRefreshTimestamp === 0 || + Date.now() - lastDatabaseRefreshTimestamp > cacheTimeToLive * 1000 + ) { + logger.info( + 'Badge database refresh cache to live exceeded, refreshing it', + ); + return true; + } + + return false; + } + + async function refreshBadgeDatabase(): Promise { + const token = await tokenManager.getToken(); + const entities = await catalog.getEntities( + { + filter: { + kind: ['Component'], + }, + }, + token, + ); + logger.info( + `Refreshing badge database with ${entities.items.length} entities`, + ); + await db.createAllBadges(entities, salt); + logger.info('Badge database refreshed, deleting obsolete hashes'); + await db.deleteObsoleteHashes(entities, salt); + + return Date.now(); + } } diff --git a/plugins/badges-backend/src/service/standaloneServer.ts b/plugins/badges-backend/src/service/standaloneServer.ts index 9e29d5dead..5def5d6499 100644 --- a/plugins/badges-backend/src/service/standaloneServer.ts +++ b/plugins/badges-backend/src/service/standaloneServer.ts @@ -21,9 +21,12 @@ import { loadBackendConfig, ServerTokenManager, SingleHostDiscovery, + useHotMemoize, } from '@backstage/backend-common'; import { createRouter } from './router'; import { IdentityApi } from '@backstage/plugin-auth-node'; +import { DatabaseBadgesStore } from '../database/badgesStore'; +import Knex from 'knex'; export interface ServerOptions { port: number; @@ -37,6 +40,13 @@ export async function startStandaloneServer( const logger = options.logger.child({ service: 'badges-backend' }); const config = await loadBackendConfig({ logger, argv: process.argv }); const discovery = SingleHostDiscovery.fromConfig(config); + const database = useHotMemoize(module, () => { + return Knex({ + client: 'better-sqlite3', + connection: ':memory:', + useNullAsDefault: true, + }); + }); logger.debug('Creating application...'); @@ -60,6 +70,9 @@ export async function startStandaloneServer( tokenManager, logger, identity, + db: await DatabaseBadgesStore.create({ + database: { getClient: async () => database }, + }), }); let service = createServiceBuilder(module) diff --git a/plugins/badges-backend/src/tests/badgeStore.test.ts b/plugins/badges-backend/src/tests/badgeStore.test.ts new file mode 100644 index 0000000000..1c6a76cb38 --- /dev/null +++ b/plugins/badges-backend/src/tests/badgeStore.test.ts @@ -0,0 +1,294 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { DatabaseBadgesStore } from '../database/badgesStore'; +import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; +import { Knex } from 'knex'; +import { Entity } from '@backstage/catalog-model'; +import { ConfigReader } from '@backstage/config'; +import { GetEntitiesResponse } from '@backstage/catalog-client'; +import crypto from 'crypto'; + +describe('DatabaseBadgesStore', () => { + const config = new ConfigReader({ + backend: { + baseUrl: 'http://127.0.0.1', + listen: { + port: 7007, + }, + }, + app: { + badges: { + obfuscate: true, + }, + }, + custom: { + 'badges-backend': { + salt: 'random-string', + cacheTimeToLive: '60', + }, + }, + }); + + const entity: Entity = { + apiVersion: 'v1', + kind: 'Component', + metadata: { + name: 'test', + }, + }; + + const entities: Entity[] = [ + entity, + { + apiVersion: 'v1', + kind: 'Component', + metadata: { + name: 'test-2', + }, + }, + { + apiVersion: 'v1', + kind: 'Component', + metadata: { + name: 'test-3', + }, + }, + ]; + + const defaultEntityListResponse: GetEntitiesResponse = { + items: entities, + }; + + const databases = TestDatabases.create(); + + async function createDatabaseBadgesStore(databaseId: TestDatabaseId) { + const knex = await databases.init(databaseId); + return { + knex, + badgeStore: await DatabaseBadgesStore.create({ + database: { getClient: async () => knex }, + }), + }; + } + + describe.each(databases.eachSupportedId())('%p', databaseId => { + let knex: Knex; + let badgeStore: DatabaseBadgesStore; + + beforeEach(async () => { + ({ knex, badgeStore } = await createDatabaseBadgesStore(databaseId)); + }); + + it('createAllBadges', async () => { + await badgeStore.createAllBadges( + defaultEntityListResponse, + config.getString('custom.badges-backend.salt'), + ); + + const storedBadges = await badgeStore.getAllBadges(); + const testedEntities: { + name: string; + namespace: string; + kind: string; + hash: string; + }[] = []; + + entities.forEach(entityInLoop => { + const kind = entityInLoop.kind.toLowerCase(); + const namespace = entityInLoop.metadata.namespace || 'default'; + const name = entityInLoop.metadata.name.toLocaleLowerCase(); + const salt = config.getString('custom.badges-backend.salt'); + const entityHash = crypto + .createHash('sha256') + .update(`${kind}:${namespace}:${name}:${salt}`) + .digest('hex'); + + testedEntities.push({ + hash: entityHash, + name: name, + namespace: namespace, + kind: kind, + }); + expect(storedBadges).toEqual(expect.arrayContaining(testedEntities)); + expect(storedBadges.length).toEqual(3); + }); + }); + + it('getBadgeFromHash', async () => { + await knex('badges').truncate(); + await knex('badges').insert([ + { + hash: 'hash1', + name: 'test', + namespace: 'default', + kind: 'component', + }, + ]); + + const storedEntity = await badgeStore.getBadgeFromHash('hash1'); + + expect(storedEntity).toEqual({ + name: 'test', + namespace: 'default', + kind: 'component', + }); + }); + + it('getHashFromEntityMetadata', async () => { + await knex('badges').truncate(); + await knex('badges').insert([ + { + hash: 'hash1', + name: 'test', + namespace: 'default', + kind: 'component', + }, + ]); + + const storedHash = await badgeStore.getHashFromEntityMetadata( + 'test', + 'default', + 'component', + ); + + expect(storedHash).toEqual({ + hash: 'hash1', + }); + }); + + it('deleteObsoleteHashes', async () => { + const entityHash = crypto + .createHash('sha256') + .update( + `component:default:test:${config.getString( + 'custom.badges-backend.salt', + )}`, + ) + .digest('hex'); + + await knex('badges').insert([ + { + hash: 'obsoleteHash', + name: 'obsoleteEntity', + namespace: 'default', + kind: 'component', + }, + { + hash: entityHash, + name: 'test', + namespace: 'default', + kind: 'component', + }, + ]); + + await badgeStore.deleteObsoleteHashes( + defaultEntityListResponse, + config.getString('custom.badges-backend.salt'), + ); + + const storedBadges = await badgeStore.getAllBadges(); + + expect(storedBadges).toEqual( + expect.not.arrayContaining([ + { + hash: 'obsoleteHash', + name: 'obsoleteEntity', + namespace: 'default', + kind: 'component', + }, + ]), + ); + expect(storedBadges.length).toEqual(1); + expect(storedBadges).toEqual( + expect.arrayContaining([ + { + hash: entityHash, + name: 'test', + namespace: 'default', + kind: 'component', + }, + ]), + ); + }); + + it('countAllBadges', async () => { + await knex('badges').truncate(); + await knex('badges').insert([ + { + hash: 'hash1', + name: 'test', + namespace: 'default', + kind: 'component', + }, + { + hash: 'hash2', + name: 'test', + namespace: 'default', + kind: 'component', + }, + { + hash: 'hash3', + name: 'test', + namespace: 'default', + kind: 'component', + }, + { + hash: 'hash4', + name: 'test', + namespace: 'default', + kind: 'component', + }, + ]); + + const storedBadgesCount = await badgeStore.countAllBadges(); + expect(storedBadgesCount).toEqual(4); + }); + + it('getAllBadges', async () => { + await knex('badges').truncate(); + await knex('badges').insert([ + { + hash: 'hash1', + name: 'test', + namespace: 'default', + kind: 'component', + }, + { + hash: 'hash2', + name: 'test', + namespace: 'default', + kind: 'component', + }, + { + hash: 'hash3', + name: 'test', + namespace: 'default', + kind: 'component', + }, + { + hash: 'hash4', + name: 'test', + namespace: 'default', + kind: 'component', + }, + ]); + + const storedBadgesCount = await badgeStore.countAllBadges(); + expect(storedBadgesCount).toEqual(4); + }); + }); +}); diff --git a/plugins/badges-backend/src/tests/router-obfuscated.test.ts b/plugins/badges-backend/src/tests/router-obfuscated.test.ts index d03cbe6d24..edf19d9f52 100644 --- a/plugins/badges-backend/src/tests/router-obfuscated.test.ts +++ b/plugins/badges-backend/src/tests/router-obfuscated.test.ts @@ -31,10 +31,16 @@ import { BackstageIdentityResponse, IdentityApiGetIdentityRequest, } from '@backstage/plugin-auth-node'; +import { BadgesStore } from '../database/badgesStore'; +import crypto from 'crypto'; describe('createRouter', () => { let app: express.Express; - let badgeBuilder: jest.Mocked; + const badgeBuilder: jest.Mocked = { + getBadges: jest.fn(), + createBadgeJson: jest.fn(), + createBadgeSvg: jest.fn(), + }; const catalog = { addLocation: jest.fn(), @@ -49,10 +55,46 @@ describe('createRouter', () => { getEntityFacets: jest.fn(), validateEntity: jest.fn(), }; - let config: Config; - let discovery: PluginEndpointDiscovery; + const getIdentity = jest + .fn() + .mockImplementation( + async ({ + request: _request, + }: IdentityApiGetIdentityRequest): Promise< + BackstageIdentityResponse | undefined + > => { + return { + identity: { + userEntityRef: 'user:default/guest', + ownershipEntityRefs: [], + type: 'user', + }, + token: 'token', + }; + }, + ); - const getIdentity = jest.fn(); + const config: Config = new ConfigReader({ + backend: { + baseUrl: 'http://127.0.0.1', + listen: { + port: 7007, + }, + }, + app: { + badges: { + obfuscate: true, + }, + }, + custom: { + 'badges-backend': { + salt: 'random-string', + cacheTimeToLive: '60', + }, + }, + }); + + let discovery: PluginEndpointDiscovery; const entity: Entity = { apiVersion: 'v1', @@ -83,49 +125,47 @@ describe('createRouter', () => { markdown: '[![...](...)]', }; + const badgeEntity = { + name: 'test', + namespace: 'default', + kind: 'component', + }; + + const badgeEntities = [ + { + hash: 'hash1', + name: 'test', + namespace: 'default', + kind: 'component', + }, + { + hash: 'hash2', + name: 'test2', + namespace: 'default', + kind: 'component', + }, + ]; + + const badgeStore: jest.Mocked = { + createAllBadges: jest.fn(), + getBadgeFromHash: jest.fn().mockImplementation(async () => { + return badgeEntity; + }), + getHashFromEntityMetadata: jest + .fn() + .mockImplementation(async () => 'niceHash'), + deleteObsoleteHashes: jest.fn(), + countAllBadges: jest.fn().mockImplementation(async () => 4), + getAllBadges: jest.fn().mockImplementation(async () => badgeEntities), + }; + + const salt = config.getString('custom.badges-backend.salt'); + const entityHash = crypto + .createHash('sha256') + .update(`component:default:test:${salt}`) + .digest('hex'); + beforeAll(async () => { - getIdentity.mockImplementation( - async ({ - request: _request, - }: IdentityApiGetIdentityRequest): Promise< - BackstageIdentityResponse | undefined - > => { - return { - identity: { - userEntityRef: 'user:default/guest', - ownershipEntityRefs: [], - type: 'user', - }, - token: 'token', - }; - }, - ); - - badgeBuilder = { - getBadges: jest.fn(), - createBadgeJson: jest.fn(), - createBadgeSvg: jest.fn(), - }; - config = new ConfigReader({ - backend: { - baseUrl: 'http://127.0.0.1', - listen: { - port: 7007, - }, - }, - app: { - badges: { - obfuscate: true, - }, - }, - custom: { - 'badges-backend': { - salt: 'random-string', - cacheTimeToLive: '60', - }, - }, - }); - discovery = SingleHostDiscovery.fromConfig(config); const tokenManager = ServerTokenManager.noop(); const router = await createRouter({ @@ -136,12 +176,13 @@ describe('createRouter', () => { tokenManager, logger: getVoidLogger(), identity: { getIdentity }, + db: badgeStore, }); app = express().use(router); }); beforeEach(() => { - jest.resetAllMocks(); + jest.clearAllMocks(); }); it('works', async () => { @@ -154,6 +195,7 @@ describe('createRouter', () => { tokenManager, logger: getVoidLogger(), identity: { getIdentity }, + db: badgeStore, }); expect(router).toBeDefined(); }); @@ -206,9 +248,7 @@ describe('createRouter', () => { badgeBuilder.getBadges.mockResolvedValueOnce([{ id: badge.id }]); badgeBuilder.createBadgeJson.mockResolvedValueOnce(badge); - const response = await request(app).get( - '/entity/3a5f91c1e66519be5394c37a8ba69c3087b7c322c600e7497dc9d517353e5bed/badge-specs', - ); + const response = await request(app).get(`/entity/hash1/badge-specs`); expect(response.status).toEqual(200); expect(response.body).toEqual([badge]); @@ -228,7 +268,7 @@ describe('createRouter', () => { badgeInfo: { id: badge.id }, context: { badgeUrl: expect.stringMatching( - /http:\/\/127.0.0.1\/api\/badges\/entity\/3a5f91c1e66519be5394c37a8ba69c3087b7c322c600e7497dc9d517353e5bed\/test-badge/, + /http:\/\/127.0.0.1\/api\/badges\/entity\/hash1\/test-badge/, ), config, entity, @@ -293,6 +333,9 @@ describe('createRouter', () => { describe('GET /entity/:namespace/:kind/:name/obfuscated', () => { catalog.getEntityByRef.mockResolvedValueOnce(entity); catalog.getEntities.mockResolvedValueOnce({ items: entities }); + badgeStore.getHashFromEntityMetadata.mockResolvedValue({ + hash: entityHash, + }); it('returns obfuscated entity', async () => { const obfuscatedEntity = await request(app) @@ -301,7 +344,7 @@ describe('createRouter', () => { expect(obfuscatedEntity.status).toEqual(200); // echo -n "component:default:test:random-string" | openssl dgst -sha256 expect(obfuscatedEntity.body).toEqual({ - hash: '3a5f91c1e66519be5394c37a8ba69c3087b7c322c600e7497dc9d517353e5bed', + hash: entityHash, }); }); @@ -315,6 +358,7 @@ describe('createRouter', () => { describe('Errors', () => { it('returns 404 for unknown entity hash', async () => { + badgeStore.getBadgeFromHash.mockResolvedValue(undefined); catalog.getEntityByRef.mockResolvedValueOnce(entity); catalog.getEntities.mockResolvedValueOnce({ items: entities }); badgeBuilder.getBadges.mockResolvedValueOnce([{ id: badge.id }]); diff --git a/plugins/badges-backend/src/tests/router.test.ts b/plugins/badges-backend/src/tests/router.test.ts index 1388b043d0..1d5fb3e9a1 100644 --- a/plugins/badges-backend/src/tests/router.test.ts +++ b/plugins/badges-backend/src/tests/router.test.ts @@ -31,6 +31,7 @@ import { BackstageIdentityResponse, IdentityApiGetIdentityRequest, } from '@backstage/plugin-auth-node'; +import { BadgesStore } from '../database/badgesStore'; describe('createRouter', () => { let app: express.Express; @@ -73,6 +74,15 @@ describe('createRouter', () => { markdown: '[![...](...)]', }; + const badgeStore: jest.Mocked = { + createAllBadges: jest.fn(), + getBadgeFromHash: jest.fn(), + getHashFromEntityMetadata: jest.fn(), + deleteObsoleteHashes: jest.fn(), + countAllBadges: jest.fn(), + getAllBadges: jest.fn(), + }; + beforeAll(async () => { badgeBuilder = { getBadges: jest.fn(), @@ -113,6 +123,7 @@ describe('createRouter', () => { tokenManager, logger: getVoidLogger(), identity: { getIdentity }, + db: badgeStore, }); app = express().use(router); }); @@ -131,6 +142,7 @@ describe('createRouter', () => { tokenManager, logger: getVoidLogger(), identity: { getIdentity }, + db: badgeStore, }); expect(router).toBeDefined(); }); diff --git a/plugins/badges/package.json b/plugins/badges/package.json index 0bed554beb..cd04e4db97 100644 --- a/plugins/badges/package.json +++ b/plugins/badges/package.json @@ -50,14 +50,9 @@ "@backstage/core-app-api": "workspace:^", "@backstage/dev-utils": "workspace:^", "@backstage/test-utils": "workspace:^", - "@testing-library/dom": "^8.0.0", - "@testing-library/jest-dom": "^5.10.1", - "@testing-library/react": "^12.1.3", - "@testing-library/user-event": "^14.0.0", + "@testing-library/jest-dom": "^5.16.5", "@types/node": "^16.11.26", - "@types/react": "^16.13.1 || ^17.0.0", - "cross-fetch": "^3.1.5", - "msw": "^1.0.0" + "cross-fetch": "^3.1.5" }, "files": [ "dist" diff --git a/plugins/badges/src/api/BadgesClient.ts b/plugins/badges/src/api/BadgesClient.ts index 5071b30437..db5019d893 100644 --- a/plugins/badges/src/api/BadgesClient.ts +++ b/plugins/badges/src/api/BadgesClient.ts @@ -18,30 +18,26 @@ import { generatePath } from 'react-router-dom'; import { ResponseError } from '@backstage/errors'; import { Entity, DEFAULT_NAMESPACE } from '@backstage/catalog-model'; import { BadgesApi, BadgeSpec } from './types'; -import { - ConfigApi, - DiscoveryApi, - IdentityApi, -} from '@backstage/core-plugin-api'; +import { ConfigApi, DiscoveryApi, FetchApi } from '@backstage/core-plugin-api'; export class BadgesClient implements BadgesApi { private readonly discoveryApi: DiscoveryApi; - private readonly identityApi: IdentityApi; + private readonly fetchApi: FetchApi; private readonly configApi: ConfigApi; constructor(options: { discoveryApi: DiscoveryApi; - identityApi: IdentityApi; + fetchApi: FetchApi; configApi: ConfigApi; }) { this.discoveryApi = options.discoveryApi; - this.identityApi = options.identityApi; + this.fetchApi = options.fetchApi; this.configApi = options.configApi; } static fromConfig(options: { + fetchApi: FetchApi; discoveryApi: DiscoveryApi; - identityApi: IdentityApi; configApi: ConfigApi; }) { return new BadgesClient(options); @@ -49,9 +45,7 @@ export class BadgesClient implements BadgesApi { public async getEntityBadgeSpecs(entity: Entity): Promise { // Check if obfuscation is enabled in the configuration - const obfuscate = - this.configApi.getOptionalBoolean('app.badges.obfuscate') ?? false; - const { token } = await this.identityApi.getCredentials(); + const obfuscate = this.configApi.getOptionalBoolean('app.badges.obfuscate'); // If obfuscation is enabled, get the hash of the entity and use that to get the badge specs if (obfuscate) { @@ -63,13 +57,7 @@ export class BadgesClient implements BadgesApi { entityHash, ); - const response = await fetch(entityHashedBadgeSpecsUrl, { - headers: token - ? { - Authorization: `Bearer ${token}`, - } - : undefined, - }); + const response = await this.fetchApi.fetch(entityHashedBadgeSpecsUrl); if (!response.ok) { throw await ResponseError.fromResponse(response); } @@ -79,13 +67,7 @@ export class BadgesClient implements BadgesApi { // If obfuscation is disabled, get the badge specs directly as the previous implementation const entityBadgeSpecsUrl = await this.getEntityBadgeSpecsUrl(entity); - const response = await fetch(entityBadgeSpecsUrl, { - headers: token - ? { - Authorization: `Bearer ${token}`, - } - : undefined, - }); + const response = await this.fetchApi.fetch(entityBadgeSpecsUrl); if (!response.ok) { throw await ResponseError.fromResponse(response); } @@ -105,15 +87,7 @@ export class BadgesClient implements BadgesApi { // This function is used to get the hash of an entity when obfuscation is enabled. It's calling the badges backend to get the hash private async getEntityHash(entityHashUrl: string): Promise { - const { token: idToken } = await this.identityApi.getCredentials(); - - const responseEntityHash = await fetch(entityHashUrl, { - headers: idToken - ? { - Authorization: `Bearer ${idToken}`, - } - : undefined, - }); + const responseEntityHash = await this.fetchApi.fetch(entityHashUrl); if (!responseEntityHash.ok) { throw await ResponseError.fromResponse(responseEntityHash); diff --git a/plugins/badges/src/plugin.ts b/plugins/badges/src/plugin.ts index eac5ee11ad..771e3ed893 100644 --- a/plugins/badges/src/plugin.ts +++ b/plugins/badges/src/plugin.ts @@ -19,8 +19,8 @@ import { createApiFactory, createComponentExtension, createPlugin, + fetchApiRef, discoveryApiRef, - identityApiRef, } from '@backstage/core-plugin-api'; /** @public */ @@ -30,14 +30,14 @@ export const badgesPlugin = createPlugin({ createApiFactory({ api: badgesApiRef, deps: { + fetchApi: fetchApiRef, discoveryApi: discoveryApiRef, - identityApi: identityApiRef, configApi: configApiRef, }, - factory: ({ discoveryApi, identityApi, configApi }) => + factory: ({ fetchApi, discoveryApi, configApi }) => new BadgesClient({ + fetchApi, discoveryApi, - identityApi, configApi, }), }), diff --git a/yarn.lock b/yarn.lock index 213f03d6b1..899440e09e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4933,6 +4933,7 @@ __metadata: resolution: "@backstage/plugin-badges-backend@workspace:plugins/badges-backend" dependencies: "@backstage/backend-common": "workspace:^" + "@backstage/backend-test-utils": "workspace:^" "@backstage/catalog-client": "workspace:^" "@backstage/catalog-model": "workspace:^" "@backstage/cli": "workspace:^" @@ -4942,9 +4943,6 @@ __metadata: "@backstage/errors": "workspace:^" "@backstage/plugin-auth-node": "workspace:^" "@backstage/test-utils": "workspace:^" - "@testing-library/jest-dom": ^5.10.1 - "@testing-library/react": ^12.1.3 - "@testing-library/user-event": ^14.0.0 "@types/express": ^4.17.6 "@types/node": "*" badge-maker: ^3.3.0 @@ -4952,8 +4950,8 @@ __metadata: cross-fetch: ^3.1.5 express: ^4.17.1 express-promise-router: ^4.1.0 + knex: ^2.4.2 lodash: ^4.17.21 - msw: ^0.49.0 supertest: ^6.3.3 winston: ^3.2.1 yn: ^4.0.0 @@ -4984,7 +4982,6 @@ __metadata: "@types/node": ^16.11.26 "@types/react": ^16.13.1 || ^17.0.0 cross-fetch: ^3.1.5 - msw: ^1.0.0 react-use: ^17.2.4 peerDependencies: react: ^16.13.1 || ^17.0.0 @@ -15026,7 +15023,7 @@ __metadata: languageName: node linkType: hard -"@testing-library/jest-dom@npm:^5.10.1, @testing-library/jest-dom@npm:^5.16.4": +"@testing-library/jest-dom@npm:^5.10.1, @testing-library/jest-dom@npm:^5.16.4, @testing-library/jest-dom@npm:^5.16.5": version: 5.16.5 resolution: "@testing-library/jest-dom@npm:5.16.5" dependencies: @@ -28987,7 +28984,7 @@ __metadata: languageName: node linkType: hard -"knex@npm:^2.0.0": +"knex@npm:^2.0.0, knex@npm:^2.4.2": version: 2.4.2 resolution: "knex@npm:2.4.2" dependencies: