diff --git a/plugins/app-backend/migrations/20211229105307_init.js b/plugins/app-backend/migrations/20211229105307_init.js index e44e58f539..29e37b35f7 100644 --- a/plugins/app-backend/migrations/20211229105307_init.js +++ b/plugins/app-backend/migrations/20211229105307_init.js @@ -20,22 +20,20 @@ * @param {import('knex').Knex} knex */ exports.up = async function up(knex) { - return knex.schema.createTable('static_asset_cache', table => { + return knex.schema.createTable('static_assets_cache', table => { table.comment( 'A cache of static assets that where previously deployed and may still be lazy-loaded by clients', ); - table - .string('path') - .primary() - .notNullable() - .comment('The path of the file'); + table.text('path').primary().notNullable().comment('The path of the file'); table .dateTime('last_modified_at') + .defaultTo(knex.fn.now()) .notNullable() .comment( 'Timestamp of when the asset was most recently seen in a deployment', ); table.binary('content').notNullable().comment('The asset content'); + table.index('last_modified_at', 'static_asset_cache_last_modified_at_idx'); }); }; @@ -43,5 +41,8 @@ exports.up = async function up(knex) { * @param {import('knex').Knex} knex */ exports.down = async function down(knex) { - return knex.schema.dropTable('static_asset_cache'); + await knex.schema.alterTable('static_assets_cache', table => { + table.dropIndex([], 'static_asset_cache_last_modified_at_idx'); + }); + await knex.schema.dropTable('static_assets_cache'); }; diff --git a/plugins/app-backend/package.json b/plugins/app-backend/package.json index e8cd4a4135..77b64b96d1 100644 --- a/plugins/app-backend/package.json +++ b/plugins/app-backend/package.json @@ -40,10 +40,13 @@ "fs-extra": "9.1.0", "helmet": "^4.0.0", "knex": "^0.95.1", + "lodash": "^4.17.21", + "luxon": "^2.0.2", "winston": "^3.2.1", "yn": "^4.0.0" }, "devDependencies": { + "@backstage/backend-test-utils": "^0.1.13", "@backstage/cli": "^0.11.0", "@backstage/types": "^0.1.1", "@types/supertest": "^2.0.8", diff --git a/plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts b/plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts new file mode 100644 index 0000000000..0230486444 --- /dev/null +++ b/plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts @@ -0,0 +1,160 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { TestDatabases } from '@backstage/backend-test-utils'; +import { StaticAssetsStore } from './StaticAssetsStore'; + +const logger = getVoidLogger(); + +describe('StaticAssetsStore', () => { + const databases = TestDatabases.create({ + ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + }); + + it.each(databases.eachSupportedId())( + 'should store and retrieve assets, %p', + async databaseId => { + const store = await StaticAssetsStore.create({ + logger, + database: await databases.init(databaseId), + }); + + await store.storeAssets([ + { + path: 'foo.txt', + content: async () => Buffer.from('foo'), + }, + { + path: 'dir/bar.txt', + content: async () => Buffer.from('bar'), + }, + ]); + + const now = new Date().getTime(); + + const foo = await store.getAsset('foo.txt'); + expect(foo!.path).toBe('foo.txt'); + expect(foo!.lastModifiedAt.getTime()).toBeGreaterThan(now - 5000); + expect(foo!.lastModifiedAt.getTime()).toBeLessThan(now + 5000); + expect(foo!.content).toEqual(Buffer.from('foo')); + + const bar = await store.getAsset('dir/bar.txt'); + expect(bar!.path).toBe('dir/bar.txt'); + expect( + Math.abs(bar!.lastModifiedAt.getTime() - foo!.lastModifiedAt.getTime()), + ).toBeLessThan(1000); + expect(bar!.content).toEqual(Buffer.from('bar')); + + await expect( + store.getAsset('does-not-exist.txt'), + ).resolves.toBeUndefined(); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'should update assets timestamps, but not contents, %p', + async databaseId => { + const store = await StaticAssetsStore.create({ + logger, + database: await databases.init(databaseId), + }); + + await store.storeAssets([ + { + path: 'foo', + content: async () => Buffer.from('foo'), + }, + { + path: 'bar', + content: async () => Buffer.from('bar'), + }, + ]); + + const oldFoo = await store.getAsset('foo'); + expect(oldFoo?.lastModifiedAt).toBeDefined(); + + const oldBar = await store.getAsset('bar'); + expect(oldBar?.lastModifiedAt).toBeDefined(); + + // SQLite dates end up with second precision, so make sure we wait at least 1s + await new Promise(resolve => setTimeout(resolve, 1500)); + + await store.storeAssets([ + { + path: 'foo', + content: async () => Buffer.from('newFoo'), + }, + ]); + + const newFoo = await store.getAsset('foo'); + expect(oldFoo!.lastModifiedAt).not.toEqual(newFoo!.lastModifiedAt); + expect(oldFoo!.lastModifiedAt.getTime()).toBeLessThan( + newFoo!.lastModifiedAt.getTime(), + ); + + // The "static" in "StaticAssetsStore" means that assets aren't allowed to change + expect(newFoo!.content).toEqual(Buffer.from('foo')); + + const sameBar = await store.getAsset('bar'); + expect(oldBar!.lastModifiedAt).toEqual(sameBar!.lastModifiedAt); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'should trim old assets, %p', + async databaseId => { + const database = await databases.init(databaseId); + const store = await StaticAssetsStore.create({ + logger, + database, + }); + + await store.storeAssets([ + { + path: 'new', + content: async () => Buffer.alloc(0), + }, + { + path: 'old', + content: async () => Buffer.alloc(0), + }, + ]); + + // Rewrite modified time of "old" to be 1h in the past + const updated = await database('static_assets_cache') + .where({ path: 'old' }) + .update({ + last_modified_at: + database.client.config.client === 'sqlite3' + ? database.raw(`datetime('now', '-3600 seconds')`) + : database.raw(`now() + interval '-3600 seconds'`), + }); + expect(updated).toBe(1); + + await expect(store.getAsset('new')).resolves.toBeDefined(); + await expect(store.getAsset('old')).resolves.toBeDefined(); + + await store.trimAssets({ maxAgeSeconds: 1800 }); + + await expect(store.getAsset('new')).resolves.toBeDefined(); + await expect(store.getAsset('old')).resolves.toBeUndefined(); + }, + 60_000, + ); +}); diff --git a/plugins/app-backend/src/lib/assets/StaticAssetsStore.ts b/plugins/app-backend/src/lib/assets/StaticAssetsStore.ts new file mode 100644 index 0000000000..7516da7116 --- /dev/null +++ b/plugins/app-backend/src/lib/assets/StaticAssetsStore.ts @@ -0,0 +1,152 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { resolvePackagePath } from '@backstage/backend-common'; +import { Knex } from 'knex'; +import { Logger } from 'winston'; +import { DateTime } from 'luxon'; +import partition from 'lodash/partition'; + +const migrationsDir = resolvePackagePath( + '@backstage/plugin-app-backend', + 'migrations', +); + +export interface StaticAssetInput { + path: string; + content(): Promise; +} + +export interface StaticAsset { + path: string; + content: Buffer; + lastModifiedAt: Date; +} + +interface StaticAssetRow { + path: string; + content: Buffer; + last_modified_at: Date; +} + +/** @internal */ +export interface StaticAssetsStoreOptions { + database: Knex; + logger: Logger; +} + +/** + * A storage for static assets that are assumed to be immutable. + * + * @internal + */ +export class StaticAssetsStore { + #db: Knex; + #logger: Logger; + + static async create(options: StaticAssetsStoreOptions) { + await options.database.migrate.latest({ + directory: migrationsDir, + }); + return new StaticAssetsStore(options); + } + + private constructor(options: StaticAssetsStoreOptions) { + this.#db = options.database; + this.#logger = options.logger; + } + + /** + * Store the provided assets. + * + * If an asset for a given path already exists the modification time will be + * updated, but the contents will not. + */ + async storeAssets(assets: StaticAssetInput[]) { + const existingRows = await this.#db( + 'static_assets_cache', + ).whereIn( + 'path', + assets.map(a => a.path), + ); + const existingAssetPaths = new Set(existingRows.map(r => r.path)); + + const [modified, added] = partition(assets, asset => + existingAssetPaths.has(asset.path), + ); + + this.#logger.info( + `Storing ${modified.length} updated assets and ${added.length} new assets`, + ); + + await this.#db('static_assets_cache') + .update({ + last_modified_at: this.#db.fn.now(), + }) + .whereIn( + 'path', + modified.map(a => a.path), + ); + + for (const asset of added) { + // We ignore conflicts with other nodes, it doesn't matter if someone else + // added the same asset just before us. + await this.#db('static_assets_cache') + .insert({ + path: asset.path, + content: await asset.content(), + }) + .onConflict('path') + .ignore(); + } + } + + /** + * Retrieve an asset from the store with the given path. + */ + async getAsset(path: string): Promise { + const [row] = await this.#db('static_assets_cache').where({ + path, + }); + if (!row) { + return undefined; + } + return { + path: row.path, + content: row.content, + lastModifiedAt: + typeof row.last_modified_at === 'string' + ? DateTime.fromSQL(row.last_modified_at, { zone: 'UTC' }).toJSDate() + : row.last_modified_at, + }; + } + + /** + * Delete any assets from the store whose modification time is older than the max age. + */ + async trimAssets(options: { maxAgeSeconds: number }) { + const { maxAgeSeconds } = options; + await this.#db('static_assets_cache') + .where( + 'last_modified_at', + '<=', + this.#db.client.config.client === 'sqlite3' + ? this.#db.raw(`datetime('now', ?)`, [`-${maxAgeSeconds} seconds`]) + : this.#db.raw(`now() + interval '${-maxAgeSeconds} seconds'`), + ) + .delete(); + } +} diff --git a/plugins/app-backend/src/lib/assets/index.ts b/plugins/app-backend/src/lib/assets/index.ts new file mode 100644 index 0000000000..0e1e495872 --- /dev/null +++ b/plugins/app-backend/src/lib/assets/index.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { StaticAssetsStore } from './StaticAssetsStore'; +export type { StaticAsset, StaticAssetInput } from './StaticAssetsStore';