diff --git a/.changeset/sixty-laws-laugh.md b/.changeset/sixty-laws-laugh.md new file mode 100644 index 0000000000..874859da4e --- /dev/null +++ b/.changeset/sixty-laws-laugh.md @@ -0,0 +1,22 @@ +--- +'@backstage/create-app': patch +--- + +Updated the configuration of the `app-backend` plugin to enable the static asset store by passing on `database` from the plugin environment to `createRouter`. + +To apply this change to an existing app, make the following change to `packages/backend/src/plugins/app.ts`: + +```diff + export default async function createPlugin({ + logger, + config, ++ database, + }: PluginEnvironment): Promise { + return await createRouter({ + logger, + config, ++ database, + appPackageName: 'app', + }); + } +``` diff --git a/.changeset/wild-pugs-call.md b/.changeset/wild-pugs-call.md new file mode 100644 index 0000000000..6bcc951ac7 --- /dev/null +++ b/.changeset/wild-pugs-call.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-app-backend': patch +--- + +Updated the cache control headers for static assets to instruct clients to cache them for 14 days. diff --git a/.changeset/witty-avocados-bow.md b/.changeset/witty-avocados-bow.md new file mode 100644 index 0000000000..406088a7dd --- /dev/null +++ b/.changeset/witty-avocados-bow.md @@ -0,0 +1,7 @@ +--- +'@backstage/plugin-app-backend': patch +--- + +Added a new asset cache that stores static assets from previous deployments in the database. This fixes an issue where users have old browser tabs open and try to lazy-load static assets that no longer exist in the latest version. + +The asset cache is enabled by passing the `database` option to `createRouter`. diff --git a/packages/backend/src/plugins/app.ts b/packages/backend/src/plugins/app.ts index b5f05f2255..0747d890e1 100644 --- a/packages/backend/src/plugins/app.ts +++ b/packages/backend/src/plugins/app.ts @@ -21,10 +21,12 @@ import { PluginEnvironment } from '../types'; export default async function createPlugin({ logger, config, + database, }: PluginEnvironment): Promise { return await createRouter({ logger, config, + database, appPackageName: 'example-app', }); } diff --git a/packages/create-app/templates/default-app/packages/backend/src/plugins/app.ts b/packages/create-app/templates/default-app/packages/backend/src/plugins/app.ts index 07fb04fca2..14e19a19b1 100644 --- a/packages/create-app/templates/default-app/packages/backend/src/plugins/app.ts +++ b/packages/create-app/templates/default-app/packages/backend/src/plugins/app.ts @@ -5,10 +5,12 @@ import { PluginEnvironment } from '../types'; export default async function createPlugin({ logger, config, + database, }: PluginEnvironment): Promise { return await createRouter({ logger, config, + database, appPackageName: 'app', }); } diff --git a/plugins/app-backend/api-report.md b/plugins/app-backend/api-report.md index d742b10442..50d1b7bc34 100644 --- a/plugins/app-backend/api-report.md +++ b/plugins/app-backend/api-report.md @@ -6,6 +6,7 @@ import { Config } from '@backstage/config'; import express from 'express'; import { Logger as Logger_2 } from 'winston'; +import { PluginDatabaseManager } from '@backstage/backend-common'; // Warning: (ae-missing-release-tag) "createRouter" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // @@ -19,6 +20,7 @@ export interface RouterOptions { appPackageName: string; // (undocumented) config: Config; + database?: PluginDatabaseManager; disableConfigInjection?: boolean; // (undocumented) logger: Logger_2; diff --git a/plugins/app-backend/migrations/20211229105307_init.js b/plugins/app-backend/migrations/20211229105307_init.js new file mode 100644 index 0000000000..29e37b35f7 --- /dev/null +++ b/plugins/app-backend/migrations/20211229105307_init.js @@ -0,0 +1,48 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// @ts-check + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function up(knex) { + return knex.schema.createTable('static_assets_cache', table => { + table.comment( + 'A cache of static assets that where previously deployed and may still be lazy-loaded by clients', + ); + table.text('path').primary().notNullable().comment('The path of the file'); + table + .dateTime('last_modified_at') + .defaultTo(knex.fn.now()) + .notNullable() + .comment( + 'Timestamp of when the asset was most recently seen in a deployment', + ); + table.binary('content').notNullable().comment('The asset content'); + table.index('last_modified_at', 'static_asset_cache_last_modified_at_idx'); + }); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function down(knex) { + await knex.schema.alterTable('static_assets_cache', table => { + table.dropIndex([], 'static_asset_cache_last_modified_at_idx'); + }); + await knex.schema.dropTable('static_assets_cache'); +}; diff --git a/plugins/app-backend/package.json b/plugins/app-backend/package.json index d5cbe9c07d..a7fb51f772 100644 --- a/plugins/app-backend/package.json +++ b/plugins/app-backend/package.json @@ -35,22 +35,29 @@ "@backstage/config": "^0.1.12", "@backstage/types": "^0.1.1", "@types/express": "^4.17.6", + "globby": "^11.0.0", "express": "^4.17.1", "express-promise-router": "^4.1.0", "fs-extra": "9.1.0", "helmet": "^4.0.0", + "knex": "^0.95.1", + "lodash": "^4.17.21", + "luxon": "^2.0.2", "winston": "^3.2.1", "yn": "^4.0.0" }, "devDependencies": { + "@backstage/backend-test-utils": "^0.1.13", "@backstage/cli": "^0.11.0", "@backstage/types": "^0.1.1", "@types/supertest": "^2.0.8", + "mock-fs": "^5.1.0", "msw": "^0.35.0", "supertest": "^6.1.3" }, "files": [ "dist", + "migrations/**/*.{js,d.ts}", "static" ] } diff --git a/plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts b/plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts new file mode 100644 index 0000000000..0230486444 --- /dev/null +++ b/plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts @@ -0,0 +1,160 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getVoidLogger } from '@backstage/backend-common'; +import { TestDatabases } from '@backstage/backend-test-utils'; +import { StaticAssetsStore } from './StaticAssetsStore'; + +const logger = getVoidLogger(); + +describe('StaticAssetsStore', () => { + const databases = TestDatabases.create({ + ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + }); + + it.each(databases.eachSupportedId())( + 'should store and retrieve assets, %p', + async databaseId => { + const store = await StaticAssetsStore.create({ + logger, + database: await databases.init(databaseId), + }); + + await store.storeAssets([ + { + path: 'foo.txt', + content: async () => Buffer.from('foo'), + }, + { + path: 'dir/bar.txt', + content: async () => Buffer.from('bar'), + }, + ]); + + const now = new Date().getTime(); + + const foo = await store.getAsset('foo.txt'); + expect(foo!.path).toBe('foo.txt'); + expect(foo!.lastModifiedAt.getTime()).toBeGreaterThan(now - 5000); + expect(foo!.lastModifiedAt.getTime()).toBeLessThan(now + 5000); + expect(foo!.content).toEqual(Buffer.from('foo')); + + const bar = await store.getAsset('dir/bar.txt'); + expect(bar!.path).toBe('dir/bar.txt'); + expect( + Math.abs(bar!.lastModifiedAt.getTime() - foo!.lastModifiedAt.getTime()), + ).toBeLessThan(1000); + expect(bar!.content).toEqual(Buffer.from('bar')); + + await expect( + store.getAsset('does-not-exist.txt'), + ).resolves.toBeUndefined(); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'should update assets timestamps, but not contents, %p', + async databaseId => { + const store = await StaticAssetsStore.create({ + logger, + database: await databases.init(databaseId), + }); + + await store.storeAssets([ + { + path: 'foo', + content: async () => Buffer.from('foo'), + }, + { + path: 'bar', + content: async () => Buffer.from('bar'), + }, + ]); + + const oldFoo = await store.getAsset('foo'); + expect(oldFoo?.lastModifiedAt).toBeDefined(); + + const oldBar = await store.getAsset('bar'); + expect(oldBar?.lastModifiedAt).toBeDefined(); + + // SQLite dates end up with second precision, so make sure we wait at least 1s + await new Promise(resolve => setTimeout(resolve, 1500)); + + await store.storeAssets([ + { + path: 'foo', + content: async () => Buffer.from('newFoo'), + }, + ]); + + const newFoo = await store.getAsset('foo'); + expect(oldFoo!.lastModifiedAt).not.toEqual(newFoo!.lastModifiedAt); + expect(oldFoo!.lastModifiedAt.getTime()).toBeLessThan( + newFoo!.lastModifiedAt.getTime(), + ); + + // The "static" in "StaticAssetsStore" means that assets aren't allowed to change + expect(newFoo!.content).toEqual(Buffer.from('foo')); + + const sameBar = await store.getAsset('bar'); + expect(oldBar!.lastModifiedAt).toEqual(sameBar!.lastModifiedAt); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'should trim old assets, %p', + async databaseId => { + const database = await databases.init(databaseId); + const store = await StaticAssetsStore.create({ + logger, + database, + }); + + await store.storeAssets([ + { + path: 'new', + content: async () => Buffer.alloc(0), + }, + { + path: 'old', + content: async () => Buffer.alloc(0), + }, + ]); + + // Rewrite modified time of "old" to be 1h in the past + const updated = await database('static_assets_cache') + .where({ path: 'old' }) + .update({ + last_modified_at: + database.client.config.client === 'sqlite3' + ? database.raw(`datetime('now', '-3600 seconds')`) + : database.raw(`now() + interval '-3600 seconds'`), + }); + expect(updated).toBe(1); + + await expect(store.getAsset('new')).resolves.toBeDefined(); + await expect(store.getAsset('old')).resolves.toBeDefined(); + + await store.trimAssets({ maxAgeSeconds: 1800 }); + + await expect(store.getAsset('new')).resolves.toBeDefined(); + await expect(store.getAsset('old')).resolves.toBeUndefined(); + }, + 60_000, + ); +}); diff --git a/plugins/app-backend/src/lib/assets/StaticAssetsStore.ts b/plugins/app-backend/src/lib/assets/StaticAssetsStore.ts new file mode 100644 index 0000000000..6b393ce036 --- /dev/null +++ b/plugins/app-backend/src/lib/assets/StaticAssetsStore.ts @@ -0,0 +1,142 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { resolvePackagePath } from '@backstage/backend-common'; +import { Knex } from 'knex'; +import { Logger } from 'winston'; +import { DateTime } from 'luxon'; +import partition from 'lodash/partition'; +import { StaticAsset, StaticAssetInput, StaticAssetProvider } from './types'; + +const migrationsDir = resolvePackagePath( + '@backstage/plugin-app-backend', + 'migrations', +); + +interface StaticAssetRow { + path: string; + content: Buffer; + last_modified_at: Date; +} + +/** @internal */ +export interface StaticAssetsStoreOptions { + database: Knex; + logger: Logger; +} + +/** + * A storage for static assets that are assumed to be immutable. + * + * @internal + */ +export class StaticAssetsStore implements StaticAssetProvider { + #db: Knex; + #logger: Logger; + + static async create(options: StaticAssetsStoreOptions) { + await options.database.migrate.latest({ + directory: migrationsDir, + }); + return new StaticAssetsStore(options); + } + + private constructor(options: StaticAssetsStoreOptions) { + this.#db = options.database; + this.#logger = options.logger; + } + + /** + * Store the provided assets. + * + * If an asset for a given path already exists the modification time will be + * updated, but the contents will not. + */ + async storeAssets(assets: StaticAssetInput[]) { + const existingRows = await this.#db( + 'static_assets_cache', + ).whereIn( + 'path', + assets.map(a => a.path), + ); + const existingAssetPaths = new Set(existingRows.map(r => r.path)); + + const [modified, added] = partition(assets, asset => + existingAssetPaths.has(asset.path), + ); + + this.#logger.info( + `Storing ${modified.length} updated assets and ${added.length} new assets`, + ); + + await this.#db('static_assets_cache') + .update({ + last_modified_at: this.#db.fn.now(), + }) + .whereIn( + 'path', + modified.map(a => a.path), + ); + + for (const asset of added) { + // We ignore conflicts with other nodes, it doesn't matter if someone else + // added the same asset just before us. + await this.#db('static_assets_cache') + .insert({ + path: asset.path, + content: await asset.content(), + }) + .onConflict('path') + .ignore(); + } + } + + /** + * Retrieve an asset from the store with the given path. + */ + async getAsset(path: string): Promise { + const [row] = await this.#db('static_assets_cache').where({ + path, + }); + if (!row) { + return undefined; + } + return { + path: row.path, + content: row.content, + lastModifiedAt: + typeof row.last_modified_at === 'string' + ? DateTime.fromSQL(row.last_modified_at, { zone: 'UTC' }).toJSDate() + : row.last_modified_at, + }; + } + + /** + * Delete any assets from the store whose modification time is older than the max age. + */ + async trimAssets(options: { maxAgeSeconds: number }) { + const { maxAgeSeconds } = options; + await this.#db('static_assets_cache') + .where( + 'last_modified_at', + '<=', + this.#db.client.config.client === 'sqlite3' + ? this.#db.raw(`datetime('now', ?)`, [`-${maxAgeSeconds} seconds`]) + : this.#db.raw(`now() + interval '${-maxAgeSeconds} seconds'`), + ) + .delete(); + } +} diff --git a/plugins/app-backend/src/lib/assets/createStaticAssetMiddleware.test.ts b/plugins/app-backend/src/lib/assets/createStaticAssetMiddleware.test.ts new file mode 100644 index 0000000000..63cb3a121f --- /dev/null +++ b/plugins/app-backend/src/lib/assets/createStaticAssetMiddleware.test.ts @@ -0,0 +1,109 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import express from 'express'; +import request from 'supertest'; +import { createStaticAssetMiddleware } from './createStaticAssetMiddleware'; +import { StaticAssetsStore } from './StaticAssetsStore'; + +const mockStore = { + getAsset: jest.fn(), +} as unknown as jest.Mocked; + +describe('createStaticAssetMiddleware', () => { + const app = express(); + app.use(createStaticAssetMiddleware(mockStore)); + app.use((_req, res) => { + res.status(404).end('Not Found'); + }); + + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('should respond with an asset', async () => { + const now = new Date(); + mockStore.getAsset.mockResolvedValueOnce({ + path: 'foo.js', + lastModifiedAt: now, + content: Buffer.from('foo'), + }); + + const res = await request(app).get('/foo.js'); + + expect(res.status).toBe(200); + expect(res.text).toBe('foo'); + expect(res.get('Content-Type')).toBe( + 'application/javascript; charset=utf-8', + ); + expect(res.get('Content-Length')).toBe('3'); + expect(res.get('Cache-Control')).toBe('public, max-age=1209600'); + expect(res.get('Last-Modified')).toBe(now.toUTCString()); + + expect(mockStore.getAsset).toHaveBeenCalledTimes(1); + expect(mockStore.getAsset).toHaveBeenCalledWith('foo.js'); + }); + + mockStore.getAsset.mockResolvedValueOnce(undefined); + + it('should respond with not found', async () => { + const res = await request(app).get('/foo.js'); + + expect(res.status).toBe(404); + expect(res.text).toBe('Not Found'); + + expect(mockStore.getAsset).toHaveBeenCalledTimes(1); + expect(mockStore.getAsset).toHaveBeenCalledWith('foo.js'); + }); + + it('should handle other content type', async () => { + mockStore.getAsset.mockResolvedValueOnce({ + path: 'foo.css', + lastModifiedAt: new Date(), + content: Buffer.from('foo'), + }); + + const res = await request(app).get('/foo.css'); + + expect(res.status).toBe(200); + expect(res.text).toBe('foo'); + expect(res.get('Content-Type')).toBe('text/css; charset=utf-8'); + expect(res.get('Content-Length')).toBe('3'); + + expect(mockStore.getAsset).toHaveBeenCalledTimes(1); + expect(mockStore.getAsset).toHaveBeenLastCalledWith('foo.css'); + }); + + it('should handle unknown content types', async () => { + mockStore.getAsset.mockResolvedValueOnce({ + path: 'foo.notavalidextension', + lastModifiedAt: new Date(), + content: Buffer.from('foo'), + }); + + const res = await request(app).get('/foo.notavalidextension'); + + expect(res.status).toBe(200); + expect(res.body).toEqual(Buffer.from('foo')); + expect(res.get('Content-Type')).toBe('application/octet-stream'); + expect(res.get('Content-Length')).toBe('3'); + + expect(mockStore.getAsset).toHaveBeenCalledTimes(1); + expect(mockStore.getAsset).toHaveBeenLastCalledWith( + 'foo.notavalidextension', + ); + }); +}); diff --git a/plugins/app-backend/src/lib/assets/createStaticAssetMiddleware.ts b/plugins/app-backend/src/lib/assets/createStaticAssetMiddleware.ts new file mode 100644 index 0000000000..60458436dd --- /dev/null +++ b/plugins/app-backend/src/lib/assets/createStaticAssetMiddleware.ts @@ -0,0 +1,64 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { extname } from 'path'; +import { RequestHandler } from 'express'; +import { StaticAssetProvider } from './types'; +import { CACHE_CONTROL_MAX_CACHE } from '../headers'; + +/** + * Creates a middleware that serves static assets from a static asset provider + * + * @internal + */ +export function createStaticAssetMiddleware( + store: StaticAssetProvider, +): RequestHandler { + return (req, res, next) => { + if (req.method !== 'GET' && req.method !== 'HEAD') { + next(); + return; + } + + // Let's not assume we're in promise-router + Promise.resolve( + (async () => { + // Drop leading slashes from the incoming path + const path = req.path.startsWith('/') ? req.path.slice(1) : req.path; + + const asset = await store.getAsset(path); + if (!asset) { + next(); + return; + } + + // Set the Content-Type header, falling back to octet-stream + const ext = extname(asset.path); + if (ext) { + res.type(ext); + } else { + res.type('bin'); + } + + // Same as our express.static override + res.setHeader('Cache-Control', CACHE_CONTROL_MAX_CACHE); + res.setHeader('Last-Modified', asset.lastModifiedAt.toUTCString()); + + res.send(asset.content); + })(), + ).catch(next); + }; +} diff --git a/plugins/app-backend/src/lib/assets/findStaticAssets.test.ts b/plugins/app-backend/src/lib/assets/findStaticAssets.test.ts new file mode 100644 index 0000000000..6527c28ea4 --- /dev/null +++ b/plugins/app-backend/src/lib/assets/findStaticAssets.test.ts @@ -0,0 +1,78 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import mockFs from 'mock-fs'; +import { findStaticAssets } from './findStaticAssets'; + +describe('findStaticAssets', () => { + afterEach(() => { + mockFs.restore(); + }); + + it('should find assets', async () => { + mockFs({ + '/test': { + 'a.js': 'alert("hello")', + 'a.js.map': '', + 'b.js': 'b', + 'b.js.map': '', + js: { + 'd.js': 'd', + 'd.js.map': '', + x: { + 'e.map': '', + y: { + 'e.map': '', + z: { + 'e.js': 'e', + 'e.map': '', + }, + }, + }, + }, + styles: { 'c.css': 'body { color: red; }' }, + }, + }); + + const assets = await findStaticAssets('/test'); + expect(assets.length).toBe(5); + expect(assets.map(a => a.path)).toEqual( + expect.arrayContaining([ + 'a.js', + 'b.js', + 'styles/c.css', + 'js/d.js', + 'js/x/y/z/e.js', + ]), + ); + + await expect( + assets.find(a => a.path === 'a.js')!.content(), + ).resolves.toEqual(Buffer.from('alert("hello")')); + await expect( + assets.find(a => a.path === 'b.js')!.content(), + ).resolves.toEqual(Buffer.from('b')); + await expect( + assets.find(a => a.path === 'styles/c.css')!.content(), + ).resolves.toEqual(Buffer.from('body { color: red; }')); + await expect( + assets.find(a => a.path === 'js/d.js')!.content(), + ).resolves.toEqual(Buffer.from('d')); + await expect( + assets.find(a => a.path === 'js/x/y/z/e.js')!.content(), + ).resolves.toEqual(Buffer.from('e')); + }); +}); diff --git a/plugins/app-backend/src/lib/assets/findStaticAssets.ts b/plugins/app-backend/src/lib/assets/findStaticAssets.ts new file mode 100644 index 0000000000..7ecea7146e --- /dev/null +++ b/plugins/app-backend/src/lib/assets/findStaticAssets.ts @@ -0,0 +1,40 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs-extra'; +import globby from 'globby'; +import { StaticAssetInput } from './types'; +import { resolveSafeChildPath } from '@backstage/backend-common'; + +/** + * Finds all static assets within a directory + * + * @internal + */ +export async function findStaticAssets( + staticDir: string, +): Promise { + const assetPaths = await globby('**/*', { + ignore: ['**/*.map'], // Ignore source maps since they're quite large + cwd: staticDir, + dot: true, + }); + + return assetPaths.map(path => ({ + path, + content: async () => fs.readFile(resolveSafeChildPath(staticDir, path)), + })); +} diff --git a/plugins/app-backend/src/lib/assets/index.ts b/plugins/app-backend/src/lib/assets/index.ts new file mode 100644 index 0000000000..3a158b4392 --- /dev/null +++ b/plugins/app-backend/src/lib/assets/index.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export { StaticAssetsStore } from './StaticAssetsStore'; +export type { + StaticAsset, + StaticAssetInput, + StaticAssetProvider, +} from './types'; +export { findStaticAssets } from './findStaticAssets'; +export { createStaticAssetMiddleware } from './createStaticAssetMiddleware'; diff --git a/plugins/app-backend/src/lib/assets/types.ts b/plugins/app-backend/src/lib/assets/types.ts new file mode 100644 index 0000000000..464d818d3e --- /dev/null +++ b/plugins/app-backend/src/lib/assets/types.ts @@ -0,0 +1,30 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface StaticAssetInput { + path: string; + content(): Promise; +} + +export interface StaticAsset { + path: string; + content: Buffer; + lastModifiedAt: Date; +} + +export interface StaticAssetProvider { + getAsset(path: string): Promise; +} diff --git a/plugins/app-backend/src/lib/headers.ts b/plugins/app-backend/src/lib/headers.ts new file mode 100644 index 0000000000..1bee2bbb37 --- /dev/null +++ b/plugins/app-backend/src/lib/headers.ts @@ -0,0 +1,18 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export const CACHE_CONTROL_NO_CACHE = 'no-store, max-age=0'; +export const CACHE_CONTROL_MAX_CACHE = 'public, max-age=1209600'; // 14 days diff --git a/plugins/app-backend/src/service/router.test.ts b/plugins/app-backend/src/service/router.test.ts index bbbf115834..30fa5454c0 100644 --- a/plugins/app-backend/src/service/router.test.ts +++ b/plugins/app-backend/src/service/router.test.ts @@ -81,7 +81,7 @@ describe('createRouter', () => { 'returns %s with default Cache-Control header', async file => { const response = await request(app).get(file); - expect(response.header['cache-control']).toBe('public, max-age=0'); + expect(response.header['cache-control']).toBe('public, max-age=1209600'); }, ); }); diff --git a/plugins/app-backend/src/service/router.ts b/plugins/app-backend/src/service/router.ts index 171684a964..b666f323db 100644 --- a/plugins/app-backend/src/service/router.ts +++ b/plugins/app-backend/src/service/router.ts @@ -14,7 +14,11 @@ * limitations under the License. */ -import { notFoundHandler, resolvePackagePath } from '@backstage/backend-common'; +import { + notFoundHandler, + PluginDatabaseManager, + resolvePackagePath, +} from '@backstage/backend-common'; import { Config } from '@backstage/config'; import helmet from 'helmet'; import express from 'express'; @@ -23,6 +27,15 @@ import fs from 'fs-extra'; import { resolve as resolvePath } from 'path'; import { Logger } from 'winston'; import { injectConfig, readConfigs } from '../lib/config'; +import { + StaticAssetsStore, + findStaticAssets, + createStaticAssetMiddleware, +} from '../lib/assets'; +import { + CACHE_CONTROL_MAX_CACHE, + CACHE_CONTROL_NO_CACHE, +} from '../lib/headers'; // express uses mime v1 while we only have types for mime v2 type Mime = { lookup(arg0: string): string }; @@ -31,6 +44,13 @@ export interface RouterOptions { config: Config; logger: Logger; + /** + * If a database is provided it will be used to cache previously deployed static assets. + * + * This is a built-in alternative to using a `staticFallbackHandler`. + */ + database?: PluginDatabaseManager; + /** * The name of the app package that content should be served from. The same app package should be * added as a dependency to the backend package in order for it to be accessible at runtime. @@ -45,6 +65,11 @@ export interface RouterOptions { * This can be used to avoid issues with clients on older deployment versions trying to access lazy * loaded content that is no longer present. Typically the requests would fall back to a long-term * object store where all recently deployed versions of the app are present. + * + * Another option is to provide a `database` that will take care of storing the static assets instead. + * + * If both `database` and `staticFallbackHandler` are provided, the `database` will attempt to serve + * static assets first, and if they are not found, the `staticFallbackHandler` will be called. */ staticFallbackHandler?: express.Handler; @@ -94,7 +119,28 @@ export async function createRouter( // Use a separate router for static content so that a fallback can be provided by backend const staticRouter = Router(); - staticRouter.use(express.static(resolvePath(appDistDir, 'static'))); + staticRouter.use( + express.static(resolvePath(appDistDir, 'static'), { + setHeaders: res => { + res.setHeader('Cache-Control', CACHE_CONTROL_MAX_CACHE); + }, + }), + ); + + if (options.database) { + const store = await StaticAssetsStore.create({ + logger, + database: await options.database.getClient(), + }); + + const assets = await findStaticAssets(staticDir); + await store.storeAssets(assets); + // Remove any assets that are older than 7 days + await store.trimAssets({ maxAgeSeconds: 60 * 60 * 24 * 7 }); + + staticRouter.use(createStaticAssetMiddleware(store)); + } + if (staticFallbackHandler) { staticRouter.use(staticFallbackHandler); } @@ -109,7 +155,7 @@ export async function createRouter( if ( (express.static.mime as unknown as Mime).lookup(path) === 'text/html' ) { - res.setHeader('Cache-Control', 'no-store, max-age=0'); + res.setHeader('Cache-Control', CACHE_CONTROL_NO_CACHE); } }, }), @@ -119,7 +165,7 @@ export async function createRouter( headers: { // The Cache-Control header instructs the browser to not cache the index.html since it might // link to static assets from recently deployed versions. - 'cache-control': 'no-store, max-age=0', + 'cache-control': CACHE_CONTROL_NO_CACHE, }, }); });