Merge pull request #8681 from backstage/rugvip/assetcache

app-backend: store static assets across deployments and improve caching
This commit is contained in:
Fredrik Adelöw
2022-01-17 16:26:17 +01:00
committed by GitHub
19 changed files with 811 additions and 5 deletions
@@ -0,0 +1,160 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { getVoidLogger } from '@backstage/backend-common';
import { TestDatabases } from '@backstage/backend-test-utils';
import { StaticAssetsStore } from './StaticAssetsStore';
const logger = getVoidLogger();
describe('StaticAssetsStore', () => {
const databases = TestDatabases.create({
ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
});
it.each(databases.eachSupportedId())(
'should store and retrieve assets, %p',
async databaseId => {
const store = await StaticAssetsStore.create({
logger,
database: await databases.init(databaseId),
});
await store.storeAssets([
{
path: 'foo.txt',
content: async () => Buffer.from('foo'),
},
{
path: 'dir/bar.txt',
content: async () => Buffer.from('bar'),
},
]);
const now = new Date().getTime();
const foo = await store.getAsset('foo.txt');
expect(foo!.path).toBe('foo.txt');
expect(foo!.lastModifiedAt.getTime()).toBeGreaterThan(now - 5000);
expect(foo!.lastModifiedAt.getTime()).toBeLessThan(now + 5000);
expect(foo!.content).toEqual(Buffer.from('foo'));
const bar = await store.getAsset('dir/bar.txt');
expect(bar!.path).toBe('dir/bar.txt');
expect(
Math.abs(bar!.lastModifiedAt.getTime() - foo!.lastModifiedAt.getTime()),
).toBeLessThan(1000);
expect(bar!.content).toEqual(Buffer.from('bar'));
await expect(
store.getAsset('does-not-exist.txt'),
).resolves.toBeUndefined();
},
60_000,
);
it.each(databases.eachSupportedId())(
'should update assets timestamps, but not contents, %p',
async databaseId => {
const store = await StaticAssetsStore.create({
logger,
database: await databases.init(databaseId),
});
await store.storeAssets([
{
path: 'foo',
content: async () => Buffer.from('foo'),
},
{
path: 'bar',
content: async () => Buffer.from('bar'),
},
]);
const oldFoo = await store.getAsset('foo');
expect(oldFoo?.lastModifiedAt).toBeDefined();
const oldBar = await store.getAsset('bar');
expect(oldBar?.lastModifiedAt).toBeDefined();
// SQLite dates end up with second precision, so make sure we wait at least 1s
await new Promise(resolve => setTimeout(resolve, 1500));
await store.storeAssets([
{
path: 'foo',
content: async () => Buffer.from('newFoo'),
},
]);
const newFoo = await store.getAsset('foo');
expect(oldFoo!.lastModifiedAt).not.toEqual(newFoo!.lastModifiedAt);
expect(oldFoo!.lastModifiedAt.getTime()).toBeLessThan(
newFoo!.lastModifiedAt.getTime(),
);
// The "static" in "StaticAssetsStore" means that assets aren't allowed to change
expect(newFoo!.content).toEqual(Buffer.from('foo'));
const sameBar = await store.getAsset('bar');
expect(oldBar!.lastModifiedAt).toEqual(sameBar!.lastModifiedAt);
},
60_000,
);
it.each(databases.eachSupportedId())(
'should trim old assets, %p',
async databaseId => {
const database = await databases.init(databaseId);
const store = await StaticAssetsStore.create({
logger,
database,
});
await store.storeAssets([
{
path: 'new',
content: async () => Buffer.alloc(0),
},
{
path: 'old',
content: async () => Buffer.alloc(0),
},
]);
// Rewrite modified time of "old" to be 1h in the past
const updated = await database('static_assets_cache')
.where({ path: 'old' })
.update({
last_modified_at:
database.client.config.client === 'sqlite3'
? database.raw(`datetime('now', '-3600 seconds')`)
: database.raw(`now() + interval '-3600 seconds'`),
});
expect(updated).toBe(1);
await expect(store.getAsset('new')).resolves.toBeDefined();
await expect(store.getAsset('old')).resolves.toBeDefined();
await store.trimAssets({ maxAgeSeconds: 1800 });
await expect(store.getAsset('new')).resolves.toBeDefined();
await expect(store.getAsset('old')).resolves.toBeUndefined();
},
60_000,
);
});
@@ -0,0 +1,142 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { resolvePackagePath } from '@backstage/backend-common';
import { Knex } from 'knex';
import { Logger } from 'winston';
import { DateTime } from 'luxon';
import partition from 'lodash/partition';
import { StaticAsset, StaticAssetInput, StaticAssetProvider } from './types';
const migrationsDir = resolvePackagePath(
'@backstage/plugin-app-backend',
'migrations',
);
interface StaticAssetRow {
path: string;
content: Buffer;
last_modified_at: Date;
}
/** @internal */
export interface StaticAssetsStoreOptions {
database: Knex;
logger: Logger;
}
/**
* A storage for static assets that are assumed to be immutable.
*
* @internal
*/
export class StaticAssetsStore implements StaticAssetProvider {
#db: Knex;
#logger: Logger;
static async create(options: StaticAssetsStoreOptions) {
await options.database.migrate.latest({
directory: migrationsDir,
});
return new StaticAssetsStore(options);
}
private constructor(options: StaticAssetsStoreOptions) {
this.#db = options.database;
this.#logger = options.logger;
}
/**
* Store the provided assets.
*
* If an asset for a given path already exists the modification time will be
* updated, but the contents will not.
*/
async storeAssets(assets: StaticAssetInput[]) {
const existingRows = await this.#db<StaticAssetRow>(
'static_assets_cache',
).whereIn(
'path',
assets.map(a => a.path),
);
const existingAssetPaths = new Set(existingRows.map(r => r.path));
const [modified, added] = partition(assets, asset =>
existingAssetPaths.has(asset.path),
);
this.#logger.info(
`Storing ${modified.length} updated assets and ${added.length} new assets`,
);
await this.#db('static_assets_cache')
.update({
last_modified_at: this.#db.fn.now(),
})
.whereIn(
'path',
modified.map(a => a.path),
);
for (const asset of added) {
// We ignore conflicts with other nodes, it doesn't matter if someone else
// added the same asset just before us.
await this.#db('static_assets_cache')
.insert({
path: asset.path,
content: await asset.content(),
})
.onConflict('path')
.ignore();
}
}
/**
* Retrieve an asset from the store with the given path.
*/
async getAsset(path: string): Promise<StaticAsset | undefined> {
const [row] = await this.#db<StaticAssetRow>('static_assets_cache').where({
path,
});
if (!row) {
return undefined;
}
return {
path: row.path,
content: row.content,
lastModifiedAt:
typeof row.last_modified_at === 'string'
? DateTime.fromSQL(row.last_modified_at, { zone: 'UTC' }).toJSDate()
: row.last_modified_at,
};
}
/**
* Delete any assets from the store whose modification time is older than the max age.
*/
async trimAssets(options: { maxAgeSeconds: number }) {
const { maxAgeSeconds } = options;
await this.#db<StaticAssetRow>('static_assets_cache')
.where(
'last_modified_at',
'<=',
this.#db.client.config.client === 'sqlite3'
? this.#db.raw(`datetime('now', ?)`, [`-${maxAgeSeconds} seconds`])
: this.#db.raw(`now() + interval '${-maxAgeSeconds} seconds'`),
)
.delete();
}
}
@@ -0,0 +1,109 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import express from 'express';
import request from 'supertest';
import { createStaticAssetMiddleware } from './createStaticAssetMiddleware';
import { StaticAssetsStore } from './StaticAssetsStore';
const mockStore = {
getAsset: jest.fn(),
} as unknown as jest.Mocked<StaticAssetsStore>;
describe('createStaticAssetMiddleware', () => {
const app = express();
app.use(createStaticAssetMiddleware(mockStore));
app.use((_req, res) => {
res.status(404).end('Not Found');
});
beforeEach(() => {
jest.resetAllMocks();
});
it('should respond with an asset', async () => {
const now = new Date();
mockStore.getAsset.mockResolvedValueOnce({
path: 'foo.js',
lastModifiedAt: now,
content: Buffer.from('foo'),
});
const res = await request(app).get('/foo.js');
expect(res.status).toBe(200);
expect(res.text).toBe('foo');
expect(res.get('Content-Type')).toBe(
'application/javascript; charset=utf-8',
);
expect(res.get('Content-Length')).toBe('3');
expect(res.get('Cache-Control')).toBe('public, max-age=1209600');
expect(res.get('Last-Modified')).toBe(now.toUTCString());
expect(mockStore.getAsset).toHaveBeenCalledTimes(1);
expect(mockStore.getAsset).toHaveBeenCalledWith('foo.js');
});
mockStore.getAsset.mockResolvedValueOnce(undefined);
it('should respond with not found', async () => {
const res = await request(app).get('/foo.js');
expect(res.status).toBe(404);
expect(res.text).toBe('Not Found');
expect(mockStore.getAsset).toHaveBeenCalledTimes(1);
expect(mockStore.getAsset).toHaveBeenCalledWith('foo.js');
});
it('should handle other content type', async () => {
mockStore.getAsset.mockResolvedValueOnce({
path: 'foo.css',
lastModifiedAt: new Date(),
content: Buffer.from('foo'),
});
const res = await request(app).get('/foo.css');
expect(res.status).toBe(200);
expect(res.text).toBe('foo');
expect(res.get('Content-Type')).toBe('text/css; charset=utf-8');
expect(res.get('Content-Length')).toBe('3');
expect(mockStore.getAsset).toHaveBeenCalledTimes(1);
expect(mockStore.getAsset).toHaveBeenLastCalledWith('foo.css');
});
it('should handle unknown content types', async () => {
mockStore.getAsset.mockResolvedValueOnce({
path: 'foo.notavalidextension',
lastModifiedAt: new Date(),
content: Buffer.from('foo'),
});
const res = await request(app).get('/foo.notavalidextension');
expect(res.status).toBe(200);
expect(res.body).toEqual(Buffer.from('foo'));
expect(res.get('Content-Type')).toBe('application/octet-stream');
expect(res.get('Content-Length')).toBe('3');
expect(mockStore.getAsset).toHaveBeenCalledTimes(1);
expect(mockStore.getAsset).toHaveBeenLastCalledWith(
'foo.notavalidextension',
);
});
});
@@ -0,0 +1,64 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { extname } from 'path';
import { RequestHandler } from 'express';
import { StaticAssetProvider } from './types';
import { CACHE_CONTROL_MAX_CACHE } from '../headers';
/**
* Creates a middleware that serves static assets from a static asset provider
*
* @internal
*/
export function createStaticAssetMiddleware(
store: StaticAssetProvider,
): RequestHandler {
return (req, res, next) => {
if (req.method !== 'GET' && req.method !== 'HEAD') {
next();
return;
}
// Let's not assume we're in promise-router
Promise.resolve(
(async () => {
// Drop leading slashes from the incoming path
const path = req.path.startsWith('/') ? req.path.slice(1) : req.path;
const asset = await store.getAsset(path);
if (!asset) {
next();
return;
}
// Set the Content-Type header, falling back to octet-stream
const ext = extname(asset.path);
if (ext) {
res.type(ext);
} else {
res.type('bin');
}
// Same as our express.static override
res.setHeader('Cache-Control', CACHE_CONTROL_MAX_CACHE);
res.setHeader('Last-Modified', asset.lastModifiedAt.toUTCString());
res.send(asset.content);
})(),
).catch(next);
};
}
@@ -0,0 +1,78 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import mockFs from 'mock-fs';
import { findStaticAssets } from './findStaticAssets';
describe('findStaticAssets', () => {
afterEach(() => {
mockFs.restore();
});
it('should find assets', async () => {
mockFs({
'/test': {
'a.js': 'alert("hello")',
'a.js.map': '',
'b.js': 'b',
'b.js.map': '',
js: {
'd.js': 'd',
'd.js.map': '',
x: {
'e.map': '',
y: {
'e.map': '',
z: {
'e.js': 'e',
'e.map': '',
},
},
},
},
styles: { 'c.css': 'body { color: red; }' },
},
});
const assets = await findStaticAssets('/test');
expect(assets.length).toBe(5);
expect(assets.map(a => a.path)).toEqual(
expect.arrayContaining([
'a.js',
'b.js',
'styles/c.css',
'js/d.js',
'js/x/y/z/e.js',
]),
);
await expect(
assets.find(a => a.path === 'a.js')!.content(),
).resolves.toEqual(Buffer.from('alert("hello")'));
await expect(
assets.find(a => a.path === 'b.js')!.content(),
).resolves.toEqual(Buffer.from('b'));
await expect(
assets.find(a => a.path === 'styles/c.css')!.content(),
).resolves.toEqual(Buffer.from('body { color: red; }'));
await expect(
assets.find(a => a.path === 'js/d.js')!.content(),
).resolves.toEqual(Buffer.from('d'));
await expect(
assets.find(a => a.path === 'js/x/y/z/e.js')!.content(),
).resolves.toEqual(Buffer.from('e'));
});
});
@@ -0,0 +1,40 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs-extra';
import globby from 'globby';
import { StaticAssetInput } from './types';
import { resolveSafeChildPath } from '@backstage/backend-common';
/**
* Finds all static assets within a directory
*
* @internal
*/
export async function findStaticAssets(
staticDir: string,
): Promise<StaticAssetInput[]> {
const assetPaths = await globby('**/*', {
ignore: ['**/*.map'], // Ignore source maps since they're quite large
cwd: staticDir,
dot: true,
});
return assetPaths.map(path => ({
path,
content: async () => fs.readFile(resolveSafeChildPath(staticDir, path)),
}));
}
@@ -0,0 +1,24 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { StaticAssetsStore } from './StaticAssetsStore';
export type {
StaticAsset,
StaticAssetInput,
StaticAssetProvider,
} from './types';
export { findStaticAssets } from './findStaticAssets';
export { createStaticAssetMiddleware } from './createStaticAssetMiddleware';
@@ -0,0 +1,30 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export interface StaticAssetInput {
path: string;
content(): Promise<Buffer>;
}
export interface StaticAsset {
path: string;
content: Buffer;
lastModifiedAt: Date;
}
export interface StaticAssetProvider {
getAsset(path: string): Promise<StaticAsset | undefined>;
}
+18
View File
@@ -0,0 +1,18 @@
/*
* Copyright 2021 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export const CACHE_CONTROL_NO_CACHE = 'no-store, max-age=0';
export const CACHE_CONTROL_MAX_CACHE = 'public, max-age=1209600'; // 14 days
@@ -81,7 +81,7 @@ describe('createRouter', () => {
'returns %s with default Cache-Control header',
async file => {
const response = await request(app).get(file);
expect(response.header['cache-control']).toBe('public, max-age=0');
expect(response.header['cache-control']).toBe('public, max-age=1209600');
},
);
});
+50 -4
View File
@@ -14,7 +14,11 @@
* limitations under the License.
*/
import { notFoundHandler, resolvePackagePath } from '@backstage/backend-common';
import {
notFoundHandler,
PluginDatabaseManager,
resolvePackagePath,
} from '@backstage/backend-common';
import { Config } from '@backstage/config';
import helmet from 'helmet';
import express from 'express';
@@ -23,6 +27,15 @@ import fs from 'fs-extra';
import { resolve as resolvePath } from 'path';
import { Logger } from 'winston';
import { injectConfig, readConfigs } from '../lib/config';
import {
StaticAssetsStore,
findStaticAssets,
createStaticAssetMiddleware,
} from '../lib/assets';
import {
CACHE_CONTROL_MAX_CACHE,
CACHE_CONTROL_NO_CACHE,
} from '../lib/headers';
// express uses mime v1 while we only have types for mime v2
type Mime = { lookup(arg0: string): string };
@@ -31,6 +44,13 @@ export interface RouterOptions {
config: Config;
logger: Logger;
/**
* If a database is provided it will be used to cache previously deployed static assets.
*
* This is a built-in alternative to using a `staticFallbackHandler`.
*/
database?: PluginDatabaseManager;
/**
* The name of the app package that content should be served from. The same app package should be
* added as a dependency to the backend package in order for it to be accessible at runtime.
@@ -45,6 +65,11 @@ export interface RouterOptions {
* This can be used to avoid issues with clients on older deployment versions trying to access lazy
* loaded content that is no longer present. Typically the requests would fall back to a long-term
* object store where all recently deployed versions of the app are present.
*
* Another option is to provide a `database` that will take care of storing the static assets instead.
*
* If both `database` and `staticFallbackHandler` are provided, the `database` will attempt to serve
* static assets first, and if they are not found, the `staticFallbackHandler` will be called.
*/
staticFallbackHandler?: express.Handler;
@@ -94,7 +119,28 @@ export async function createRouter(
// Use a separate router for static content so that a fallback can be provided by backend
const staticRouter = Router();
staticRouter.use(express.static(resolvePath(appDistDir, 'static')));
staticRouter.use(
express.static(resolvePath(appDistDir, 'static'), {
setHeaders: res => {
res.setHeader('Cache-Control', CACHE_CONTROL_MAX_CACHE);
},
}),
);
if (options.database) {
const store = await StaticAssetsStore.create({
logger,
database: await options.database.getClient(),
});
const assets = await findStaticAssets(staticDir);
await store.storeAssets(assets);
// Remove any assets that are older than 7 days
await store.trimAssets({ maxAgeSeconds: 60 * 60 * 24 * 7 });
staticRouter.use(createStaticAssetMiddleware(store));
}
if (staticFallbackHandler) {
staticRouter.use(staticFallbackHandler);
}
@@ -109,7 +155,7 @@ export async function createRouter(
if (
(express.static.mime as unknown as Mime).lookup(path) === 'text/html'
) {
res.setHeader('Cache-Control', 'no-store, max-age=0');
res.setHeader('Cache-Control', CACHE_CONTROL_NO_CACHE);
}
},
}),
@@ -119,7 +165,7 @@ export async function createRouter(
headers: {
// The Cache-Control header instructs the browser to not cache the index.html since it might
// link to static assets from recently deployed versions.
'cache-control': 'no-store, max-age=0',
'cache-control': CACHE_CONTROL_NO_CACHE,
},
});
});