app-backend: add StaticAssetsStore
Signed-off-by: Patrik Oldsberg <poldsberg@gmail.com>
This commit is contained in:
Patrik Oldsberg
@@ -0,0 +1,160 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { getVoidLogger } from '@backstage/backend-common';
|
||||
import { TestDatabases } from '@backstage/backend-test-utils';
|
||||
import { StaticAssetsStore } from './StaticAssetsStore';
|
||||
|
||||
const logger = getVoidLogger();
|
||||
|
||||
describe('StaticAssetsStore', () => {
|
||||
const databases = TestDatabases.create({
|
||||
ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
|
||||
});
|
||||
|
||||
it.each(databases.eachSupportedId())(
|
||||
'should store and retrieve assets, %p',
|
||||
async databaseId => {
|
||||
const store = await StaticAssetsStore.create({
|
||||
logger,
|
||||
database: await databases.init(databaseId),
|
||||
});
|
||||
|
||||
await store.storeAssets([
|
||||
{
|
||||
path: 'foo.txt',
|
||||
content: async () => Buffer.from('foo'),
|
||||
},
|
||||
{
|
||||
path: 'dir/bar.txt',
|
||||
content: async () => Buffer.from('bar'),
|
||||
},
|
||||
]);
|
||||
|
||||
const now = new Date().getTime();
|
||||
|
||||
const foo = await store.getAsset('foo.txt');
|
||||
expect(foo!.path).toBe('foo.txt');
|
||||
expect(foo!.lastModifiedAt.getTime()).toBeGreaterThan(now - 5000);
|
||||
expect(foo!.lastModifiedAt.getTime()).toBeLessThan(now + 5000);
|
||||
expect(foo!.content).toEqual(Buffer.from('foo'));
|
||||
|
||||
const bar = await store.getAsset('dir/bar.txt');
|
||||
expect(bar!.path).toBe('dir/bar.txt');
|
||||
expect(
|
||||
Math.abs(bar!.lastModifiedAt.getTime() - foo!.lastModifiedAt.getTime()),
|
||||
).toBeLessThan(1000);
|
||||
expect(bar!.content).toEqual(Buffer.from('bar'));
|
||||
|
||||
await expect(
|
||||
store.getAsset('does-not-exist.txt'),
|
||||
).resolves.toBeUndefined();
|
||||
},
|
||||
60_000,
|
||||
);
|
||||
|
||||
it.each(databases.eachSupportedId())(
|
||||
'should update assets timestamps, but not contents, %p',
|
||||
async databaseId => {
|
||||
const store = await StaticAssetsStore.create({
|
||||
logger,
|
||||
database: await databases.init(databaseId),
|
||||
});
|
||||
|
||||
await store.storeAssets([
|
||||
{
|
||||
path: 'foo',
|
||||
content: async () => Buffer.from('foo'),
|
||||
},
|
||||
{
|
||||
path: 'bar',
|
||||
content: async () => Buffer.from('bar'),
|
||||
},
|
||||
]);
|
||||
|
||||
const oldFoo = await store.getAsset('foo');
|
||||
expect(oldFoo?.lastModifiedAt).toBeDefined();
|
||||
|
||||
const oldBar = await store.getAsset('bar');
|
||||
expect(oldBar?.lastModifiedAt).toBeDefined();
|
||||
|
||||
// SQLite dates end up with second precision, so make sure we wait at least 1s
|
||||
await new Promise(resolve => setTimeout(resolve, 1500));
|
||||
|
||||
await store.storeAssets([
|
||||
{
|
||||
path: 'foo',
|
||||
content: async () => Buffer.from('newFoo'),
|
||||
},
|
||||
]);
|
||||
|
||||
const newFoo = await store.getAsset('foo');
|
||||
expect(oldFoo!.lastModifiedAt).not.toEqual(newFoo!.lastModifiedAt);
|
||||
expect(oldFoo!.lastModifiedAt.getTime()).toBeLessThan(
|
||||
newFoo!.lastModifiedAt.getTime(),
|
||||
);
|
||||
|
||||
// The "static" in "StaticAssetsStore" means that assets aren't allowed to change
|
||||
expect(newFoo!.content).toEqual(Buffer.from('foo'));
|
||||
|
||||
const sameBar = await store.getAsset('bar');
|
||||
expect(oldBar!.lastModifiedAt).toEqual(sameBar!.lastModifiedAt);
|
||||
},
|
||||
60_000,
|
||||
);
|
||||
|
||||
it.each(databases.eachSupportedId())(
|
||||
'should trim old assets, %p',
|
||||
async databaseId => {
|
||||
const database = await databases.init(databaseId);
|
||||
const store = await StaticAssetsStore.create({
|
||||
logger,
|
||||
database,
|
||||
});
|
||||
|
||||
await store.storeAssets([
|
||||
{
|
||||
path: 'new',
|
||||
content: async () => Buffer.alloc(0),
|
||||
},
|
||||
{
|
||||
path: 'old',
|
||||
content: async () => Buffer.alloc(0),
|
||||
},
|
||||
]);
|
||||
|
||||
// Rewrite modified time of "old" to be 1h in the past
|
||||
const updated = await database('static_assets_cache')
|
||||
.where({ path: 'old' })
|
||||
.update({
|
||||
last_modified_at:
|
||||
database.client.config.client === 'sqlite3'
|
||||
? database.raw(`datetime('now', '-3600 seconds')`)
|
||||
: database.raw(`now() + interval '-3600 seconds'`),
|
||||
});
|
||||
expect(updated).toBe(1);
|
||||
|
||||
await expect(store.getAsset('new')).resolves.toBeDefined();
|
||||
await expect(store.getAsset('old')).resolves.toBeDefined();
|
||||
|
||||
await store.trimAssets({ maxAgeSeconds: 1800 });
|
||||
|
||||
await expect(store.getAsset('new')).resolves.toBeDefined();
|
||||
await expect(store.getAsset('old')).resolves.toBeUndefined();
|
||||
},
|
||||
60_000,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,152 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import { resolvePackagePath } from '@backstage/backend-common';
|
||||
import { Knex } from 'knex';
|
||||
import { Logger } from 'winston';
|
||||
import { DateTime } from 'luxon';
|
||||
import partition from 'lodash/partition';
|
||||
|
||||
const migrationsDir = resolvePackagePath(
|
||||
'@backstage/plugin-app-backend',
|
||||
'migrations',
|
||||
);
|
||||
|
||||
export interface StaticAssetInput {
|
||||
path: string;
|
||||
content(): Promise<Buffer>;
|
||||
}
|
||||
|
||||
export interface StaticAsset {
|
||||
path: string;
|
||||
content: Buffer;
|
||||
lastModifiedAt: Date;
|
||||
}
|
||||
|
||||
interface StaticAssetRow {
|
||||
path: string;
|
||||
content: Buffer;
|
||||
last_modified_at: Date;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export interface StaticAssetsStoreOptions {
|
||||
database: Knex;
|
||||
logger: Logger;
|
||||
}
|
||||
|
||||
/**
|
||||
* A storage for static assets that are assumed to be immutable.
|
||||
*
|
||||
* @internal
|
||||
*/
|
||||
export class StaticAssetsStore {
|
||||
#db: Knex;
|
||||
#logger: Logger;
|
||||
|
||||
static async create(options: StaticAssetsStoreOptions) {
|
||||
await options.database.migrate.latest({
|
||||
directory: migrationsDir,
|
||||
});
|
||||
return new StaticAssetsStore(options);
|
||||
}
|
||||
|
||||
private constructor(options: StaticAssetsStoreOptions) {
|
||||
this.#db = options.database;
|
||||
this.#logger = options.logger;
|
||||
}
|
||||
|
||||
/**
|
||||
* Store the provided assets.
|
||||
*
|
||||
* If an asset for a given path already exists the modification time will be
|
||||
* updated, but the contents will not.
|
||||
*/
|
||||
async storeAssets(assets: StaticAssetInput[]) {
|
||||
const existingRows = await this.#db<StaticAssetRow>(
|
||||
'static_assets_cache',
|
||||
).whereIn(
|
||||
'path',
|
||||
assets.map(a => a.path),
|
||||
);
|
||||
const existingAssetPaths = new Set(existingRows.map(r => r.path));
|
||||
|
||||
const [modified, added] = partition(assets, asset =>
|
||||
existingAssetPaths.has(asset.path),
|
||||
);
|
||||
|
||||
this.#logger.info(
|
||||
`Storing ${modified.length} updated assets and ${added.length} new assets`,
|
||||
);
|
||||
|
||||
await this.#db('static_assets_cache')
|
||||
.update({
|
||||
last_modified_at: this.#db.fn.now(),
|
||||
})
|
||||
.whereIn(
|
||||
'path',
|
||||
modified.map(a => a.path),
|
||||
);
|
||||
|
||||
for (const asset of added) {
|
||||
// We ignore conflicts with other nodes, it doesn't matter if someone else
|
||||
// added the same asset just before us.
|
||||
await this.#db('static_assets_cache')
|
||||
.insert({
|
||||
path: asset.path,
|
||||
content: await asset.content(),
|
||||
})
|
||||
.onConflict('path')
|
||||
.ignore();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve an asset from the store with the given path.
|
||||
*/
|
||||
async getAsset(path: string): Promise<StaticAsset | undefined> {
|
||||
const [row] = await this.#db<StaticAssetRow>('static_assets_cache').where({
|
||||
path,
|
||||
});
|
||||
if (!row) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
path: row.path,
|
||||
content: row.content,
|
||||
lastModifiedAt:
|
||||
typeof row.last_modified_at === 'string'
|
||||
? DateTime.fromSQL(row.last_modified_at, { zone: 'UTC' }).toJSDate()
|
||||
: row.last_modified_at,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete any assets from the store whose modification time is older than the max age.
|
||||
*/
|
||||
async trimAssets(options: { maxAgeSeconds: number }) {
|
||||
const { maxAgeSeconds } = options;
|
||||
await this.#db<StaticAssetRow>('static_assets_cache')
|
||||
.where(
|
||||
'last_modified_at',
|
||||
'<=',
|
||||
this.#db.client.config.client === 'sqlite3'
|
||||
? this.#db.raw(`datetime('now', ?)`, [`-${maxAgeSeconds} seconds`])
|
||||
: this.#db.raw(`now() + interval '${-maxAgeSeconds} seconds'`),
|
||||
)
|
||||
.delete();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
/*
|
||||
* Copyright 2021 The Backstage Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
export { StaticAssetsStore } from './StaticAssetsStore';
|
||||
export type { StaticAsset, StaticAssetInput } from './StaticAssetsStore';
|
||||
Reference in New Issue
Block a user