paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 47 Packages Projects Releases Wiki Activity

Merge pull request #5867 from backstage/freben/test-databases-in-catalog

Browse Source 
Use `TestDatabases` to exercise the experimental code in the catalog backend
This commit is contained in:
Fredrik Adelöw
2021-06-08 15:09:40 +02:00
committed by GitHub
parent b51b42a126 0711954a94
commit 260e31da98
8 changed files with 883 additions and 760 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/giant-snakes-watch.md
+5
View File
@@ -0,0 +1,5 @@
---
'@backstage/backend-test-utils': patch
---
Skip running docker tests unless in CI
packages/backend-test-utils/src/database/TestDatabases.ts
+5 -1
View File
@@ -56,7 +56,11 @@ export class TestDatabases {
disableDocker: isDockerDisabledForTests(),
};
const { ids, disableDocker } = Object.assign(defaultOptions, options ?? {});
const { ids, disableDocker } = Object.assign(
{},
defaultOptions,
options ?? {},
);
const supportedIds = ids.filter(id => {
const properties = allDatabases[id];
packages/backend-test-utils/src/util/isDockerDisabledForTests.ts
+9 -1
View File
@@ -15,5 +15,13 @@
*/
export function isDockerDisabledForTests() {
return Boolean(process.env.BACKSTAGE_TEST_DISABLE_DOCKER);
// If we are not running in continuous integration, the default is to skip
// the (relatively heavy, long running) docker based tests. If you want to
// still run local tests for all databases, just pass either the CI=1 env
// parameter to your test runner, or individual connection strings per
// database.
return (
Boolean(process.env.BACKSTAGE_TEST_DISABLE_DOCKER) ||
!Boolean(process.env.CI)
);
}
plugins/catalog-backend/package.json
+1
View File
@@ -65,6 +65,7 @@
"yup": "^0.29.3"
},
"devDependencies": {
"@backstage/backend-test-utils": "^0.1.1",
"@backstage/cli": "^0.6.14",
"@backstage/test-utils": "^0.1.13",
"@types/core-js": "^2.5.4",
plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.test.ts
+1
View File
@@ -37,6 +37,7 @@ describe('DefaultCatalogProcessingEngine', () => {
beforeEach(() => {
jest.resetAllMocks();
});
it('should process stuff', async () => {
orchestrator.process.mockResolvedValue({
ok: true,
plugins/catalog-backend/src/next/DefaultLocationStore.test.ts
+134 -101
View File
@@ -13,128 +13,161 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils';
import { v4 as uuid } from 'uuid';
import { DatabaseManager } from './database/DatabaseManager';
import { DefaultLocationStore } from './DefaultLocationStore';
const createLocationStore = async () => {
const knex = await DatabaseManager.createTestDatabaseConnection();
await DatabaseManager.createDatabase(knex);
const connection = { applyMutation: jest.fn() };
const store = new DefaultLocationStore(knex);
await store.connect(connection);
return { store, connection };
};
describe('DefaultLocationStore', () => {
it('should do a full sync with the locations on connect', async () => {
const { connection } = await createLocationStore();
expect(connection.applyMutation).toHaveBeenCalledWith({
type: 'full',
entities: [],
});
const databases = TestDatabases.create({
ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
});
describe('listLocations', () => {
it('lists empty locations when there is no locations', async () => {
const { store } = await createLocationStore();
expect(await store.listLocations()).toEqual([]);
});
async function createLocationStore(databaseId: TestDatabaseId) {
const knex = await databases.init(databaseId);
await DatabaseManager.createDatabase(knex);
const connection = { applyMutation: jest.fn() };
const store = new DefaultLocationStore(knex);
await store.connect(connection);
return { store, connection };
}
it('lists locations that are added to the db', async () => {
const { store } = await createLocationStore();
await store.createLocation({
target:
'https://github.com/backstage/demo/blob/master/catalog-info.yml',
type: 'url',
it.each(databases.eachSupportedId())(
'should do a full sync with the locations on connect, %p',
async databaseId => {
const { connection } = await createLocationStore(databaseId);
expect(connection.applyMutation).toHaveBeenCalledWith({
type: 'full',
entities: [],
});
},
60_000,
);
const listLocations = await store.listLocations();
expect(listLocations).toHaveLength(1);
expect(listLocations).toEqual(
expect.arrayContaining([
expect.objectContaining({
target:
'https://github.com/backstage/demo/blob/master/catalog-info.yml',
type: 'url',
}),
]),
);
});
describe('listLocations', () => {
it.each(databases.eachSupportedId())(
'lists empty locations when there is no locations, %p',
async databaseId => {
const { store } = await createLocationStore(databaseId);
expect(await store.listLocations()).toEqual([]);
},
60_000,
);
it.each(databases.eachSupportedId())(
'lists locations that are added to the db, %p',
async databaseId => {
const { store } = await createLocationStore(databaseId);
await store.createLocation({
target:
'https://github.com/backstage/demo/blob/master/catalog-info.yml',
type: 'url',
});
const listLocations = await store.listLocations();
expect(listLocations).toHaveLength(1);
expect(listLocations).toEqual(
expect.arrayContaining([
expect.objectContaining({
target:
'https://github.com/backstage/demo/blob/master/catalog-info.yml',
type: 'url',
}),
]),
);
},
60_000,
);
});
describe('createLocation', () => {
it('throws when the location already exists', async () => {
const { store } = await createLocationStore();
const spec = {
target:
'https://github.com/backstage/demo/blob/master/catalog-info.yml',
type: 'url',
};
await store.createLocation(spec);
await expect(() => store.createLocation(spec)).rejects.toThrow(
new RegExp(`Location ${spec.type}:${spec.target} already exists`),
);
});
it.each(databases.eachSupportedId())(
'throws when the location already exists, %p',
async databaseId => {
const { store } = await createLocationStore(databaseId);
const spec = {
target:
'https://github.com/backstage/demo/blob/master/catalog-info.yml',
type: 'url',
};
await store.createLocation(spec);
await expect(() => store.createLocation(spec)).rejects.toThrow(
new RegExp(`Location ${spec.type}:${spec.target} already exists`),
);
},
60_000,
);
it('calls apply mutation when adding a new location', async () => {
const { store, connection } = await createLocationStore();
await store.createLocation({
target:
'https://github.com/backstage/demo/blob/master/catalog-info.yml',
type: 'url',
});
it.each(databases.eachSupportedId())(
'calls apply mutation when adding a new location, %p',
async databaseId => {
const { store, connection } = await createLocationStore(databaseId);
await store.createLocation({
target:
'https://github.com/backstage/demo/blob/master/catalog-info.yml',
type: 'url',
});
expect(connection.applyMutation).toHaveBeenCalledWith({
type: 'delta',
removed: [],
added: expect.arrayContaining([
expect.objectContaining({
spec: {
target:
'https://github.com/backstage/demo/blob/master/catalog-info.yml',
type: 'url',
},
}),
]),
});
});
expect(connection.applyMutation).toHaveBeenCalledWith({
type: 'delta',
removed: [],
added: expect.arrayContaining([
expect.objectContaining({
spec: {
target:
'https://github.com/backstage/demo/blob/master/catalog-info.yml',
type: 'url',
},
}),
]),
});
},
60_000,
);
});
describe('deleteLocation', () => {
it('throws if the location does not exist', async () => {
const { store } = await createLocationStore();
const id = uuid();
await expect(() => store.deleteLocation(id)).rejects.toThrow(
new RegExp(`Found no location with ID ${id}`),
);
});
it.each(databases.eachSupportedId())(
'throws if the location does not exist, %p',
async databaseId => {
const { store } = await createLocationStore(databaseId);
const id = uuid();
await expect(() => store.deleteLocation(id)).rejects.toThrow(
new RegExp(`Found no location with ID ${id}`),
);
},
60_000,
);
it('calls apply mutation when adding a new location', async () => {
const { store, connection } = await createLocationStore();
it.each(databases.eachSupportedId())(
'calls apply mutation when adding a new location, %p',
async databaseId => {
const { store, connection } = await createLocationStore(databaseId);
const location = await store.createLocation({
target:
'https://github.com/backstage/demo/blob/master/catalog-info.yml',
type: 'url',
});
const location = await store.createLocation({
target:
'https://github.com/backstage/demo/blob/master/catalog-info.yml',
type: 'url',
});
await store.deleteLocation(location.id);
await store.deleteLocation(location.id);
expect(connection.applyMutation).toHaveBeenCalledWith({
type: 'delta',
added: [],
removed: expect.arrayContaining([
expect.objectContaining({
spec: {
target:
'https://github.com/backstage/demo/blob/master/catalog-info.yml',
type: 'url',
},
}),
]),
});
});
expect(connection.applyMutation).toHaveBeenCalledWith({
type: 'delta',
added: [],
removed: expect.arrayContaining([
expect.objectContaining({
spec: {
target:
'https://github.com/backstage/demo/blob/master/catalog-info.yml',
type: 'url',
},
}),
]),
});
},
60_000,
);
});
});
plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts
+571 -504
View File
File diff suppressed because it is too large Load Diff
plugins/catalog-backend/src/next/stitching/Stitcher.test.ts
+157 -153
View File
@@ -15,8 +15,8 @@
*/
import { getVoidLogger } from '@backstage/backend-common';
import { TestDatabases } from '@backstage/backend-test-utils';
import { Entity } from '@backstage/catalog-model';
import { Knex } from 'knex';
import { DatabaseManager } from '../database/DatabaseManager';
import {
DbFinalEntitiesRow,
@@ -28,171 +28,175 @@ import {
import { Stitcher } from './Stitcher';
describe('Stitcher', () => {
let db: Knex;
const databases = TestDatabases.create({
ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'],
});
const logger = getVoidLogger();
beforeEach(async () => {
db = await DatabaseManager.createTestDatabaseConnection();
await DatabaseManager.createDatabase(db);
});
it.each(databases.eachSupportedId())(
'runs the happy path for %p',
async databaseId => {
const db = await databases.init(databaseId);
await DatabaseManager.createDatabase(db);
it('runs the happy path', async () => {
const stitcher = new Stitcher(db, logger);
let entities: DbFinalEntitiesRow[];
let entity: Entity;
const stitcher = new Stitcher(db, logger);
let entities: DbFinalEntitiesRow[];
let entity: Entity;
await db<DbRefreshStateRow>('refresh_state').insert([
{
entity_id: 'my-id',
entity_ref: 'k:ns/n',
unprocessed_entity: JSON.stringify({}),
processed_entity: JSON.stringify({
apiVersion: 'a',
kind: 'k',
metadata: {
name: 'n',
namespace: 'ns',
},
spec: {
k: 'v',
},
}),
errors: '[]',
next_update_at: db.fn.now(),
last_discovery_at: db.fn.now(),
},
]);
await db<DbRefreshStateReferencesRow>('refresh_state_references').insert([
{ source_key: 'a', target_entity_ref: 'k:ns/n' },
]);
await db<DbRelationsRow>('relations').insert([
{
originating_entity_id: 'my-id',
source_entity_ref: 'k:ns/n',
type: 'looksAt',
target_entity_ref: 'k:ns/other',
},
]);
await stitcher.stitch(new Set(['k:ns/n']));
entities = await db<DbFinalEntitiesRow>('final_entities');
expect(entities.length).toBe(1);
entity = JSON.parse(entities[0].final_entity!);
expect(entity).toEqual({
relations: [
await db<DbRefreshStateRow>('refresh_state').insert([
{
type: 'looksAt',
target: {
entity_id: 'my-id',
entity_ref: 'k:ns/n',
unprocessed_entity: JSON.stringify({}),
processed_entity: JSON.stringify({
apiVersion: 'a',
kind: 'k',
namespace: 'ns',
name: 'other',
},
metadata: {
name: 'n',
namespace: 'ns',
},
spec: {
k: 'v',
},
}),
errors: '[]',
next_update_at: db.fn.now(),
last_discovery_at: db.fn.now(),
},
],
apiVersion: 'a',
kind: 'k',
metadata: {
name: 'n',
namespace: 'ns',
etag: expect.any(String),
generation: 1,
uid: 'my-id',
},
spec: {
k: 'v',
},
});
expect(entity.metadata.etag).toEqual(entities[0].hash);
const firstHash = entities[0].hash;
const search = await db<DbSearchRow>('search');
expect(search).toEqual(
expect.arrayContaining([
{ entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/other' },
{ entity_id: 'my-id', key: 'apiversion', value: 'a' },
{ entity_id: 'my-id', key: 'kind', value: 'k' },
{ entity_id: 'my-id', key: 'metadata.name', value: 'n' },
{ entity_id: 'my-id', key: 'metadata.namespace', value: 'ns' },
{ entity_id: 'my-id', key: 'metadata.uid', value: 'my-id' },
{ entity_id: 'my-id', key: 'spec.k', value: 'v' },
]),
);
// Re-stitch without any changes
await stitcher.stitch(new Set(['k:ns/n']));
entities = await db<DbFinalEntitiesRow>('final_entities');
expect(entities.length).toBe(1);
entity = JSON.parse(entities[0].final_entity!);
expect(entities[0].hash).toEqual(firstHash);
expect(entity.metadata.etag).toEqual(firstHash);
// Now add one more relation and re-stitch
await db<DbRelationsRow>('relations').insert([
{
originating_entity_id: 'my-id',
source_entity_ref: 'k:ns/n',
type: 'looksAt',
target_entity_ref: 'k:ns/third',
},
]);
await stitcher.stitch(new Set(['k:ns/n']));
entities = await db<DbFinalEntitiesRow>('final_entities');
expect(entities.length).toBe(1);
entity = JSON.parse(entities[0].final_entity!);
expect(entity).toEqual({
relations: expect.arrayContaining([
]);
await db<DbRefreshStateReferencesRow>('refresh_state_references').insert([
{ source_key: 'a', target_entity_ref: 'k:ns/n' },
]);
await db<DbRelationsRow>('relations').insert([
{
originating_entity_id: 'my-id',
source_entity_ref: 'k:ns/n',
type: 'looksAt',
target: {
kind: 'k',
namespace: 'ns',
name: 'other',
},
target_entity_ref: 'k:ns/other',
},
]);
await stitcher.stitch(new Set(['k:ns/n']));
entities = await db<DbFinalEntitiesRow>('final_entities');
expect(entities.length).toBe(1);
entity = JSON.parse(entities[0].final_entity!);
expect(entity).toEqual({
relations: [
{
type: 'looksAt',
target: {
kind: 'k',
namespace: 'ns',
name: 'other',
},
},
],
apiVersion: 'a',
kind: 'k',
metadata: {
name: 'n',
namespace: 'ns',
etag: expect.any(String),
generation: 1,
uid: 'my-id',
},
spec: {
k: 'v',
},
});
expect(entity.metadata.etag).toEqual(entities[0].hash);
const firstHash = entities[0].hash;
const search = await db<DbSearchRow>('search');
expect(search).toEqual(
expect.arrayContaining([
{ entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/other' },
{ entity_id: 'my-id', key: 'apiversion', value: 'a' },
{ entity_id: 'my-id', key: 'kind', value: 'k' },
{ entity_id: 'my-id', key: 'metadata.name', value: 'n' },
{ entity_id: 'my-id', key: 'metadata.namespace', value: 'ns' },
{ entity_id: 'my-id', key: 'metadata.uid', value: 'my-id' },
{ entity_id: 'my-id', key: 'spec.k', value: 'v' },
]),
);
// Re-stitch without any changes
await stitcher.stitch(new Set(['k:ns/n']));
entities = await db<DbFinalEntitiesRow>('final_entities');
expect(entities.length).toBe(1);
entity = JSON.parse(entities[0].final_entity!);
expect(entities[0].hash).toEqual(firstHash);
expect(entity.metadata.etag).toEqual(firstHash);
// Now add one more relation and re-stitch
await db<DbRelationsRow>('relations').insert([
{
originating_entity_id: 'my-id',
source_entity_ref: 'k:ns/n',
type: 'looksAt',
target: {
kind: 'k',
namespace: 'ns',
name: 'third',
},
target_entity_ref: 'k:ns/third',
},
]),
apiVersion: 'a',
kind: 'k',
metadata: {
name: 'n',
namespace: 'ns',
etag: expect.any(String),
generation: 1,
uid: 'my-id',
},
spec: {
k: 'v',
},
});
]);
expect(entities[0].hash).not.toEqual(firstHash);
expect(entities[0].hash).toEqual(entity.metadata.etag);
await stitcher.stitch(new Set(['k:ns/n']));
expect(await db<DbSearchRow>('search')).toEqual(
expect.arrayContaining([
{ entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/other' },
{ entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/third' },
{ entity_id: 'my-id', key: 'apiversion', value: 'a' },
{ entity_id: 'my-id', key: 'kind', value: 'k' },
{ entity_id: 'my-id', key: 'metadata.name', value: 'n' },
{ entity_id: 'my-id', key: 'metadata.namespace', value: 'ns' },
{ entity_id: 'my-id', key: 'metadata.uid', value: 'my-id' },
{ entity_id: 'my-id', key: 'spec.k', value: 'v' },
]),
);
});
entities = await db<DbFinalEntitiesRow>('final_entities');
expect(entities.length).toBe(1);
entity = JSON.parse(entities[0].final_entity!);
expect(entity).toEqual({
relations: expect.arrayContaining([
{
type: 'looksAt',
target: {
kind: 'k',
namespace: 'ns',
name: 'other',
},
},
{
type: 'looksAt',
target: {
kind: 'k',
namespace: 'ns',
name: 'third',
},
},
]),
apiVersion: 'a',
kind: 'k',
metadata: {
name: 'n',
namespace: 'ns',
etag: expect.any(String),
generation: 1,
uid: 'my-id',
},
spec: {
k: 'v',
},
});
expect(entities[0].hash).not.toEqual(firstHash);
expect(entities[0].hash).toEqual(entity.metadata.etag);
expect(await db<DbSearchRow>('search')).toEqual(
expect.arrayContaining([
{ entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/other' },
{ entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/third' },
{ entity_id: 'my-id', key: 'apiversion', value: 'a' },
{ entity_id: 'my-id', key: 'kind', value: 'k' },
{ entity_id: 'my-id', key: 'metadata.name', value: 'n' },
{ entity_id: 'my-id', key: 'metadata.namespace', value: 'ns' },
{ entity_id: 'my-id', key: 'metadata.uid', value: 'my-id' },
{ entity_id: 'my-id', key: 'spec.k', value: 'v' },
]),
);
},
60_000,
);
});