diff --git a/.changeset/giant-snakes-watch.md b/.changeset/giant-snakes-watch.md new file mode 100644 index 0000000000..553518b678 --- /dev/null +++ b/.changeset/giant-snakes-watch.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-test-utils': patch +--- + +Skip running docker tests unless in CI diff --git a/packages/backend-test-utils/src/database/TestDatabases.ts b/packages/backend-test-utils/src/database/TestDatabases.ts index a7b08cd593..ab5846d84d 100644 --- a/packages/backend-test-utils/src/database/TestDatabases.ts +++ b/packages/backend-test-utils/src/database/TestDatabases.ts @@ -56,7 +56,11 @@ export class TestDatabases { disableDocker: isDockerDisabledForTests(), }; - const { ids, disableDocker } = Object.assign(defaultOptions, options ?? {}); + const { ids, disableDocker } = Object.assign( + {}, + defaultOptions, + options ?? {}, + ); const supportedIds = ids.filter(id => { const properties = allDatabases[id]; diff --git a/packages/backend-test-utils/src/util/isDockerDisabledForTests.ts b/packages/backend-test-utils/src/util/isDockerDisabledForTests.ts index 60d27bc154..4aeea59c50 100644 --- a/packages/backend-test-utils/src/util/isDockerDisabledForTests.ts +++ b/packages/backend-test-utils/src/util/isDockerDisabledForTests.ts @@ -15,5 +15,13 @@ */ export function isDockerDisabledForTests() { - return Boolean(process.env.BACKSTAGE_TEST_DISABLE_DOCKER); + // If we are not running in continuous integration, the default is to skip + // the (relatively heavy, long running) docker based tests. If you want to + // still run local tests for all databases, just pass either the CI=1 env + // parameter to your test runner, or individual connection strings per + // database. + return ( + Boolean(process.env.BACKSTAGE_TEST_DISABLE_DOCKER) || + !Boolean(process.env.CI) + ); } diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index ca9c95409c..6773917d91 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -65,6 +65,7 @@ "yup": "^0.29.3" }, "devDependencies": { + "@backstage/backend-test-utils": "^0.1.1", "@backstage/cli": "^0.6.14", "@backstage/test-utils": "^0.1.13", "@types/core-js": "^2.5.4", diff --git a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.test.ts b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.test.ts index 0532342c5d..5ea9de2b5b 100644 --- a/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.test.ts +++ b/plugins/catalog-backend/src/next/DefaultCatalogProcessingEngine.test.ts @@ -37,6 +37,7 @@ describe('DefaultCatalogProcessingEngine', () => { beforeEach(() => { jest.resetAllMocks(); }); + it('should process stuff', async () => { orchestrator.process.mockResolvedValue({ ok: true, diff --git a/plugins/catalog-backend/src/next/DefaultLocationStore.test.ts b/plugins/catalog-backend/src/next/DefaultLocationStore.test.ts index 382f0ccaf5..a989c1f81f 100644 --- a/plugins/catalog-backend/src/next/DefaultLocationStore.test.ts +++ b/plugins/catalog-backend/src/next/DefaultLocationStore.test.ts @@ -13,128 +13,161 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; import { v4 as uuid } from 'uuid'; import { DatabaseManager } from './database/DatabaseManager'; import { DefaultLocationStore } from './DefaultLocationStore'; -const createLocationStore = async () => { - const knex = await DatabaseManager.createTestDatabaseConnection(); - await DatabaseManager.createDatabase(knex); - const connection = { applyMutation: jest.fn() }; - const store = new DefaultLocationStore(knex); - await store.connect(connection); - return { store, connection }; -}; - describe('DefaultLocationStore', () => { - it('should do a full sync with the locations on connect', async () => { - const { connection } = await createLocationStore(); - - expect(connection.applyMutation).toHaveBeenCalledWith({ - type: 'full', - entities: [], - }); + const databases = TestDatabases.create({ + ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], }); - describe('listLocations', () => { - it('lists empty locations when there is no locations', async () => { - const { store } = await createLocationStore(); - expect(await store.listLocations()).toEqual([]); - }); + async function createLocationStore(databaseId: TestDatabaseId) { + const knex = await databases.init(databaseId); + await DatabaseManager.createDatabase(knex); + const connection = { applyMutation: jest.fn() }; + const store = new DefaultLocationStore(knex); + await store.connect(connection); + return { store, connection }; + } - it('lists locations that are added to the db', async () => { - const { store } = await createLocationStore(); - await store.createLocation({ - target: - 'https://github.com/backstage/demo/blob/master/catalog-info.yml', - type: 'url', + it.each(databases.eachSupportedId())( + 'should do a full sync with the locations on connect, %p', + async databaseId => { + const { connection } = await createLocationStore(databaseId); + + expect(connection.applyMutation).toHaveBeenCalledWith({ + type: 'full', + entities: [], }); + }, + 60_000, + ); - const listLocations = await store.listLocations(); - expect(listLocations).toHaveLength(1); - expect(listLocations).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - target: - 'https://github.com/backstage/demo/blob/master/catalog-info.yml', - type: 'url', - }), - ]), - ); - }); + describe('listLocations', () => { + it.each(databases.eachSupportedId())( + 'lists empty locations when there is no locations, %p', + async databaseId => { + const { store } = await createLocationStore(databaseId); + expect(await store.listLocations()).toEqual([]); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'lists locations that are added to the db, %p', + async databaseId => { + const { store } = await createLocationStore(databaseId); + await store.createLocation({ + target: + 'https://github.com/backstage/demo/blob/master/catalog-info.yml', + type: 'url', + }); + + const listLocations = await store.listLocations(); + expect(listLocations).toHaveLength(1); + expect(listLocations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + target: + 'https://github.com/backstage/demo/blob/master/catalog-info.yml', + type: 'url', + }), + ]), + ); + }, + 60_000, + ); }); describe('createLocation', () => { - it('throws when the location already exists', async () => { - const { store } = await createLocationStore(); - const spec = { - target: - 'https://github.com/backstage/demo/blob/master/catalog-info.yml', - type: 'url', - }; - await store.createLocation(spec); - await expect(() => store.createLocation(spec)).rejects.toThrow( - new RegExp(`Location ${spec.type}:${spec.target} already exists`), - ); - }); + it.each(databases.eachSupportedId())( + 'throws when the location already exists, %p', + async databaseId => { + const { store } = await createLocationStore(databaseId); + const spec = { + target: + 'https://github.com/backstage/demo/blob/master/catalog-info.yml', + type: 'url', + }; + await store.createLocation(spec); + await expect(() => store.createLocation(spec)).rejects.toThrow( + new RegExp(`Location ${spec.type}:${spec.target} already exists`), + ); + }, + 60_000, + ); - it('calls apply mutation when adding a new location', async () => { - const { store, connection } = await createLocationStore(); - await store.createLocation({ - target: - 'https://github.com/backstage/demo/blob/master/catalog-info.yml', - type: 'url', - }); + it.each(databases.eachSupportedId())( + 'calls apply mutation when adding a new location, %p', + async databaseId => { + const { store, connection } = await createLocationStore(databaseId); + await store.createLocation({ + target: + 'https://github.com/backstage/demo/blob/master/catalog-info.yml', + type: 'url', + }); - expect(connection.applyMutation).toHaveBeenCalledWith({ - type: 'delta', - removed: [], - added: expect.arrayContaining([ - expect.objectContaining({ - spec: { - target: - 'https://github.com/backstage/demo/blob/master/catalog-info.yml', - type: 'url', - }, - }), - ]), - }); - }); + expect(connection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + removed: [], + added: expect.arrayContaining([ + expect.objectContaining({ + spec: { + target: + 'https://github.com/backstage/demo/blob/master/catalog-info.yml', + type: 'url', + }, + }), + ]), + }); + }, + 60_000, + ); }); describe('deleteLocation', () => { - it('throws if the location does not exist', async () => { - const { store } = await createLocationStore(); - const id = uuid(); - await expect(() => store.deleteLocation(id)).rejects.toThrow( - new RegExp(`Found no location with ID ${id}`), - ); - }); + it.each(databases.eachSupportedId())( + 'throws if the location does not exist, %p', + async databaseId => { + const { store } = await createLocationStore(databaseId); + const id = uuid(); + await expect(() => store.deleteLocation(id)).rejects.toThrow( + new RegExp(`Found no location with ID ${id}`), + ); + }, + 60_000, + ); - it('calls apply mutation when adding a new location', async () => { - const { store, connection } = await createLocationStore(); + it.each(databases.eachSupportedId())( + 'calls apply mutation when adding a new location, %p', + async databaseId => { + const { store, connection } = await createLocationStore(databaseId); - const location = await store.createLocation({ - target: - 'https://github.com/backstage/demo/blob/master/catalog-info.yml', - type: 'url', - }); + const location = await store.createLocation({ + target: + 'https://github.com/backstage/demo/blob/master/catalog-info.yml', + type: 'url', + }); - await store.deleteLocation(location.id); + await store.deleteLocation(location.id); - expect(connection.applyMutation).toHaveBeenCalledWith({ - type: 'delta', - added: [], - removed: expect.arrayContaining([ - expect.objectContaining({ - spec: { - target: - 'https://github.com/backstage/demo/blob/master/catalog-info.yml', - type: 'url', - }, - }), - ]), - }); - }); + expect(connection.applyMutation).toHaveBeenCalledWith({ + type: 'delta', + added: [], + removed: expect.arrayContaining([ + expect.objectContaining({ + spec: { + target: + 'https://github.com/backstage/demo/blob/master/catalog-info.yml', + type: 'url', + }, + }), + ]), + }); + }, + 60_000, + ); }); }); diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts index 4a390058b8..7d98a32377 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts @@ -13,8 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -// import { DefaultProcessingDatabase } from './DefaultProcessingDatabase'; + import { getVoidLogger } from '@backstage/backend-common'; +import { TestDatabaseId, TestDatabases } from '@backstage/backend-test-utils'; import { Entity, stringifyEntityRef } from '@backstage/catalog-model'; import { JsonObject } from '@backstage/config'; import { Knex } from 'knex'; @@ -28,27 +29,30 @@ import { } from './tables'; describe('Default Processing Database', () => { - let db: Knex; - let processingDatabase: DefaultProcessingDatabase; const logger = getVoidLogger(); + const databases = TestDatabases.create({ + ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + }); - const insertRefRow = async (ref: DbRefreshStateReferencesRow) => { + async function createDatabase(databaseId: TestDatabaseId) { + const knex = await databases.init(databaseId); + await DatabaseManager.createDatabase(knex); + return { + knex, + db: new DefaultProcessingDatabase(knex, logger), + }; + } + + const insertRefRow = async (db: Knex, ref: DbRefreshStateReferencesRow) => { return db('refresh_state_references').insert( ref, ); }; - const insertRefreshStateRow = async (ref: DbRefreshStateRow) => { + const insertRefreshStateRow = async (db: Knex, ref: DbRefreshStateRow) => { await db('refresh_state').insert(ref); }; - beforeEach(async () => { - db = await DatabaseManager.createTestDatabaseConnection(); - await DatabaseManager.createDatabase(db); - - processingDatabase = new DefaultProcessingDatabase(db, logger); - }); - describe('updateProcessedEntity', () => { let id: string; let processedEntity: Entity; @@ -68,146 +72,170 @@ describe('Default Processing Database', () => { }; }); - it('fails when there is no processing state for the entity', async () => { - await processingDatabase.transaction(async tx => { - await expect(() => - processingDatabase.updateProcessedEntity(tx, { + it.each(databases.eachSupportedId())( + 'fails when there is no processing state for the entity, %p', + async databaseId => { + const { db } = await createDatabase(databaseId); + await db.transaction(async tx => { + await expect(() => + db.updateProcessedEntity(tx, { + id, + processedEntity, + state: new Map(), + relations: [], + deferredEntities: [], + }), + ).rejects.toThrow(`Processing state not found for ${id}`); + }); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'updates the refresh state entry with the cache, processed entity and errors, %p', + async databaseId => { + const { knex, db } = await createDatabase(databaseId); + await insertRefreshStateRow(knex, { + entity_id: id, + entity_ref: 'location:default/fakelocation', + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '[]', + next_update_at: '2021-04-01 13:37:00', + last_discovery_at: '2021-04-01 13:37:00', + }); + + const state = new Map(); + state.set('hello', { t: 'something' }); + + await db.transaction(tx => + db.updateProcessedEntity(tx, { + id, + processedEntity, + state, + relations: [], + deferredEntities: [], + errors: "['something broke']", + }), + ); + + const entities = await knex( + 'refresh_state', + ).select(); + expect(entities.length).toBe(1); + expect(entities[0].processed_entity).toEqual( + JSON.stringify(processedEntity), + ); + expect(entities[0].cache).toEqual(JSON.stringify(state)); + expect(entities[0].errors).toEqual("['something broke']"); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'removes old relations and stores the new relationships, %p', + async databaseId => { + const { knex, db } = await createDatabase(databaseId); + await insertRefreshStateRow(knex, { + entity_id: id, + entity_ref: 'location:default/fakelocation', + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '[]', + next_update_at: '2021-04-01 13:37:00', + last_discovery_at: '2021-04-01 13:37:00', + }); + + const relations = [ + { + source: { + kind: 'Component', + namespace: 'Default', + name: 'foo', + }, + target: { + kind: 'Component', + namespace: 'Default', + name: 'foo', + }, + type: 'memberOf', + }, + ]; + + await db.transaction(tx => + db.updateProcessedEntity(tx, { + id, + processedEntity, + state: new Map(), + relations: relations, + deferredEntities: [], + }), + ); + + const savedRelations = await knex('relations') + .where({ originating_entity_id: id }) + .select(); + expect(savedRelations.length).toBe(1); + expect(savedRelations[0]).toEqual({ + originating_entity_id: id, + source_entity_ref: 'component:default/foo', + type: 'memberOf', + target_entity_ref: 'component:default/foo', + }); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'adds deferred entities to the the refresh_state table to be picked up later, %p', + async databaseId => { + const { knex, db } = await createDatabase(databaseId); + await insertRefreshStateRow(knex, { + entity_id: id, + entity_ref: 'location:default/fakelocation', + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '[]', + next_update_at: '2021-04-01 13:37:00', + last_discovery_at: '2021-04-01 13:37:00', + }); + + const deferredEntities = [ + { + apiVersion: '1', + kind: 'Location', + metadata: { + name: 'next', + }, + }, + ]; + + await db.transaction(tx => + db.updateProcessedEntity(tx, { id, processedEntity, state: new Map(), relations: [], - deferredEntities: [], + deferredEntities, }), - ).rejects.toThrow(`Processing state not found for ${id}`); - }); - }); + ); - it('updates the refresh state entry with the cache, processed entity and errors', async () => { - await insertRefreshStateRow({ - entity_id: id, - entity_ref: 'location:default/fakelocation', - unprocessed_entity: '{}', - processed_entity: '{}', - errors: '[]', - next_update_at: '2021-04-01 13:37:00', - last_discovery_at: '2021-04-01 13:37:00', - }); + const refreshStateEntries = await knex( + 'refresh_state', + ) + .where({ entity_ref: stringifyEntityRef(deferredEntities[0]) }) + .select(); - const state = new Map(); - state.set('hello', { t: 'something' }); - - await processingDatabase.transaction(tx => - processingDatabase.updateProcessedEntity(tx, { - id, - processedEntity, - state, - relations: [], - deferredEntities: [], - errors: "['something broke']", - }), - ); - - const entities = await db('refresh_state').select(); - expect(entities.length).toBe(1); - expect(entities[0].processed_entity).toEqual( - JSON.stringify(processedEntity), - ); - expect(entities[0].cache).toEqual(JSON.stringify(state)); - expect(entities[0].errors).toEqual("['something broke']"); - }); - - it('removes old relations and stores the new relationships', async () => { - await insertRefreshStateRow({ - entity_id: id, - entity_ref: 'location:default/fakelocation', - unprocessed_entity: '{}', - processed_entity: '{}', - errors: '[]', - next_update_at: '2021-04-01 13:37:00', - last_discovery_at: '2021-04-01 13:37:00', - }); - - const relations = [ - { - source: { - kind: 'Component', - namespace: 'Default', - name: 'foo', - }, - target: { - kind: 'Component', - namespace: 'Default', - name: 'foo', - }, - type: 'memberOf', - }, - ]; - - await processingDatabase.transaction(tx => - processingDatabase.updateProcessedEntity(tx, { - id, - processedEntity, - state: new Map(), - relations: relations, - deferredEntities: [], - }), - ); - - const savedRelations = await db('relations') - .where({ originating_entity_id: id }) - .select(); - expect(savedRelations.length).toBe(1); - expect(savedRelations[0]).toEqual({ - originating_entity_id: id, - source_entity_ref: 'component:default/foo', - type: 'memberOf', - target_entity_ref: 'component:default/foo', - }); - }); - - it('adds deferred entities to the the refresh_state table to be picked up later', async () => { - await insertRefreshStateRow({ - entity_id: id, - entity_ref: 'location:default/fakelocation', - unprocessed_entity: '{}', - processed_entity: '{}', - errors: '[]', - next_update_at: '2021-04-01 13:37:00', - last_discovery_at: '2021-04-01 13:37:00', - }); - - const deferredEntities = [ - { - apiVersion: '1', - kind: 'Location', - metadata: { - name: 'next', - }, - }, - ]; - - await processingDatabase.transaction(tx => - processingDatabase.updateProcessedEntity(tx, { - id, - processedEntity, - state: new Map(), - relations: [], - deferredEntities, - }), - ); - - const refreshStateEntries = await db('refresh_state') - .where({ entity_ref: stringifyEntityRef(deferredEntities[0]) }) - .select(); - - expect(refreshStateEntries).toHaveLength(1); - }); + expect(refreshStateEntries).toHaveLength(1); + }, + 60_000, + ); }); describe('replaceUnprocessedEntities', () => { - const createLocations = async (entityRefs: string[]) => { + const createLocations = async (db: Knex, entityRefs: string[]) => { for (const ref of entityRefs) { - await insertRefreshStateRow({ + await insertRefreshStateRow(db, { entity_id: uuid.v4(), entity_ref: ref, unprocessed_entity: '{}', @@ -219,416 +247,455 @@ describe('Default Processing Database', () => { } }; - it('replaces all existing state correctly for simple dependency chains', async () => { - /* + it.each(databases.eachSupportedId())( + 'replaces all existing state correctly for simple dependency chains, %p', + async databaseId => { + const { knex, db } = await createDatabase(databaseId); + /* config -> location:default/root -> location:default/root-1 -> location:default/root-2 database -> location:default/second -> location:default/root-2 - */ - await createLocations([ - 'location:default/root', - 'location:default/root-1', - 'location:default/root-2', - 'location:default/second', - ]); + */ + await createLocations(knex, [ + 'location:default/root', + 'location:default/root-1', + 'location:default/root-2', + 'location:default/second', + ]); - await insertRefRow({ - source_key: 'config', - target_entity_ref: 'location:default/root', - }); + await insertRefRow(knex, { + source_key: 'config', + target_entity_ref: 'location:default/root', + }); - await insertRefRow({ - source_key: 'database', - target_entity_ref: 'location:default/second', - }); + await insertRefRow(knex, { + source_key: 'database', + target_entity_ref: 'location:default/second', + }); - await insertRefRow({ - source_entity_ref: 'location:default/root', - target_entity_ref: 'location:default/root-1', - }); + await insertRefRow(knex, { + source_entity_ref: 'location:default/root', + target_entity_ref: 'location:default/root-1', + }); - await insertRefRow({ - source_entity_ref: 'location:default/root-1', - target_entity_ref: 'location:default/root-2', - }); + await insertRefRow(knex, { + source_entity_ref: 'location:default/root-1', + target_entity_ref: 'location:default/root-2', + }); - await insertRefRow({ - source_entity_ref: 'location:default/second', - target_entity_ref: 'location:default/root-2', - }); + await insertRefRow(knex, { + source_entity_ref: 'location:default/second', + target_entity_ref: 'location:default/root-2', + }); - await processingDatabase.transaction(tx => - processingDatabase.replaceUnprocessedEntities(tx, { - type: 'full', - sourceKey: 'config', - items: [ - { - apiVersion: '1.0.0', - metadata: { - name: 'new-root', - }, - kind: 'Location', - } as Entity, - ], - }), - ); + await db.transaction(tx => + db.replaceUnprocessedEntities(tx, { + type: 'full', + sourceKey: 'config', + items: [ + { + apiVersion: '1.0.0', + metadata: { + name: 'new-root', + }, + kind: 'Location', + } as Entity, + ], + }), + ); - const currentRefreshState = await db( - 'refresh_state', - ).select(); + const currentRefreshState = await knex( + 'refresh_state', + ).select(); - const currentRefRowState = await db( - 'refresh_state_references', - ).select(); + const currentRefRowState = await knex( + 'refresh_state_references', + ).select(); - for (const ref of ['location:default/root', 'location:default/root-1']) { - expect(currentRefreshState.some(t => t.entity_ref === ref)).toBeFalsy(); - } + for (const ref of [ + 'location:default/root', + 'location:default/root-1', + ]) { + expect( + currentRefreshState.some(t => t.entity_ref === ref), + ).toBeFalsy(); + } - expect( - currentRefreshState.some( - t => t.entity_ref === 'location:default/new-root', - ), - ).toBeTruthy(); + expect( + currentRefreshState.some( + t => t.entity_ref === 'location:default/new-root', + ), + ).toBeTruthy(); - expect( - currentRefRowState.some( - t => - t.source_entity_ref === 'location:default/root' && - t.target_entity_ref === 'location:default/root-1', - ), - ).toBeFalsy(); + expect( + currentRefRowState.some( + t => + t.source_entity_ref === 'location:default/root' && + t.target_entity_ref === 'location:default/root-1', + ), + ).toBeFalsy(); - expect( - currentRefRowState.some( - t => - t.source_entity_ref === 'location:default/root-1' && - t.target_entity_ref === 'location:default/root-2', - ), - ).toBeFalsy(); + expect( + currentRefRowState.some( + t => + t.source_entity_ref === 'location:default/root-1' && + t.target_entity_ref === 'location:default/root-2', + ), + ).toBeFalsy(); - expect( - currentRefRowState.some( - t => - t.target_entity_ref === 'location:default/root-1' && - t.source_key === 'config', - ), - ).toBeFalsy(); + expect( + currentRefRowState.some( + t => + t.target_entity_ref === 'location:default/root-1' && + t.source_key === 'config', + ), + ).toBeFalsy(); - expect( - currentRefRowState.some( - t => - t.target_entity_ref === 'location:default/new-root' && - t.source_key === 'config', - ), - ).toBeTruthy(); - }); + expect( + currentRefRowState.some( + t => + t.target_entity_ref === 'location:default/new-root' && + t.source_key === 'config', + ), + ).toBeTruthy(); + }, + 60_000, + ); - it('should work for more complex chains', async () => { - /* + it.each(databases.eachSupportedId())( + 'should work for more complex chains, %p', + async databaseId => { + const { knex, db } = await createDatabase(databaseId); + /* config -> location:default/root -> location:default/root-1 -> location:default/root-2 config -> location:default/root -> location:default/root-1a -> location:default/root-2 */ - await createLocations([ - 'location:default/root', - 'location:default/root-1', - 'location:default/root-2', - 'location:default/root-1a', - ]); + await createLocations(knex, [ + 'location:default/root', + 'location:default/root-1', + 'location:default/root-2', + 'location:default/root-1a', + ]); - await insertRefRow({ - source_key: 'config', - target_entity_ref: 'location:default/root', - }); - - await insertRefRow({ - source_entity_ref: 'location:default/root', - target_entity_ref: 'location:default/root-1', - }); - - await insertRefRow({ - source_entity_ref: 'location:default/root', - target_entity_ref: 'location:default/root-1a', - }); - - await insertRefRow({ - source_entity_ref: 'location:default/root-1', - target_entity_ref: 'location:default/root-2', - }); - - await insertRefRow({ - source_entity_ref: 'location:default/root-1a', - target_entity_ref: 'location:default/root-2', - }); - - await processingDatabase.transaction(async tx => { - await processingDatabase.replaceUnprocessedEntities(tx, { - type: 'full', - sourceKey: 'config', - items: [ - { - apiVersion: '1.0.0', - metadata: { - name: 'new-root', - }, - kind: 'Location', - } as Entity, - ], + await insertRefRow(knex, { + source_key: 'config', + target_entity_ref: 'location:default/root', }); - }); - const currentRefreshState = await db( - 'refresh_state', - ).select(); - - const currentRefRowState = await db( - 'refresh_state_references', - ).select(); - - const deletedRefs = [ - 'location:default/root', - 'location:default/root-1', - 'location:default/root-1a', - 'location:default/root-2', - ]; - - for (const ref of deletedRefs) { - expect(currentRefreshState.some(t => t.entity_ref === ref)).toBeFalsy(); - } - - expect( - currentRefreshState.some( - t => t.entity_ref === 'location:default/new-root', - ), - ).toBeTruthy(); - - expect( - currentRefRowState.some( - t => - t.source_key === 'config' && - t.target_entity_ref === 'location:default/new-root', - ), - ).toBeTruthy(); - - expect( - currentRefRowState.some( - t => - t.source_key === 'config' && - t.target_entity_ref === 'location:default/root', - ), - ).toBeFalsy(); - - expect( - currentRefRowState.some( - t => - t.source_entity_ref === 'location:default/root' && - t.target_entity_ref === 'location:default/root-1', - ), - ).toBeFalsy(); - - expect( - currentRefRowState.some( - t => - t.source_entity_ref === 'location:default/root' && - t.target_entity_ref === 'location:default/root-1a', - ), - ).toBeFalsy(); - - expect( - currentRefRowState.some( - t => - t.source_entity_ref === 'location:default/root-1' && - t.target_entity_ref === 'location:default/root-2', - ), - ).toBeFalsy(); - - expect( - currentRefRowState.some( - t => - t.source_entity_ref === 'location:default/root-1a' && - t.target_entity_ref === 'location:default/root-2', - ), - ).toBeFalsy(); - }); - - it('should add new locations using the delta options', async () => { - await processingDatabase.transaction(async tx => { - await processingDatabase.replaceUnprocessedEntities(tx, { - type: 'delta', - sourceKey: 'lols', - removed: [], - added: [ - { - apiVersion: '1.0.0', - metadata: { - name: 'new-root', - }, - kind: 'Location', - } as Entity, - ], + await insertRefRow(knex, { + source_entity_ref: 'location:default/root', + target_entity_ref: 'location:default/root-1', }); - }); - const currentRefreshState = await db( - 'refresh_state', - ).select(); + await insertRefRow(knex, { + source_entity_ref: 'location:default/root', + target_entity_ref: 'location:default/root-1a', + }); - const currentRefRowState = await db( - 'refresh_state_references', - ).select(); + await insertRefRow(knex, { + source_entity_ref: 'location:default/root-1', + target_entity_ref: 'location:default/root-2', + }); - expect( - currentRefreshState.some( - t => t.entity_ref === 'location:default/new-root', - ), - ).toBeTruthy(); + await insertRefRow(knex, { + source_entity_ref: 'location:default/root-1a', + target_entity_ref: 'location:default/root-2', + }); - expect( - currentRefRowState.some( - t => - t.source_key === 'lols' && - t.target_entity_ref === 'location:default/new-root', - ), - ).toBeTruthy(); - }); + await db.transaction(async tx => { + await db.replaceUnprocessedEntities(tx, { + type: 'full', + sourceKey: 'config', + items: [ + { + apiVersion: '1.0.0', + metadata: { + name: 'new-root', + }, + kind: 'Location', + } as Entity, + ], + }); + }); - it('should not remove locations that are referenced elsewhere', async () => { - /* + const currentRefreshState = await knex( + 'refresh_state', + ).select(); + + const currentRefRowState = await knex( + 'refresh_state_references', + ).select(); + + const deletedRefs = [ + 'location:default/root', + 'location:default/root-1', + 'location:default/root-1a', + 'location:default/root-2', + ]; + + for (const ref of deletedRefs) { + expect( + currentRefreshState.some(t => t.entity_ref === ref), + ).toBeFalsy(); + } + + expect( + currentRefreshState.some( + t => t.entity_ref === 'location:default/new-root', + ), + ).toBeTruthy(); + + expect( + currentRefRowState.some( + t => + t.source_key === 'config' && + t.target_entity_ref === 'location:default/new-root', + ), + ).toBeTruthy(); + + expect( + currentRefRowState.some( + t => + t.source_key === 'config' && + t.target_entity_ref === 'location:default/root', + ), + ).toBeFalsy(); + + expect( + currentRefRowState.some( + t => + t.source_entity_ref === 'location:default/root' && + t.target_entity_ref === 'location:default/root-1', + ), + ).toBeFalsy(); + + expect( + currentRefRowState.some( + t => + t.source_entity_ref === 'location:default/root' && + t.target_entity_ref === 'location:default/root-1a', + ), + ).toBeFalsy(); + + expect( + currentRefRowState.some( + t => + t.source_entity_ref === 'location:default/root-1' && + t.target_entity_ref === 'location:default/root-2', + ), + ).toBeFalsy(); + + expect( + currentRefRowState.some( + t => + t.source_entity_ref === 'location:default/root-1a' && + t.target_entity_ref === 'location:default/root-2', + ), + ).toBeFalsy(); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'should add new locations using the delta options, %p', + async databaseId => { + const { knex, db } = await createDatabase(databaseId); + await db.transaction(async tx => { + await db.replaceUnprocessedEntities(tx, { + type: 'delta', + sourceKey: 'lols', + removed: [], + added: [ + { + apiVersion: '1.0.0', + metadata: { + name: 'new-root', + }, + kind: 'Location', + } as Entity, + ], + }); + }); + + const currentRefreshState = await knex( + 'refresh_state', + ).select(); + + const currentRefRowState = await knex( + 'refresh_state_references', + ).select(); + + expect( + currentRefreshState.some( + t => t.entity_ref === 'location:default/new-root', + ), + ).toBeTruthy(); + + expect( + currentRefRowState.some( + t => + t.source_key === 'lols' && + t.target_entity_ref === 'location:default/new-root', + ), + ).toBeTruthy(); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'should not remove locations that are referenced elsewhere, %p', + async databaseId => { + const { knex, db } = await createDatabase(databaseId); + /* config-1 -> location:default/root config-2 -> location:default/root */ - await createLocations(['location:default/root']); + await createLocations(knex, ['location:default/root']); - await insertRefRow({ - source_key: 'config-1', - target_entity_ref: 'location:default/root', - }); - await insertRefRow({ - source_key: 'config-2', - target_entity_ref: 'location:default/root', - }); - - await processingDatabase.transaction(async tx => { - await processingDatabase.replaceUnprocessedEntities(tx, { - type: 'full', - sourceKey: 'config-1', - items: [], + await insertRefRow(knex, { + source_key: 'config-1', + target_entity_ref: 'location:default/root', }); - }); - - const currentRefreshState = await db( - 'refresh_state', - ).select(); - - const currentRefRowState = await db( - 'refresh_state_references', - ).select(); - - expect(currentRefRowState).toEqual([ - expect.objectContaining({ + await insertRefRow(knex, { source_key: 'config-2', target_entity_ref: 'location:default/root', - }), - ]); - - expect(currentRefreshState).toEqual([ - expect.objectContaining({ - entity_ref: 'location:default/root', - }), - ]); - }); - - it('should remove old locations using the delta options', async () => { - await createLocations(['location:default/new-root']); - - await insertRefRow({ - source_key: 'lols', - target_entity_ref: 'location:default/new-root', - }); - - await processingDatabase.transaction(async tx => { - await processingDatabase.replaceUnprocessedEntities(tx, { - type: 'delta', - sourceKey: 'lols', - added: [], - removed: [ - { - apiVersion: '1.0.0', - metadata: { - name: 'new-root', - }, - kind: 'Location', - } as Entity, - ], }); - }); - const currentRefreshState = await db( - 'refresh_state', - ).select(); + await db.transaction(async tx => { + await db.replaceUnprocessedEntities(tx, { + type: 'full', + sourceKey: 'config-1', + items: [], + }); + }); - const currentRefRowState = await db( - 'refresh_state_references', - ).select(); + const currentRefreshState = await knex( + 'refresh_state', + ).select(); - expect( - currentRefreshState.some( - t => t.entity_ref === 'location:default/new-root', - ), - ).toBeFalsy(); + const currentRefRowState = await knex( + 'refresh_state_references', + ).select(); - expect( - currentRefRowState.some( - t => - t.source_key === 'lols' && - t.target_entity_ref === 'location:default/new-root', - ), - ).toBeFalsy(); - }); + expect(currentRefRowState).toEqual([ + expect.objectContaining({ + source_key: 'config-2', + target_entity_ref: 'location:default/root', + }), + ]); + + expect(currentRefreshState).toEqual([ + expect.objectContaining({ + entity_ref: 'location:default/root', + }), + ]); + }, + 60_000, + ); + + it.each(databases.eachSupportedId())( + 'should remove old locations using the delta options, %p', + async databaseId => { + const { knex, db } = await createDatabase(databaseId); + await createLocations(knex, ['location:default/new-root']); + + await insertRefRow(knex, { + source_key: 'lols', + target_entity_ref: 'location:default/new-root', + }); + + await db.transaction(async tx => { + await db.replaceUnprocessedEntities(tx, { + type: 'delta', + sourceKey: 'lols', + added: [], + removed: [ + { + apiVersion: '1.0.0', + metadata: { + name: 'new-root', + }, + kind: 'Location', + } as Entity, + ], + }); + }); + + const currentRefreshState = await knex( + 'refresh_state', + ).select(); + + const currentRefRowState = await knex( + 'refresh_state_references', + ).select(); + + expect( + currentRefreshState.some( + t => t.entity_ref === 'location:default/new-root', + ), + ).toBeFalsy(); + + expect( + currentRefRowState.some( + t => + t.source_key === 'lols' && + t.target_entity_ref === 'location:default/new-root', + ), + ).toBeFalsy(); + }, + 60_000, + ); }); describe('getProcessableEntities', () => { - it('should return entities to process', async () => { - const entity = JSON.stringify({ - kind: 'Location', - apiVersion: '1.0.0', - metadata: { - name: 'xyz', - }, - } as Entity); + it.each(databases.eachSupportedId())( + 'should return entities to process, %p', + async databaseId => { + const { knex, db } = await createDatabase(databaseId); + const entity = JSON.stringify({ + kind: 'Location', + apiVersion: '1.0.0', + metadata: { + name: 'xyz', + }, + } as Entity); - await db('refresh_state').insert({ - entity_id: '2', - entity_ref: 'location:default/new-root', - unprocessed_entity: entity, - errors: '[]', - next_update_at: '2019-01-01 23:00:00', - last_discovery_at: '2021-04-01 13:37:00', - }); - - await db('refresh_state').insert({ - entity_id: '1', - entity_ref: 'location:default/foobar', - unprocessed_entity: entity, - errors: '[]', - next_update_at: '2042-01-01 23:00:00', - last_discovery_at: '2021-04-01 13:37:00', - }); - - await processingDatabase.transaction(async tx => { - // request two items but only one can be processed. - const result = await processingDatabase.getProcessableEntities(tx, { - processBatchSize: 2, + await knex('refresh_state').insert({ + entity_id: '2', + entity_ref: 'location:default/new-root', + unprocessed_entity: entity, + errors: '[]', + next_update_at: '2019-01-01 23:00:00', + last_discovery_at: '2021-04-01 13:37:00', }); - expect(result.items.length).toEqual(1); - expect(result.items[0].entityRef).toEqual('location:default/new-root'); - // should not return the same item as there's nothing left to process. - await expect( - processingDatabase.getProcessableEntities(tx, { + await knex('refresh_state').insert({ + entity_id: '1', + entity_ref: 'location:default/foobar', + unprocessed_entity: entity, + errors: '[]', + next_update_at: '2042-01-01 23:00:00', + last_discovery_at: '2021-04-01 13:37:00', + }); + + await db.transaction(async tx => { + // request two items but only one can be processed. + const result = await db.getProcessableEntities(tx, { processBatchSize: 2, - }), - ).resolves.toEqual({ items: [] }); - }); - }); + }); + expect(result.items.length).toEqual(1); + expect(result.items[0].entityRef).toEqual( + 'location:default/new-root', + ); + + // should not return the same item as there's nothing left to process. + await expect( + db.getProcessableEntities(tx, { + processBatchSize: 2, + }), + ).resolves.toEqual({ items: [] }); + }); + }, + 60_000, + ); }); }); diff --git a/plugins/catalog-backend/src/next/stitching/Stitcher.test.ts b/plugins/catalog-backend/src/next/stitching/Stitcher.test.ts index 89f1fda0b3..81c55c4c50 100644 --- a/plugins/catalog-backend/src/next/stitching/Stitcher.test.ts +++ b/plugins/catalog-backend/src/next/stitching/Stitcher.test.ts @@ -15,8 +15,8 @@ */ import { getVoidLogger } from '@backstage/backend-common'; +import { TestDatabases } from '@backstage/backend-test-utils'; import { Entity } from '@backstage/catalog-model'; -import { Knex } from 'knex'; import { DatabaseManager } from '../database/DatabaseManager'; import { DbFinalEntitiesRow, @@ -28,171 +28,175 @@ import { import { Stitcher } from './Stitcher'; describe('Stitcher', () => { - let db: Knex; + const databases = TestDatabases.create({ + ids: ['POSTGRES_13', 'POSTGRES_9', 'SQLITE_3'], + }); const logger = getVoidLogger(); - beforeEach(async () => { - db = await DatabaseManager.createTestDatabaseConnection(); - await DatabaseManager.createDatabase(db); - }); + it.each(databases.eachSupportedId())( + 'runs the happy path for %p', + async databaseId => { + const db = await databases.init(databaseId); + await DatabaseManager.createDatabase(db); - it('runs the happy path', async () => { - const stitcher = new Stitcher(db, logger); - let entities: DbFinalEntitiesRow[]; - let entity: Entity; + const stitcher = new Stitcher(db, logger); + let entities: DbFinalEntitiesRow[]; + let entity: Entity; - await db('refresh_state').insert([ - { - entity_id: 'my-id', - entity_ref: 'k:ns/n', - unprocessed_entity: JSON.stringify({}), - processed_entity: JSON.stringify({ - apiVersion: 'a', - kind: 'k', - metadata: { - name: 'n', - namespace: 'ns', - }, - spec: { - k: 'v', - }, - }), - errors: '[]', - next_update_at: db.fn.now(), - last_discovery_at: db.fn.now(), - }, - ]); - await db('refresh_state_references').insert([ - { source_key: 'a', target_entity_ref: 'k:ns/n' }, - ]); - await db('relations').insert([ - { - originating_entity_id: 'my-id', - source_entity_ref: 'k:ns/n', - type: 'looksAt', - target_entity_ref: 'k:ns/other', - }, - ]); - - await stitcher.stitch(new Set(['k:ns/n'])); - - entities = await db('final_entities'); - - expect(entities.length).toBe(1); - entity = JSON.parse(entities[0].final_entity!); - expect(entity).toEqual({ - relations: [ + await db('refresh_state').insert([ { - type: 'looksAt', - target: { + entity_id: 'my-id', + entity_ref: 'k:ns/n', + unprocessed_entity: JSON.stringify({}), + processed_entity: JSON.stringify({ + apiVersion: 'a', kind: 'k', - namespace: 'ns', - name: 'other', - }, + metadata: { + name: 'n', + namespace: 'ns', + }, + spec: { + k: 'v', + }, + }), + errors: '[]', + next_update_at: db.fn.now(), + last_discovery_at: db.fn.now(), }, - ], - apiVersion: 'a', - kind: 'k', - metadata: { - name: 'n', - namespace: 'ns', - etag: expect.any(String), - generation: 1, - uid: 'my-id', - }, - spec: { - k: 'v', - }, - }); - - expect(entity.metadata.etag).toEqual(entities[0].hash); - const firstHash = entities[0].hash; - - const search = await db('search'); - expect(search).toEqual( - expect.arrayContaining([ - { entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/other' }, - { entity_id: 'my-id', key: 'apiversion', value: 'a' }, - { entity_id: 'my-id', key: 'kind', value: 'k' }, - { entity_id: 'my-id', key: 'metadata.name', value: 'n' }, - { entity_id: 'my-id', key: 'metadata.namespace', value: 'ns' }, - { entity_id: 'my-id', key: 'metadata.uid', value: 'my-id' }, - { entity_id: 'my-id', key: 'spec.k', value: 'v' }, - ]), - ); - - // Re-stitch without any changes - await stitcher.stitch(new Set(['k:ns/n'])); - - entities = await db('final_entities'); - expect(entities.length).toBe(1); - entity = JSON.parse(entities[0].final_entity!); - expect(entities[0].hash).toEqual(firstHash); - expect(entity.metadata.etag).toEqual(firstHash); - - // Now add one more relation and re-stitch - await db('relations').insert([ - { - originating_entity_id: 'my-id', - source_entity_ref: 'k:ns/n', - type: 'looksAt', - target_entity_ref: 'k:ns/third', - }, - ]); - - await stitcher.stitch(new Set(['k:ns/n'])); - - entities = await db('final_entities'); - - expect(entities.length).toBe(1); - entity = JSON.parse(entities[0].final_entity!); - expect(entity).toEqual({ - relations: expect.arrayContaining([ + ]); + await db('refresh_state_references').insert([ + { source_key: 'a', target_entity_ref: 'k:ns/n' }, + ]); + await db('relations').insert([ { + originating_entity_id: 'my-id', + source_entity_ref: 'k:ns/n', type: 'looksAt', - target: { - kind: 'k', - namespace: 'ns', - name: 'other', - }, + target_entity_ref: 'k:ns/other', }, + ]); + + await stitcher.stitch(new Set(['k:ns/n'])); + + entities = await db('final_entities'); + + expect(entities.length).toBe(1); + entity = JSON.parse(entities[0].final_entity!); + expect(entity).toEqual({ + relations: [ + { + type: 'looksAt', + target: { + kind: 'k', + namespace: 'ns', + name: 'other', + }, + }, + ], + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'n', + namespace: 'ns', + etag: expect.any(String), + generation: 1, + uid: 'my-id', + }, + spec: { + k: 'v', + }, + }); + + expect(entity.metadata.etag).toEqual(entities[0].hash); + const firstHash = entities[0].hash; + + const search = await db('search'); + expect(search).toEqual( + expect.arrayContaining([ + { entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/other' }, + { entity_id: 'my-id', key: 'apiversion', value: 'a' }, + { entity_id: 'my-id', key: 'kind', value: 'k' }, + { entity_id: 'my-id', key: 'metadata.name', value: 'n' }, + { entity_id: 'my-id', key: 'metadata.namespace', value: 'ns' }, + { entity_id: 'my-id', key: 'metadata.uid', value: 'my-id' }, + { entity_id: 'my-id', key: 'spec.k', value: 'v' }, + ]), + ); + + // Re-stitch without any changes + await stitcher.stitch(new Set(['k:ns/n'])); + + entities = await db('final_entities'); + expect(entities.length).toBe(1); + entity = JSON.parse(entities[0].final_entity!); + expect(entities[0].hash).toEqual(firstHash); + expect(entity.metadata.etag).toEqual(firstHash); + + // Now add one more relation and re-stitch + await db('relations').insert([ { + originating_entity_id: 'my-id', + source_entity_ref: 'k:ns/n', type: 'looksAt', - target: { - kind: 'k', - namespace: 'ns', - name: 'third', - }, + target_entity_ref: 'k:ns/third', }, - ]), - apiVersion: 'a', - kind: 'k', - metadata: { - name: 'n', - namespace: 'ns', - etag: expect.any(String), - generation: 1, - uid: 'my-id', - }, - spec: { - k: 'v', - }, - }); + ]); - expect(entities[0].hash).not.toEqual(firstHash); - expect(entities[0].hash).toEqual(entity.metadata.etag); + await stitcher.stitch(new Set(['k:ns/n'])); - expect(await db('search')).toEqual( - expect.arrayContaining([ - { entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/other' }, - { entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/third' }, - { entity_id: 'my-id', key: 'apiversion', value: 'a' }, - { entity_id: 'my-id', key: 'kind', value: 'k' }, - { entity_id: 'my-id', key: 'metadata.name', value: 'n' }, - { entity_id: 'my-id', key: 'metadata.namespace', value: 'ns' }, - { entity_id: 'my-id', key: 'metadata.uid', value: 'my-id' }, - { entity_id: 'my-id', key: 'spec.k', value: 'v' }, - ]), - ); - }); + entities = await db('final_entities'); + + expect(entities.length).toBe(1); + entity = JSON.parse(entities[0].final_entity!); + expect(entity).toEqual({ + relations: expect.arrayContaining([ + { + type: 'looksAt', + target: { + kind: 'k', + namespace: 'ns', + name: 'other', + }, + }, + { + type: 'looksAt', + target: { + kind: 'k', + namespace: 'ns', + name: 'third', + }, + }, + ]), + apiVersion: 'a', + kind: 'k', + metadata: { + name: 'n', + namespace: 'ns', + etag: expect.any(String), + generation: 1, + uid: 'my-id', + }, + spec: { + k: 'v', + }, + }); + + expect(entities[0].hash).not.toEqual(firstHash); + expect(entities[0].hash).toEqual(entity.metadata.etag); + + expect(await db('search')).toEqual( + expect.arrayContaining([ + { entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/other' }, + { entity_id: 'my-id', key: 'relations.looksat', value: 'k:ns/third' }, + { entity_id: 'my-id', key: 'apiversion', value: 'a' }, + { entity_id: 'my-id', key: 'kind', value: 'k' }, + { entity_id: 'my-id', key: 'metadata.name', value: 'n' }, + { entity_id: 'my-id', key: 'metadata.namespace', value: 'ns' }, + { entity_id: 'my-id', key: 'metadata.uid', value: 'my-id' }, + { entity_id: 'my-id', key: 'spec.k', value: 'v' }, + ]), + ); + }, + 60_000, + ); });