diff --git a/.changeset/curly-months-count.md b/.changeset/curly-months-count.md new file mode 100644 index 0000000000..8541c8c5c2 --- /dev/null +++ b/.changeset/curly-months-count.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-catalog-backend': patch +--- + +Fixed a bug where internal references within the catalog were broken when new entities where added through entity providers, such as registering a new location or adding one in configuration. These broken references then caused some entities to be incorrectly marked as orphaned and prevented refresh from working properly. diff --git a/.changeset/pink-windows-suffer.md b/.changeset/pink-windows-suffer.md new file mode 100644 index 0000000000..f27af1f4a3 --- /dev/null +++ b/.changeset/pink-windows-suffer.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-techdocs': patch +--- + +Added a check for the TechDocs annotation on the entity diff --git a/.changeset/smart-cooks-drop.md b/.changeset/smart-cooks-drop.md new file mode 100644 index 0000000000..5ce59f2837 --- /dev/null +++ b/.changeset/smart-cooks-drop.md @@ -0,0 +1,5 @@ +--- +'@backstage/plugin-kubernetes': patch +--- + +Added a check for the Kubernetes annotation on the entity diff --git a/packages/integration/package.json b/packages/integration/package.json index 10fba2c6b3..1866ef18de 100644 --- a/packages/integration/package.json +++ b/packages/integration/package.json @@ -42,7 +42,7 @@ "@backstage/config-loader": "^0.6.7", "@backstage/test-utils": "^0.1.17", "@types/jest": "^26.0.7", - "@types/luxon": "^1.25.0", + "@types/luxon": "^2.0.4", "msw": "^0.29.0" }, "files": [ diff --git a/packages/integration/src/github/GithubCredentialsProvider.test.ts b/packages/integration/src/github/GithubCredentialsProvider.test.ts index 9450d96ab7..0f51db85be 100644 --- a/packages/integration/src/github/GithubCredentialsProvider.test.ts +++ b/packages/integration/src/github/GithubCredentialsProvider.test.ts @@ -76,7 +76,7 @@ describe('GithubCredentialsProvider tests', () => { octokit.apps.createInstallationAccessToken.mockResolvedValueOnce({ data: { - expires_at: DateTime.local().plus({ hour: 1 }).toString(), + expires_at: DateTime.local().plus({ hours: 1 }).toString(), token: 'secret_token', }, } as RestEndpointMethodTypes['apps']['createInstallationAccessToken']['response']); @@ -120,7 +120,7 @@ describe('GithubCredentialsProvider tests', () => { octokit.apps.createInstallationAccessToken.mockResolvedValueOnce({ data: { - expires_at: DateTime.local().plus({ hour: 1 }).toString(), + expires_at: DateTime.local().plus({ hours: 1 }).toString(), token: 'secret_token', }, } as RestEndpointMethodTypes['apps']['createInstallationAccessToken']['response']); @@ -151,7 +151,7 @@ describe('GithubCredentialsProvider tests', () => { octokit.apps.createInstallationAccessToken.mockResolvedValueOnce({ data: { - expires_at: DateTime.local().plus({ hour: 1 }).toString(), + expires_at: DateTime.local().plus({ hours: 1 }).toString(), token: 'secret_token', }, } as RestEndpointMethodTypes['apps']['createInstallationAccessToken']['response']); @@ -272,7 +272,7 @@ describe('GithubCredentialsProvider tests', () => { octokit.apps.createInstallationAccessToken.mockResolvedValueOnce({ data: { - expires_at: DateTime.local().plus({ hour: 1 }).toString(), + expires_at: DateTime.local().plus({ hours: 1 }).toString(), token: 'secret_token', }, } as RestEndpointMethodTypes['apps']['createInstallationAccessToken']['response']); diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts index c10a37396b..2a28951e6b 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.test.ts @@ -66,123 +66,6 @@ describe('Default Processing Database', () => { await db('refresh_state').insert(ref); }; - describe('addUprocessedEntities', () => { - function mockEntity(name: string, type: string): Entity { - return { - apiVersion: '1', - kind: 'Component', - metadata: { - name, - }, - spec: { - type, - }, - }; - } - - it.each(databases.eachSupportedId())( - 'updates refresh state with varying location keys, %p', - async databaseId => { - const mockWarn = jest.fn(); - const { db } = await createDatabase(databaseId, { - debug: jest.fn(), - warn: mockWarn, - } as unknown as Logger); - await db.transaction(async tx => { - const knexTx = tx as Knex.Transaction; - - const steps = [ - { - locationKey: undefined, - expectedLocationKey: null, - type: 'a', - expectedType: 'a', - }, - { - locationKey: undefined, - expectedLocationKey: null, - type: 'b', - expectedType: 'b', - }, - { - locationKey: 'x', - expectedLocationKey: 'x', - type: 'c', - expectedType: 'c', - }, - { - locationKey: 'y', - expectedLocationKey: 'x', - type: 'd', - expectedType: 'c', - expectConflict: true, - }, - { - locationKey: undefined, - expectedLocationKey: 'x', - type: 'e', - expectedType: 'c', - expectConflict: true, - }, - { - locationKey: 'x', - expectedLocationKey: 'x', - type: 'f', - expectedType: 'f', - }, - ]; - for (const step of steps) { - mockWarn.mockClear(); - - await db.addUnprocessedEntities(tx, { - sourceKey: 'testing', - entities: [ - { - entity: mockEntity('1', step.type), - locationKey: step.locationKey, - }, - ], - }); - - if (step.expectConflict) { - // eslint-disable-next-line jest/no-conditional-expect - expect(mockWarn).toHaveBeenCalledWith( - expect.stringMatching(/^Detected conflicting entityRef/), - ); - } else { - // eslint-disable-next-line jest/no-conditional-expect - expect(mockWarn).not.toHaveBeenCalled(); - } - - const entities = await knexTx( - 'refresh_state', - ).select(); - expect(entities).toEqual([ - expect.objectContaining({ - entity_ref: 'component:default/1', - location_key: step.expectedLocationKey, - }), - ]); - const entity = JSON.parse(entities[0].unprocessed_entity) as Entity; - expect(entity.spec?.type).toEqual(step.expectedType); - - await expect( - knexTx( - 'refresh_state_references', - ).select(), - ).resolves.toEqual([ - expect.objectContaining({ - source_key: 'testing', - target_entity_ref: 'component:default/1', - }), - ]); - } - }); - }, - 60_000, - ); - }); - describe('updateProcessedEntity', () => { let id: string; let processedEntity: Entity; @@ -408,6 +291,148 @@ describe('Default Processing Database', () => { }, 60_000, ); + + it.each(databases.eachSupportedId())( + 'updates unprocessed entities with varying location keys, %p', + async databaseId => { + const mockLogger = { + debug: jest.fn(), + error: jest.fn(), + warn: jest.fn(), + }; + const { knex, db } = await createDatabase( + databaseId, + mockLogger as unknown as Logger, + ); + + await insertRefreshStateRow(knex, { + entity_id: id, + entity_ref: 'location:default/fakelocation', + unprocessed_entity: '{}', + processed_entity: '{}', + errors: '[]', + next_update_at: '2021-04-01 13:37:00', + last_discovery_at: '2021-04-01 13:37:00', + }); + + await db.transaction(async tx => { + const knexTx = tx as Knex.Transaction; + + const steps = [ + { + locationKey: undefined, + expectedLocationKey: null, + testKey: 'a', + expectedTestKey: 'a', + }, + { + locationKey: undefined, + expectedLocationKey: null, + testKey: 'b', + expectedTestKey: 'b', + }, + { + locationKey: 'x', + expectedLocationKey: 'x', + testKey: 'c', + expectedTestKey: 'c', + }, + { + locationKey: 'y', + expectedLocationKey: 'x', + testKey: 'd', + expectedTestKey: 'c', + expectConflict: true, + }, + { + locationKey: undefined, + expectedLocationKey: 'x', + testKey: 'e', + expectedTestKey: 'c', + expectConflict: true, + }, + { + locationKey: 'x', + expectedLocationKey: 'x', + testKey: 'f', + expectedTestKey: 'f', + }, + ]; + for (const step of steps) { + mockLogger.debug.mockClear(); + mockLogger.warn.mockClear(); + mockLogger.error.mockClear(); + + await db.updateProcessedEntity(tx, { + id, + processedEntity, + resultHash: '', + relations: [], + deferredEntities: [ + { + entity: { + apiVersion: '1', + kind: 'Component', + metadata: { + name: '1', + }, + spec: { + type: step.testKey, + }, + }, + locationKey: step.locationKey, + }, + ], + }); + + if (step.expectConflict) { + // eslint-disable-next-line jest/no-conditional-expect + expect(mockLogger.warn).toHaveBeenCalledWith( + expect.stringMatching(/^Detected conflicting entityRef/), + ); + } else { + // eslint-disable-next-line jest/no-conditional-expect + expect(mockLogger.warn).not.toHaveBeenCalled(); + } + + const states = await knexTx( + 'refresh_state', + ).select(); + expect(states).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + entity_ref: 'component:default/1', + location_key: step.expectedLocationKey, + }), + expect.objectContaining({ + entity_ref: 'location:default/fakelocation', + location_key: null, + }), + ]), + ); + const unprocessed = states.find( + state => state.entity_ref === 'component:default/1', + )!; + const entity = JSON.parse(unprocessed.unprocessed_entity) as Entity; + expect(entity.spec?.type).toEqual(step.expectedTestKey); + + await expect( + knexTx( + 'refresh_state_references', + ).select(), + ).resolves.toEqual([ + expect.objectContaining({ + source_entity_ref: 'location:default/fakelocation', + target_entity_ref: 'component:default/1', + }), + ]); + + expect(mockLogger.error).not.toHaveBeenCalled(); + } + }); + }, + 60_000, + ); }); describe('updateEntityCache', () => { @@ -731,6 +756,14 @@ describe('Default Processing Database', () => { 'should add new locations using the delta options, %p', async databaseId => { const { knex, db } = await createDatabase(databaseId); + + // Existing state and references should stay + await createLocations(knex, ['location:default/existing']); + await insertRefRow(knex, { + source_key: 'lols', + target_entity_ref: 'location:default/existing', + }); + await db.transaction(async tx => { await db.replaceUnprocessedEntities(tx, { type: 'delta', @@ -772,6 +805,20 @@ describe('Default Processing Database', () => { t.target_entity_ref === 'location:default/new-root', ), ).toBeTruthy(); + + expect( + currentRefreshState.some( + t => t.entity_ref === 'location:default/existing', + ), + ).toBeTruthy(); + + expect( + currentRefRowState.some( + t => + t.source_key === 'lols' && + t.target_entity_ref === 'location:default/existing', + ), + ).toBeTruthy(); }, 60_000, ); diff --git a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts index 33f01b14eb..3dfb4beb80 100644 --- a/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts +++ b/plugins/catalog-backend/src/next/database/DefaultProcessingDatabase.ts @@ -31,7 +31,6 @@ import { DbRelationsRow, } from './tables'; import { - AddUnprocessedEntitiesOptions, GetProcessableEntitiesResult, ProcessingDatabase, RefreshStateItem, @@ -151,63 +150,6 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { .where('entity_id', id); } - private deduplicateRelations(rows: DbRelationsRow[]): DbRelationsRow[] { - return lodash.uniqBy( - rows, - r => `${r.source_entity_ref}:${r.target_entity_ref}:${r.type}`, - ); - } - - private async createDelta( - tx: Knex.Transaction, - options: ReplaceUnprocessedEntitiesOptions, - ): Promise<{ toAdd: DeferredEntity[]; toRemove: string[] }> { - if (options.type === 'delta') { - return { - toAdd: options.added, - toRemove: options.removed.map(e => stringifyEntityRef(e.entity)), - }; - } - - // Grab all of the existing references from the same source, and their locationKeys as well - const oldRefs = await tx( - 'refresh_state_references', - ) - .where({ source_key: options.sourceKey }) - .leftJoin('refresh_state', { - target_entity_ref: 'entity_ref', - }) - .select(['target_entity_ref', 'location_key']); - - const items = options.items.map(deferred => ({ - deferred, - ref: stringifyEntityRef(deferred.entity), - })); - - const oldRefsSet = new Map( - oldRefs.map(r => [r.target_entity_ref, r.location_key]), - ); - const newRefsSet = new Set(items.map(item => item.ref)); - - const toAdd = new Array(); - const toRemove = oldRefs - .map(row => row.target_entity_ref) - .filter(ref => !newRefsSet.has(ref)); - - for (const item of items) { - if (!oldRefsSet.has(item.ref)) { - // Add any entity that does not exist in the database - toAdd.push(item.deferred); - } else if (oldRefsSet.get(item.ref) !== item.deferred.locationKey) { - // Remove and then re-add any entity that exists, but with a different location key - toRemove.push(item.ref); - toAdd.push(item.deferred); - } - } - - return { toAdd, toRemove }; - } - async replaceUnprocessedEntities( txOpaque: Transaction, options: ReplaceUnprocessedEntitiesOptions, @@ -332,143 +274,40 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { } if (toAdd.length) { - await this.addUnprocessedEntities(tx, { - sourceKey: options.sourceKey, - entities: toAdd, - }); - } - } + for (const { entity, locationKey } of toAdd) { + const entityRef = stringifyEntityRef(entity); - /** - * Add a set of deferred entities for processing. - * The entities will be added at the front of the processing queue. - */ - async addUnprocessedEntities( - txOpaque: Transaction, - options: AddUnprocessedEntitiesOptions, - ): Promise { - const tx = txOpaque as Knex.Transaction; - - // Keeps track of the entities that we end up inserting to update refresh_state_references afterwards - const stateReferences = new Array(); - const conflictingStateReferences = new Array(); - - // Upsert all of the unprocessed entities into the refresh_state table, by - // their entity ref. - for (const { entity, locationKey } of options.entities) { - const entityRef = stringifyEntityRef(entity); - const serializedEntity = JSON.stringify(entity); - - // We optimistically try to update any existing refresh state first, as this is by far - // the most common case. - const refreshResult = await tx('refresh_state') - .update({ - unprocessed_entity: serializedEntity, - location_key: locationKey, - last_discovery_at: tx.fn.now(), - // We only get to this point if a processed entity actually had any changes, or - // if an entity provider requested this mutation, meaning that we can safely - // bump the deferred entities to the front of the queue for immediate processing. - next_update_at: tx.fn.now(), - }) - .where('entity_ref', entityRef) - .andWhere(inner => { - if (!locationKey) { - return inner.whereNull('location_key'); - } - return inner - .where('location_key', locationKey) - .orWhereNull('location_key'); - }); - - if (refreshResult === 0) { - // In the event that we can't update an existing refresh state, we first try to insert a new row try { - let query = tx('refresh_state').insert({ - entity_id: uuid(), - entity_ref: entityRef, - unprocessed_entity: serializedEntity, - errors: '', - location_key: locationKey, - next_update_at: tx.fn.now(), - last_discovery_at: tx.fn.now(), - }); - - // TODO(Rugvip): only tested towards Postgres and SQLite - // We have to do this because the only way to detect if there was a conflict with - // SQLite is to catch the error, while Postgres needs to ignore the conflict to not - // break the ongoing transaction. - if (tx.client.config.client !== 'sqlite3') { - query = query.onConflict('entity_ref').ignore(); + let ok = await this.insertUnprocessedEntity(tx, entity, locationKey); + if (!ok) { + ok = await this.updateUnprocessedEntity(tx, entity, locationKey); } - const result: { /* postgres */ rowCount?: number } = await query; - if (result.rowCount === 0) { - throw new ConflictError( - 'Insert failed due to conflicting entity_ref', + if (ok) { + await tx( + 'refresh_state_references', + ).insert({ + source_key: options.sourceKey, + target_entity_ref: entityRef, + }); + } else { + const conflictingKey = await this.checkLocationKeyConflict( + tx, + entityRef, + locationKey, ); + if (conflictingKey) { + this.options.logger.warn( + `Source ${options.sourceKey} detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`, + ); + } } } catch (error) { - if ( - !error.message.includes('UNIQUE constraint failed') && - error.name !== 'ConflictError' - ) { - throw error; - } - // If the row can't be inserted, we have a conflict, but it could be either - // because of a conflicting locationKey or a race with another instance, so check - // whether the conflicting entity has the same entityRef but a different locationKey - const [conflictingEntity] = await tx( - 'refresh_state', - ) - .where({ entity_ref: entityRef }) - .select(); - - // If the location key matches it means we just had a race trigger, which we can safely ignore - if ( - !conflictingEntity || - conflictingEntity.location_key !== locationKey - ) { - this.options.logger.warn( - `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingEntity.location_key} and now also ${locationKey}`, - ); - conflictingStateReferences.push(entityRef); - continue; - } + this.options.logger.error( + `Failed to add '${entityRef}' from source '${options.sourceKey}', ${error}`, + ); } } - - // Skipped on locationKey conflict - stateReferences.push(entityRef); - } - - // Replace all references for the originating entity or source and then create new ones - if ('sourceKey' in options) { - await tx('refresh_state_references') - .whereNotIn('target_entity_ref', conflictingStateReferences) - .andWhere({ source_key: options.sourceKey }) - .delete(); - await tx.batchInsert( - 'refresh_state_references', - stateReferences.map(entityRef => ({ - source_key: options.sourceKey, - target_entity_ref: entityRef, - })), - BATCH_SIZE, - ); - } else { - await tx('refresh_state_references') - .whereNotIn('target_entity_ref', conflictingStateReferences) - .andWhere({ source_entity_ref: options.sourceEntityRef }) - .delete(); - await tx.batchInsert( - 'refresh_state_references', - stateReferences.map(entityRef => ({ - source_entity_ref: options.sourceEntityRef, - target_entity_ref: entityRef, - })), - BATCH_SIZE, - ); } } @@ -599,4 +438,243 @@ export class DefaultProcessingDatabase implements ProcessingDatabase { throw rethrowError(e); } } + + /** + * Attempts to update an existing refresh state row, returning true if it was + * updated and false if there was no entity with a matching ref and location key. + * + * Updating the entity will also cause it to be scheduled for immediate processing. + */ + private async updateUnprocessedEntity( + tx: Knex.Transaction, + entity: Entity, + locationKey?: string, + ): Promise { + const entityRef = stringifyEntityRef(entity); + const serializedEntity = JSON.stringify(entity); + + const refreshResult = await tx('refresh_state') + .update({ + unprocessed_entity: serializedEntity, + location_key: locationKey, + last_discovery_at: tx.fn.now(), + // We only get to this point if a processed entity actually had any changes, or + // if an entity provider requested this mutation, meaning that we can safely + // bump the deferred entities to the front of the queue for immediate processing. + next_update_at: tx.fn.now(), + }) + .where('entity_ref', entityRef) + .andWhere(inner => { + if (!locationKey) { + return inner.whereNull('location_key'); + } + return inner + .where('location_key', locationKey) + .orWhereNull('location_key'); + }); + + return refreshResult === 1; + } + + /** + * Attempts to insert a new refresh state row for the given entity, returning + * true if successful and false if there was a conflict. + */ + private async insertUnprocessedEntity( + tx: Knex.Transaction, + entity: Entity, + locationKey?: string, + ): Promise { + const entityRef = stringifyEntityRef(entity); + const serializedEntity = JSON.stringify(entity); + + try { + let query = tx('refresh_state').insert({ + entity_id: uuid(), + entity_ref: entityRef, + unprocessed_entity: serializedEntity, + errors: '', + location_key: locationKey, + next_update_at: tx.fn.now(), + last_discovery_at: tx.fn.now(), + }); + + // TODO(Rugvip): only tested towards Postgres and SQLite + // We have to do this because the only way to detect if there was a conflict with + // SQLite is to catch the error, while Postgres needs to ignore the conflict to not + // break the ongoing transaction. + if (tx.client.config.client !== 'sqlite3') { + query = query.onConflict('entity_ref').ignore() as any; // type here does not match runtime + } + + // Postgres gives as an object with rowCount, SQLite gives us an array + const result: { rowCount?: number; length?: number } = await query; + return result.rowCount === 1 || result.length === 1; + } catch (error) { + // SQLite reached this rather than the rowCount check above + if (error.message.includes('UNIQUE constraint failed')) { + return false; + } + throw error; + } + } + + /** + * Checks whether a refresh state exists for the given entity that has a + * location key that does not match the provided location key. + * + * @returns The conflicting key if there is one. + */ + private async checkLocationKeyConflict( + tx: Knex.Transaction, + entityRef: string, + locationKey?: string, + ): Promise { + const row = await tx('refresh_state') + .select('location_key') + .where('entity_ref', entityRef) + .first(); + + const conflictingKey = row?.location_key; + + // If there's no existing key we can't have a conflict + if (!conflictingKey) { + return undefined; + } + + if (conflictingKey !== locationKey) { + return conflictingKey; + } + return undefined; + } + + private deduplicateRelations(rows: DbRelationsRow[]): DbRelationsRow[] { + return lodash.uniqBy( + rows, + r => `${r.source_entity_ref}:${r.target_entity_ref}:${r.type}`, + ); + } + + private async createDelta( + tx: Knex.Transaction, + options: ReplaceUnprocessedEntitiesOptions, + ): Promise<{ toAdd: DeferredEntity[]; toRemove: string[] }> { + if (options.type === 'delta') { + return { + toAdd: options.added, + toRemove: options.removed.map(e => stringifyEntityRef(e.entity)), + }; + } + + // Grab all of the existing references from the same source, and their locationKeys as well + const oldRefs = await tx( + 'refresh_state_references', + ) + .where({ source_key: options.sourceKey }) + .leftJoin('refresh_state', { + target_entity_ref: 'entity_ref', + }) + .select(['target_entity_ref', 'location_key']); + + const items = options.items.map(deferred => ({ + deferred, + ref: stringifyEntityRef(deferred.entity), + })); + + const oldRefsSet = new Map( + oldRefs.map(r => [r.target_entity_ref, r.location_key]), + ); + const newRefsSet = new Set(items.map(item => item.ref)); + + const toAdd = new Array(); + const toRemove = oldRefs + .map(row => row.target_entity_ref) + .filter(ref => !newRefsSet.has(ref)); + + for (const item of items) { + if (!oldRefsSet.has(item.ref)) { + // Add any entity that does not exist in the database + toAdd.push(item.deferred); + } else if (oldRefsSet.get(item.ref) !== item.deferred.locationKey) { + // Remove and then re-add any entity that exists, but with a different location key + toRemove.push(item.ref); + toAdd.push(item.deferred); + } + } + + return { toAdd, toRemove }; + } + + /** + * Add a set of deferred entities for processing. + * The entities will be added at the front of the processing queue. + */ + private async addUnprocessedEntities( + txOpaque: Transaction, + options: { + sourceEntityRef: string; + entities: DeferredEntity[]; + }, + ): Promise { + const tx = txOpaque as Knex.Transaction; + + // Keeps track of the entities that we end up inserting to update refresh_state_references afterwards + const stateReferences = new Array(); + const conflictingStateReferences = new Array(); + + // Upsert all of the unprocessed entities into the refresh_state table, by + // their entity ref. + for (const { entity, locationKey } of options.entities) { + const entityRef = stringifyEntityRef(entity); + + const updated = await this.updateUnprocessedEntity( + tx, + entity, + locationKey, + ); + if (updated) { + stateReferences.push(entityRef); + continue; + } + + const inserted = await this.insertUnprocessedEntity( + tx, + entity, + locationKey, + ); + if (inserted) { + stateReferences.push(entityRef); + continue; + } + + // If the row can't be inserted, we have a conflict, but it could be either + // because of a conflicting locationKey or a race with another instance, so check + // whether the conflicting entity has the same entityRef but a different locationKey + const conflictingKey = await this.checkLocationKeyConflict( + tx, + entityRef, + locationKey, + ); + if (conflictingKey) { + this.options.logger.warn( + `Detected conflicting entityRef ${entityRef} already referenced by ${conflictingKey} and now also ${locationKey}`, + ); + conflictingStateReferences.push(entityRef); + } + } + + // Replace all references for the originating entity or source and then create new ones + await tx('refresh_state_references') + .whereNotIn('target_entity_ref', conflictingStateReferences) + .andWhere({ source_entity_ref: options.sourceEntityRef }) + .delete(); + await tx.batchInsert( + 'refresh_state_references', + stateReferences.map(entityRef => ({ + source_entity_ref: options.sourceEntityRef, + target_entity_ref: entityRef, + })), + BATCH_SIZE, + ); + } } diff --git a/plugins/catalog-backend/src/next/database/types.ts b/plugins/catalog-backend/src/next/database/types.ts index aabd28afc8..3f1a4784fb 100644 --- a/plugins/catalog-backend/src/next/database/types.ts +++ b/plugins/catalog-backend/src/next/database/types.ts @@ -20,16 +20,6 @@ import { DateTime } from 'luxon'; import { Transaction } from '../../database/types'; import { DeferredEntity } from '../processing/types'; -export type AddUnprocessedEntitiesOptions = - | { - sourceEntityRef: string; - entities: DeferredEntity[]; - } - | { - sourceKey: string; - entities: DeferredEntity[]; - }; - export type AddUnprocessedEntitiesResult = {}; export type UpdateProcessedEntityOptions = { diff --git a/plugins/kubernetes/api-report.md b/plugins/kubernetes/api-report.md index fc1127595d..fcbf349d7e 100644 --- a/plugins/kubernetes/api-report.md +++ b/plugins/kubernetes/api-report.md @@ -33,6 +33,11 @@ export function formatClusterLink( options: FormatClusterLinkOptions, ): string | undefined; +// Warning: (ae-missing-release-tag) "isKubernetesAvailable" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export const isKubernetesAvailable: (entity: Entity) => boolean; + // Warning: (ae-forgotten-export) The symbol "KubernetesAuthProvidersApi" needs to be exported by the entry point index.d.ts // Warning: (ae-missing-release-tag) "KubernetesAuthProviders" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // diff --git a/plugins/kubernetes/src/Router.tsx b/plugins/kubernetes/src/Router.tsx index 11c9b488fb..f98d3ca45e 100644 --- a/plugins/kubernetes/src/Router.tsx +++ b/plugins/kubernetes/src/Router.tsx @@ -27,6 +27,12 @@ const KUBERNETES_ANNOTATION = 'backstage.io/kubernetes-id'; const KUBERNETES_LABEL_SELECTOR_QUERY_ANNOTATION = 'backstage.io/kubernetes-label-selector'; +export const isKubernetesAvailable = (entity: Entity) => + Boolean(entity.metadata.annotations?.[KUBERNETES_ANNOTATION]) || + Boolean( + entity.metadata.annotations?.[KUBERNETES_LABEL_SELECTOR_QUERY_ANNOTATION], + ); + type Props = { /** @deprecated The entity is now grabbed from context instead */ entity?: Entity; diff --git a/plugins/kubernetes/src/index.ts b/plugins/kubernetes/src/index.ts index 63f68f0af1..08af757d47 100644 --- a/plugins/kubernetes/src/index.ts +++ b/plugins/kubernetes/src/index.ts @@ -25,6 +25,6 @@ export { kubernetesPlugin as plugin, EntityKubernetesContent, } from './plugin'; -export { Router } from './Router'; +export { Router, isKubernetesAvailable } from './Router'; export * from './kubernetes-auth-provider'; export * from './utils/clusterLinks'; diff --git a/plugins/sentry/package.json b/plugins/sentry/package.json index 19a0bc0509..88bae10ef6 100644 --- a/plugins/sentry/package.json +++ b/plugins/sentry/package.json @@ -56,7 +56,7 @@ "@testing-library/react": "^11.2.5", "@testing-library/user-event": "^13.1.8", "@types/jest": "^26.0.7", - "@types/luxon": "^1.27.0", + "@types/luxon": "^2.0.4", "@types/node": "^14.14.32", "@types/react": "*", "cross-fetch": "^3.0.6", diff --git a/plugins/splunk-on-call/package.json b/plugins/splunk-on-call/package.json index 71354743c6..289b9f25dd 100644 --- a/plugins/splunk-on-call/package.json +++ b/plugins/splunk-on-call/package.json @@ -55,7 +55,7 @@ "@testing-library/react": "^11.2.5", "@testing-library/user-event": "^13.1.8", "@types/jest": "^26.0.7", - "@types/luxon": "^1.25.0", + "@types/luxon": "^2.0.4", "@types/node": "^14.14.32", "cross-fetch": "^3.0.6", "msw": "^0.29.0", diff --git a/plugins/techdocs/api-report.md b/plugins/techdocs/api-report.md index 082dfc1db4..087cdf803a 100644 --- a/plugins/techdocs/api-report.md +++ b/plugins/techdocs/api-report.md @@ -167,6 +167,11 @@ export const EntityTechdocsContent: (_props: { entity?: Entity | undefined; }) => JSX.Element; +// Warning: (ae-missing-release-tag) "isTechDocsAvailable" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) +// +// @public (undocumented) +export const isTechDocsAvailable: (entity: Entity) => boolean; + // Warning: (ae-missing-release-tag) "PanelType" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal) // // @public (undocumented) diff --git a/plugins/techdocs/src/Router.tsx b/plugins/techdocs/src/Router.tsx index 45b0dc756c..102cda979a 100644 --- a/plugins/techdocs/src/Router.tsx +++ b/plugins/techdocs/src/Router.tsx @@ -25,6 +25,9 @@ import { MissingAnnotationEmptyState } from '@backstage/core-components'; const TECHDOCS_ANNOTATION = 'backstage.io/techdocs-ref'; +export const isTechDocsAvailable = (entity: Entity) => + Boolean(entity?.metadata?.annotations?.[TECHDOCS_ANNOTATION]); + export const Router = () => { return ( diff --git a/plugins/techdocs/src/index.ts b/plugins/techdocs/src/index.ts index ba24f1c39d..50e25c5582 100644 --- a/plugins/techdocs/src/index.ts +++ b/plugins/techdocs/src/index.ts @@ -44,4 +44,4 @@ export { TechDocsReaderPage, } from './plugin'; export * from './reader'; -export { EmbeddedDocsRouter, Router } from './Router'; +export { EmbeddedDocsRouter, Router, isTechDocsAvailable } from './Router'; diff --git a/plugins/xcmetrics/package.json b/plugins/xcmetrics/package.json index fc86ff572e..d3df9b6718 100644 --- a/plugins/xcmetrics/package.json +++ b/plugins/xcmetrics/package.json @@ -43,7 +43,7 @@ "@testing-library/react": "^11.2.5", "@testing-library/user-event": "^13.1.8", "@types/jest": "^26.0.7", - "@types/luxon": "^1.27.0", + "@types/luxon": "^2.0.4", "@types/node": "^14.14.32", "cross-fetch": "^3.0.6", "msw": "^0.29.0" diff --git a/plugins/xcmetrics/src/components/BuildList/BuildList.tsx b/plugins/xcmetrics/src/components/BuildList/BuildList.tsx index f9e8bfdd98..5a6e1bd322 100644 --- a/plugins/xcmetrics/src/components/BuildList/BuildList.tsx +++ b/plugins/xcmetrics/src/components/BuildList/BuildList.tsx @@ -39,7 +39,7 @@ export const BuildList = () => { const tableRef = useRef(); const initialFilters = { - from: DateTime.now().minus({ year: 1 }).toISODate(), + from: DateTime.now().minus({ years: 1 }).toISODate(), to: DateTime.now().toISODate(), }; diff --git a/yarn.lock b/yarn.lock index 7657498ff7..2f6e3a8b5e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7090,10 +7090,10 @@ resolved "https://registry.npmjs.org/@types/lunr/-/lunr-2.3.3.tgz#ec985618fd2712c010f8edab4f1ae7784ad7c583" integrity sha512-09sXZZVsB3Ib41U0fC+O1O+4UOZT1bl/e+/QubPxpqDWHNEchvx/DEb1KJMOwq6K3MTNzZFoNSzVdR++o1DVnw== -"@types/luxon@^1.25.0", "@types/luxon@^1.27.0": - version "1.27.0" - resolved "https://registry.npmjs.org/@types/luxon/-/luxon-1.27.0.tgz#1e3b5a7f8ca6944349c43498b4442b742c71ab0b" - integrity sha512-rr2lNXsErnA/ARtgFn46NtQjUa66cuwZYeo/2K7oqqxhJErhXgHBPyNKCo+pfOC3L7HFwtao8ebViiU9h4iAxA== +"@types/luxon@^2.0.4": + version "2.0.4" + resolved "https://registry.npmjs.org/@types/luxon/-/luxon-2.0.4.tgz#f7b5a86ccd843c0ccaddfaedd9ee1081bc1cde3b" + integrity sha512-l3xuhmyF2kBldy15SeY6d6HbK2BacEcSK1qTF1ISPtPHr29JH0C1fndz9ExXLKpGl0J6pZi+dGp1i5xesMt60Q== "@types/markdown-to-jsx@^6.11.3": version "6.11.3" @@ -7621,12 +7621,11 @@ "@types/estree" "*" "@types/terser-webpack-plugin@^5.0.4": - version "5.0.4" - resolved "https://registry.npmjs.org/@types/terser-webpack-plugin/-/terser-webpack-plugin-5.0.4.tgz#852949e1d124e6c895cfa8cb17eb6265e4fdfe59" - integrity sha512-1iyfZpMNNA/h/Q8uBpwuVhxKfKQHc98PD9NaCTrg22nj6d8aUvT79KBMtRLmR43v1PtCB0r1/gWGdNXrrMEK7A== + version "5.2.0" + resolved "https://registry.npmjs.org/@types/terser-webpack-plugin/-/terser-webpack-plugin-5.2.0.tgz#6aaec696593216917f9f03266bed222f8253483b" + integrity sha512-iHDR2pRfFjGyDqCALX2tgUgFtGoQf2AJhKpC2XD1IMBQVJF2bny6WChGRDKj9eaZJl4F2RmvBhxJNtVPj7aTRw== dependencies: - terser "^5.3.8" - webpack "^5.1.0" + terser-webpack-plugin "*" "@types/testing-library__jest-dom@^5.9.1": version "5.9.5" @@ -11342,9 +11341,9 @@ core-js@^3.0.4, core-js@^3.6.5, core-js@^3.8.2: integrity sha512-GUbtPllXMYRzIgHNZ4dTYTcUemls2cni83Q4Q/TrFONHfhcg9oEGOtaGHfb0cpzec60P96UKPvMkjX1jET8rUw== core-js@^3.6.0, core-js@^3.8.3: - version "3.18.0" - resolved "https://registry.npmjs.org/core-js/-/core-js-3.18.0.tgz#9af3f4a6df9ba3428a3fb1b171f1503b3f40cc49" - integrity sha512-WJeQqq6jOYgVgg4NrXKL0KLQhi0CT4ZOCvFL+3CQ5o7I6J8HkT5wd53EadMfqTDp1so/MT1J+w2ujhWcCJtN7w== + version "3.18.1" + resolved "https://registry.npmjs.org/core-js/-/core-js-3.18.1.tgz#289d4be2ce0085d40fc1244c0b1a54c00454622f" + integrity sha512-vJlUi/7YdlCZeL6fXvWNaLUPh/id12WXj3MbkMw5uOyF0PfWPBNOCNbs53YqgrvtujLNlt9JQpruyIKkUZ+PKA== core-util-is@1.0.2, core-util-is@~1.0.0: version "1.0.2" @@ -17540,10 +17539,10 @@ jest-worker@^26.5.0, jest-worker@^26.6.2: merge-stream "^2.0.0" supports-color "^7.0.0" -jest-worker@^27.0.2: - version "27.0.6" - resolved "https://registry.npmjs.org/jest-worker/-/jest-worker-27.0.6.tgz#a5fdb1e14ad34eb228cfe162d9f729cdbfa28aed" - integrity sha512-qupxcj/dRuA3xHPMUd40gr2EaAurFbkwzOh7wfPaeE9id7hyjURRQoqNfHifHK3XjJU6YJJUQKILGUnwGPEOCA== +jest-worker@^27.0.6: + version "27.2.2" + resolved "https://registry.npmjs.org/jest-worker/-/jest-worker-27.2.2.tgz#636deeae8068abbf2b34b4eb9505f8d4e5bd625c" + integrity sha512-aG1xq9KgWB2CPC8YdMIlI8uZgga2LFNcGbHJxO8ctfXAydSaThR4EewKQGg3tBOC+kS3vhPGgymsBdi9VINjPw== dependencies: "@types/node" "*" merge-stream "^2.0.0" @@ -23021,9 +23020,9 @@ react-hook-form@^6.15.4: integrity sha512-prq82ofMbnRyj5wqDe8hsTRcdR25jQ+B8KtCS7BLCzjFHAwNuCjRwzPuP4eYLsEBjEIeYd6try+pdLdw0kPkpg== react-hook-form@^7.12.2: - version "7.15.3" - resolved "https://registry.npmjs.org/react-hook-form/-/react-hook-form-7.15.3.tgz#7d222dcfd43137be127dca2843149f09bd25d4b1" - integrity sha512-z30aZoEHkWE8oZvad4OcYSBI0kQua/T5sFGH9tB2HfeykFnP/pGXNap8lDio4/U1yPj2ffpbvRIvqKd/6jjBVA== + version "7.16.1" + resolved "https://registry.npmjs.org/react-hook-form/-/react-hook-form-7.16.1.tgz#669046df378a71949e5cf8a2398cbe20d5cb27bc" + integrity sha512-kcLDmSmlyLUFx2UU5bG/o4+3NeK753fhKodJa8gkplXohGkpAq0/p+TR24OWjZmkEc3ES7ppC5v5d6KUk+fJTA== react-hot-loader@^4.12.21: version "4.13.0" @@ -24498,7 +24497,7 @@ schema-utils@^3.0.0: ajv "^6.12.5" ajv-keywords "^3.5.2" -schema-utils@^3.1.0: +schema-utils@^3.1.0, schema-utils@^3.1.1: version "3.1.1" resolved "https://registry.npmjs.org/schema-utils/-/schema-utils-3.1.1.tgz#bc74c4b6b6995c1d88f76a8b77bea7219e0c8281" integrity sha512-Y5PQxS4ITlC+EahLuXaY86TXfR7Dc5lw294alXOq86JAHCihAIZfqv8nNCWvaEJvaC51uN9hbLGeV0cFBdH+Fw== @@ -25075,6 +25074,14 @@ source-map-support@^0.5.16, source-map-support@^0.5.17, source-map-support@^0.5. buffer-from "^1.0.0" source-map "^0.6.0" +source-map-support@~0.5.20: + version "0.5.20" + resolved "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.20.tgz#12166089f8f5e5e8c56926b377633392dd2cb6c9" + integrity sha512-n1lZZ8Ve4ksRqizaBQgxXDgKwttHDhyfQjA6YZZn8+AroHbsIz+JjwxQDxbp+7y5OYCI8t1Yk7etjD9CRd2hIw== + dependencies: + buffer-from "^1.0.0" + source-map "^0.6.0" + source-map-url@^0.4.0: version "0.4.0" resolved "https://registry.npmjs.org/source-map-url/-/source-map-url-0.4.0.tgz#3e935d7ddd73631b97659956d55128e87b5084a3" @@ -26202,6 +26209,18 @@ terminal-link@^2.0.0: ansi-escapes "^4.2.1" supports-hyperlinks "^2.0.0" +terser-webpack-plugin@*, terser-webpack-plugin@^5.1.3: + version "5.2.4" + resolved "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.2.4.tgz#ad1be7639b1cbe3ea49fab995cbe7224b31747a1" + integrity sha512-E2CkNMN+1cho04YpdANyRrn8CyN4yMy+WdFKZIySFZrGXZxJwJP6PMNGGc/Mcr6qygQHUUqRxnAPmi0M9f00XA== + dependencies: + jest-worker "^27.0.6" + p-limit "^3.1.0" + schema-utils "^3.1.1" + serialize-javascript "^6.0.0" + source-map "^0.6.1" + terser "^5.7.2" + terser-webpack-plugin@^1.4.3: version "1.4.3" resolved "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-1.4.3.tgz#5ecaf2dbdc5fb99745fd06791f46fc9ddb1c9a7c" @@ -26232,18 +26251,6 @@ terser-webpack-plugin@^4.2.3: terser "^5.3.4" webpack-sources "^1.4.3" -terser-webpack-plugin@^5.1.3: - version "5.1.4" - resolved "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.1.4.tgz#c369cf8a47aa9922bd0d8a94fe3d3da11a7678a1" - integrity sha512-C2WkFwstHDhVEmsmlCxrXUtVklS+Ir1A7twrYzrDrQQOIMOaVAYykaoo/Aq1K0QRkMoY2hhvDQY1cm4jnIMFwA== - dependencies: - jest-worker "^27.0.2" - p-limit "^3.1.0" - schema-utils "^3.0.0" - serialize-javascript "^6.0.0" - source-map "^0.6.1" - terser "^5.7.0" - terser@^4.1.2, terser@^4.6.3: version "4.8.0" resolved "https://registry.npmjs.org/terser/-/terser-4.8.0.tgz#63056343d7c70bb29f3af665865a46fe03a0df17" @@ -26253,7 +26260,7 @@ terser@^4.1.2, terser@^4.6.3: source-map "~0.6.1" source-map-support "~0.5.12" -terser@^5.3.4, terser@^5.3.8, terser@^5.7.0: +terser@^5.3.4: version "5.7.1" resolved "https://registry.npmjs.org/terser/-/terser-5.7.1.tgz#2dc7a61009b66bb638305cb2a824763b116bf784" integrity sha512-b3e+d5JbHAe/JSjwsC3Zn55wsBIM7AsHLjKxT31kGCldgbpFePaFo+PiddtO6uwRZWRw7sPXmAN8dTW61xmnSg== @@ -26262,6 +26269,15 @@ terser@^5.3.4, terser@^5.3.8, terser@^5.7.0: source-map "~0.7.2" source-map-support "~0.5.19" +terser@^5.7.2: + version "5.9.0" + resolved "https://registry.npmjs.org/terser/-/terser-5.9.0.tgz#47d6e629a522963240f2b55fcaa3c99083d2c351" + integrity sha512-h5hxa23sCdpzcye/7b8YqbE5OwKca/ni0RQz1uRX3tGh8haaGHqcuSqbGRybuAKNdntZ0mDgFNXPJ48xQ2RXKQ== + dependencies: + commander "^2.20.0" + source-map "~0.7.2" + source-map-support "~0.5.20" + test-exclude@^6.0.0: version "6.0.0" resolved "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz#04a8698661d805ea6fa293b6cb9e63ac044ef15e" @@ -27879,36 +27895,6 @@ webpack@^5, webpack@^5.48.0: watchpack "^2.2.0" webpack-sources "^3.2.0" -webpack@^5.1.0: - version "5.49.0" - resolved "https://registry.npmjs.org/webpack/-/webpack-5.49.0.tgz#e250362b781a9fb614ba0a97ed67c66b9c5310cd" - integrity sha512-XarsANVf28A7Q3KPxSnX80EkCcuOer5hTOEJWJNvbskOZ+EK3pobHarGHceyUZMxpsTHBHhlV7hiQyLZzGosYw== - dependencies: - "@types/eslint-scope" "^3.7.0" - "@types/estree" "^0.0.50" - "@webassemblyjs/ast" "1.11.1" - "@webassemblyjs/wasm-edit" "1.11.1" - "@webassemblyjs/wasm-parser" "1.11.1" - acorn "^8.4.1" - acorn-import-assertions "^1.7.6" - browserslist "^4.14.5" - chrome-trace-event "^1.0.2" - enhanced-resolve "^5.8.0" - es-module-lexer "^0.7.1" - eslint-scope "5.1.1" - events "^3.2.0" - glob-to-regexp "^0.4.1" - graceful-fs "^4.2.4" - json-parse-better-errors "^1.0.2" - loader-runner "^4.2.0" - mime-types "^2.1.27" - neo-async "^2.6.2" - schema-utils "^3.1.0" - tapable "^2.1.1" - terser-webpack-plugin "^5.1.3" - watchpack "^2.2.0" - webpack-sources "^3.2.0" - websocket-driver@>=0.5.1: version "0.7.3" resolved "https://registry.npmjs.org/websocket-driver/-/websocket-driver-0.7.3.tgz#a2d4e0d4f4f116f1e6297eba58b05d430100e9f9"