diff --git a/plugins/catalog-backend/src/next/processing/DefaultCatalogProcessingOrchestrator.ts b/plugins/catalog-backend/src/next/processing/DefaultCatalogProcessingOrchestrator.ts index e94c34a660..01640f34b1 100644 --- a/plugins/catalog-backend/src/next/processing/DefaultCatalogProcessingOrchestrator.ts +++ b/plugins/catalog-backend/src/next/processing/DefaultCatalogProcessingOrchestrator.ts @@ -24,7 +24,7 @@ import { stringifyLocationReference, } from '@backstage/catalog-model'; import { ConflictError, InputError, NotAllowedError } from '@backstage/errors'; -import { JsonObject, JsonValue } from '@backstage/config'; +import { JsonValue } from '@backstage/config'; import { ScmIntegrationRegistry } from '@backstage/integration'; import path from 'path'; import { Logger } from 'winston'; @@ -33,7 +33,6 @@ import { CatalogProcessorParser, } from '../../ingestion/processors'; import * as results from '../../ingestion/processors/results'; -import { CatalogProcessorCache } from '../../ingestion/processors/types'; import { CatalogProcessingOrchestrator, EntityProcessingRequest, @@ -47,78 +46,19 @@ import { toAbsoluteUrl, validateEntity, validateEntityEnvelope, + isObject, } from './util'; import { CatalogRulesEnforcer } from '../../ingestion/CatalogRules'; +import { ProcessorCacheManager } from './ProcessorCacheManager'; type Context = { entityRef: string; location: LocationSpec; originLocation: LocationSpec; collector: ProcessorOutputCollector; - cache: ProcessorCache; + cache: ProcessorCacheManager; }; -function isObject(value: JsonValue | undefined): value is JsonObject { - return typeof value === 'object' && value !== null && !Array.isArray(value); -} - -class DefaultCatalogProcessorCache implements CatalogProcessorCache { - private newState?: JsonObject; - constructor(private readonly existingState: JsonObject) {} - - async get( - key: string, - ): Promise { - return this.existingState[key] as ItemType | undefined; - } - - async set( - key: string, - value: ItemType, - ): Promise { - if (!this.newState) { - this.newState = {}; - } - - this.newState[key] = value; - } - - collect(): JsonObject | undefined { - return this.newState; - } -} - -class ProcessorCache { - private caches = new Map(); - - constructor(private readonly existingState: JsonObject) {} - forProcessor(processor: CatalogProcessor): CatalogProcessorCache { - // constructor name will be deprecated in the future when we make `getProcessorName` required in the implementation - const name = processor.getProcessorName?.() ?? processor.constructor.name; - const cache = this.caches.get(name); - if (cache) { - return cache; - } - - const existing = this.existingState[name]; - - const newCache = new DefaultCatalogProcessorCache( - isObject(existing) ? existing : {}, - ); - this.caches.set(name, newCache); - return newCache; - } - - collect(): JsonObject { - const result: JsonObject = {}; - for (const [key, value] of this.caches.entries()) { - result[key] = value.collect(); - } - - return result; - } -} - export class DefaultCatalogProcessingOrchestrator implements CatalogProcessingOrchestrator { @@ -149,7 +89,7 @@ export class DefaultCatalogProcessingOrchestrator ); // Cache that is scoped to the entity and processor - const cache = new ProcessorCache( + const cache = new ProcessorCacheManager( isObject(state) && isObject(state.cache) ? state.cache : {}, ); diff --git a/plugins/catalog-backend/src/next/processing/ProcessorCacheManager.test.ts b/plugins/catalog-backend/src/next/processing/ProcessorCacheManager.test.ts new file mode 100644 index 0000000000..bfdfe93d05 --- /dev/null +++ b/plugins/catalog-backend/src/next/processing/ProcessorCacheManager.test.ts @@ -0,0 +1,62 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { CatalogProcessor } from '../../ingestion/processors'; +import { ProcessorCacheManager } from './ProcessorCacheManager'; + +class MyProcessor implements CatalogProcessor { + getProcessorName = () => 'my-processor'; +} + +class OtherProcessor implements CatalogProcessor {} + +describe('ProcessorCacheManager', () => { + const myProcessor = new MyProcessor(); + const otherProcessor = new OtherProcessor(); + + it('should forward existing state and collect new state', async () => { + const cache = new ProcessorCacheManager({ + 'my-processor': { 'my-key': 'my-value' }, + }); + + // Should be empty to begin with + expect(cache.collect()).toEqual({}); + + // instance should be cached + const processorCache = cache.forProcessor(myProcessor); + expect(processorCache).toBe(cache.forProcessor(myProcessor)); + + // Initial values should be visible, writes should not + await expect(processorCache.get('my-key')).resolves.toBe( + 'my-value', + ); + processorCache.set('my-key', 'my-new-value'); + await expect(processorCache.get('my-key')).resolves.toBe( + 'my-value', + ); + + // There should be isolation between processors + await expect( + cache.forProcessor(otherProcessor).get('my-key'), + ).resolves.toBeUndefined(); + + // Collecting the state and passing it to a new manager should make the new values visible + const newCache = new ProcessorCacheManager(cache.collect()); + await expect( + newCache.forProcessor(myProcessor).get('my-key'), + ).resolves.toBe('my-new-value'); + }); +}); diff --git a/plugins/catalog-backend/src/next/processing/ProcessorCacheManager.ts b/plugins/catalog-backend/src/next/processing/ProcessorCacheManager.ts new file mode 100644 index 0000000000..7eddc102d2 --- /dev/null +++ b/plugins/catalog-backend/src/next/processing/ProcessorCacheManager.ts @@ -0,0 +1,78 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { JsonObject, JsonValue } from '@backstage/config'; +import { CatalogProcessor } from '../../ingestion/processors'; +import { CatalogProcessorCache } from '../../ingestion/processors/types'; +import { isObject } from './util'; + +class SingleProcessorCache implements CatalogProcessorCache { + private newState?: JsonObject; + constructor(private readonly existingState: JsonObject) {} + + async get( + key: string, + ): Promise { + return this.existingState[key] as ItemType | undefined; + } + + async set( + key: string, + value: ItemType, + ): Promise { + if (!this.newState) { + this.newState = {}; + } + + this.newState[key] = value; + } + + collect(): JsonObject | undefined { + return this.newState; + } +} + +export class ProcessorCacheManager { + private caches = new Map(); + + constructor(private readonly existingState: JsonObject) {} + + forProcessor(processor: CatalogProcessor): CatalogProcessorCache { + // constructor name will be deprecated in the future when we make `getProcessorName` required in the implementation + const name = processor.getProcessorName?.() ?? processor.constructor.name; + const cache = this.caches.get(name); + if (cache) { + return cache; + } + + const existing = this.existingState[name]; + + const newCache = new SingleProcessorCache( + isObject(existing) ? existing : {}, + ); + this.caches.set(name, newCache); + return newCache; + } + + collect(): JsonObject { + const result: JsonObject = {}; + for (const [key, value] of this.caches.entries()) { + result[key] = value.collect(); + } + + return result; + } +} diff --git a/plugins/catalog-backend/src/next/processing/util.ts b/plugins/catalog-backend/src/next/processing/util.ts index 8b06a76d16..ad8fe8be60 100644 --- a/plugins/catalog-backend/src/next/processing/util.ts +++ b/plugins/catalog-backend/src/next/processing/util.ts @@ -24,6 +24,7 @@ import { ORIGIN_LOCATION_ANNOTATION, stringifyEntityRef, } from '@backstage/catalog-model'; +import { JsonObject, JsonValue } from '@backstage/config'; import { InputError } from '@backstage/errors'; import { ScmIntegrationRegistry } from '@backstage/integration'; import path from 'path'; @@ -76,6 +77,10 @@ export function toAbsoluteUrl( } } +export function isObject(value: JsonValue | undefined): value is JsonObject { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + export const validateEntity = entitySchemaValidator(); export const validateEntityEnvelope = entityEnvelopeSchemaValidator();