Merge pull request #24225 from backstage/rugvip/migrations

app-backend: added migration test + fix namespace migration

plugins/app-backend/migrations/20240113144027_assets-namespace.js

@@ -20,19 +20,63 @@
  * @param {import('knex').Knex} knex
  */
 exports.up = async function up(knex) {
+  const isMySQL = knex.client.config.client.includes('mysql');
+
   await knex.schema.alterTable('static_assets_cache', table => {
     // The namespace is used to allow operations on asset groups, e.g. delete all assets in a namespace
-    table.string('namespace').comment('The namespace of the file');
-    table.index('namespace', 'static_asset_cache_namespace_idx');
+    table
+      .string('namespace')
+      .defaultTo('default')
+      .comment('The namespace of the file');
+
+    if (!isMySQL) {
+      table.dropPrimary();
+      table.primary(['namespace', 'path']);
+    }
   });
+
+  if (isMySQL) {
+    await knex.schema.raw(
+      'drop index static_assets_cache_path_idx on static_assets_cache;',
+    );
+    await knex.schema.raw(
+      'create unique index static_assets_cache_path_idx on static_assets_cache(namespace, path(254));',
+    );
+  }
 };
 
 /**
  * @param {import('knex').Knex} knex
  */
 exports.down = async function down(knex) {
+  const isMySQL = knex.client.config.client.includes('mysql');
+
+  await knex
+    .delete()
+    .from('static_assets_cache')
+    .whereNot('namespace', 'default');
+
+  if (isMySQL) {
+    await knex.schema.raw(
+      'drop index static_assets_cache_path_idx on static_assets_cache;',
+    );
+  }
+
   await knex.schema.alterTable('static_assets_cache', table => {
-    table.dropIndex([], 'static_asset_cache_namespace_idx');
+    if (!isMySQL) {
+      table.dropPrimary();
+    }
+
     table.dropColumn('namespace');
+
+    if (!isMySQL) {
+      table.primary(['path']);
+    }
   });
+
+  if (isMySQL) {
+    await knex.schema.raw(
+      'create unique index static_assets_cache_path_idx on static_assets_cache(path(254));',
+    );
+  }
 };

plugins/app-backend/src/lib/assets/StaticAssetsStore.test.ts

@@ -36,9 +36,7 @@ function createDatabaseManager(
 jest.setTimeout(60_000);
 
 describe('StaticAssetsStore', () => {
-  const databases = TestDatabases.create({
-    ids: ['MYSQL_8', 'POSTGRES_16', 'POSTGRES_12', 'SQLITE_3'],
-  });
+  const databases = TestDatabases.create();
 
   it.each(databases.eachSupportedId())(
     'should store and retrieve assets, %p',

plugins/app-backend/src/lib/assets/StaticAssetsStore.ts

@@ -50,7 +50,7 @@ export interface StaticAssetsStoreOptions {
 export class StaticAssetsStore implements StaticAssetProvider {
   #db: Knex;
   #logger: Logger;
-  #namespace: string | null;
+  #namespace: string;
 
   static async create(options: StaticAssetsStoreOptions) {
     const { database } = options;
@@ -68,7 +68,7 @@ export class StaticAssetsStore implements StaticAssetProvider {
   private constructor(client: Knex, logger: Logger, namespace?: string) {
     this.#db = client;
     this.#logger = logger;
-    this.#namespace = namespace ?? null;
+    this.#namespace = namespace ?? 'default';
   }
 
   /**
@@ -120,7 +120,7 @@ export class StaticAssetsStore implements StaticAssetProvider {
           content: await asset.content(),
           namespace: this.#namespace,
         })
-        .onConflict('path')
+        .onConflict(['namespace', 'path'])
         .ignore();
     }
   }

plugins/app-backend/src/migrations.test.ts

@@ -0,0 +1,141 @@
+/*
+ * Copyright 2022 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Knex } from 'knex';
+import { TestDatabases } from '@backstage/backend-test-utils';
+import fs from 'fs';
+
+const migrationsDir = `${__dirname}/../migrations`;
+const migrationsFiles = fs.readdirSync(migrationsDir).sort();
+
+async function migrateUpOnce(knex: Knex): Promise<void> {
+  await knex.migrate.up({ directory: migrationsDir });
+}
+
+async function migrateDownOnce(knex: Knex): Promise<void> {
+  await knex.migrate.down({ directory: migrationsDir });
+}
+
+async function migrateUntilBefore(knex: Knex, target: string): Promise<void> {
+  const index = migrationsFiles.indexOf(target);
+  if (index === -1) {
+    throw new Error(`Migration ${target} not found`);
+  }
+  for (let i = 0; i < index; i++) {
+    await migrateUpOnce(knex);
+  }
+}
+
+jest.setTimeout(60_000);
+
+describe('migrations', () => {
+  const databases = TestDatabases.create();
+
+  it.each(databases.eachSupportedId())(
+    '20211229105307_init.js, %p',
+    async databaseId => {
+      const knex = await databases.init(databaseId);
+
+      await migrateUntilBefore(knex, '20211229105307_init.js');
+      await migrateUpOnce(knex);
+
+      await knex('static_assets_cache').insert({
+        path: 'main.js',
+        content: Buffer.from('some-script'),
+        last_modified_at: knex.fn.now(),
+      });
+
+      await expect(knex('static_assets_cache')).resolves.toEqual([
+        {
+          path: 'main.js',
+          content: Buffer.from('some-script'),
+          last_modified_at: expect.anything(),
+        },
+      ]);
+
+      await migrateDownOnce(knex);
+
+      // This looks odd - you might expect a .toThrow at the end but that
+      // actually is flaky for some reason specifically on sqlite when
+      // performing multiple runs in sequence
+      await expect(knex('static_assets_cache')).rejects.toEqual(
+        expect.anything(),
+      );
+
+      await knex.destroy();
+    },
+  );
+
+  it.each(databases.eachSupportedId())(
+    '20240113144027_assets-namespace.js, %p',
+    async databaseId => {
+      const knex = await databases.init(databaseId);
+
+      await migrateUntilBefore(knex, '20240113144027_assets-namespace.js');
+
+      await knex('static_assets_cache').insert({
+        path: 'main.js',
+        content: Buffer.from('some-script'),
+        last_modified_at: knex.fn.now(),
+      });
+
+      await migrateUpOnce(knex);
+
+      await expect(knex('static_assets_cache')).resolves.toEqual([
+        {
+          path: 'main.js',
+          content: Buffer.from('some-script'),
+          namespace: 'default',
+          last_modified_at: expect.anything(),
+        },
+      ]);
+
+      await knex('static_assets_cache').insert({
+        path: 'main.js',
+        content: Buffer.from('other-script'),
+        namespace: 'other',
+        last_modified_at: knex.fn.now(),
+      });
+
+      await expect(knex('static_assets_cache')).resolves.toEqual([
+        {
+          path: 'main.js',
+          content: Buffer.from('some-script'),
+          namespace: 'default',
+          last_modified_at: expect.anything(),
+        },
+        {
+          path: 'main.js',
+          content: Buffer.from('other-script'),
+          namespace: 'other',
+          last_modified_at: expect.anything(),
+        },
+      ]);
+
+      await migrateDownOnce(knex);
+
+      await expect(knex('static_assets_cache')).resolves.toEqual([
+        {
+          path: 'main.js',
+          content: Buffer.from('some-script'),
+          last_modified_at: expect.anything(),
+        },
+      ]);
+
+      await knex.destroy();
+    },
+  );
+});

plugins/app-backend/src/setupTests.ts

@@ -14,4 +14,8 @@
  * limitations under the License.
  */
 
-export {};
+import { TestDatabases } from '@backstage/backend-test-utils';
+
+TestDatabases.setDefaults({
+  ids: ['MYSQL_8', 'POSTGRES_16', 'POSTGRES_12', 'SQLITE_3'],
+});