From 1220cf84d0771dbf985c86aba736007a23598a11 Mon Sep 17 00:00:00 2001 From: Heikki Hellgren Date: Tue, 17 Sep 2024 08:56:38 +0300 Subject: [PATCH 01/12] feat(backend): allow rate limiting requests to the backend uses redis for storing the data if it has been configured; otherwise falls back to memory configuration allows controlling almost all possible configuration available in the `express-rate-limit` library. Signed-off-by: Heikki Hellgren --- .changeset/famous-terms-rescue.md | 15 ++ packages/backend-defaults/config.d.ts | 68 ++++- packages/backend-defaults/package.json | 2 + .../report-rootHttpRouter.api.md | 11 + .../rootHttpRouter/http/MiddlewareFactory.ts | 86 ++++++- .../http/RateLimitStoreFactory.test.ts | 84 +++++++ .../http/RateLimitStoreFactory.ts | 69 ++++++ .../entrypoints/rootHttpRouter/http/index.ts | 1 + .../rootHttpRouterServiceFactory.ts | 5 +- yarn.lock | 234 +++++++++++++++++- 10 files changed, 558 insertions(+), 17 deletions(-) create mode 100644 .changeset/famous-terms-rescue.md create mode 100644 packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.test.ts create mode 100644 packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.ts diff --git a/.changeset/famous-terms-rescue.md b/.changeset/famous-terms-rescue.md new file mode 100644 index 0000000000..073f3a02ba --- /dev/null +++ b/.changeset/famous-terms-rescue.md @@ -0,0 +1,15 @@ +--- +'@backstage/backend-defaults': patch +--- + +Added new rate limit middleware to allow rate limiting requests to the backend + +Rate limiting can be turned on by adding the following configuration to `app-config.yaml`: + +```yaml +backend: + rateLimit: + enabled: true + windowMs: 60000 + limit: 100 +``` diff --git a/packages/backend-defaults/config.d.ts b/packages/backend-defaults/config.d.ts index 1ef054a537..7866b17c29 100644 --- a/packages/backend-defaults/config.d.ts +++ b/packages/backend-defaults/config.d.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { HumanDuration } from '@backstage/types'; +import { HumanDuration, JsonValue } from '@backstage/types'; export interface Config { app: { @@ -790,6 +790,72 @@ export interface Config { headers?: { [name: string]: string }; }; + /** + * Rate limiting options + */ + rateLimit?: { + /** + * Store to use for rate limiting. If not defined, the store will be automatically + * decided based on `backend.cache.store` value. If + * redis is not available, the store will be memory. + */ + store?: 'memory' | 'redis'; + /** + * Rate limiting enabled. Defaults to false. + */ + enabled?: boolean; + /** + * Time frame in milliseconds or as human duration for which requests are checked/remembered. + * Defaults to 6000ms. + */ + window?: number | HumanDuration; + /** + * The maximum number of connections to allow during the `window` before rate limiting the client. + * Defaults to 5. + */ + limit?: number; + /** + * The response body to send back when a client is rate limited. + * Defaults to 'Too many requests, please try again later.'. + */ + message?: JsonValue; + /** + * The HTTP status code to send back when a client is rate limited. + * Defaults to 429. + */ + statusCode?: number; + /** + * Whether to send the legacy rate limit headers for the limit. + * Defaults to true. + */ + legacyHeaders?: boolean; + /** + * Whether to enable support for headers conforming the RateLimit header fields for HTTP + * standardization. Defaults to undefined. + */ + standardHeaders?: 'draft-6' | 'draft-7'; + /** + * Whether to pass requests in case of store failure. + * Defaults to false. + */ + passOnStoreError?: boolean; + /** + * List of allowed IP addresses that are not rate limited. + * Defaults to [127.0.0.1]. + */ + ipAllowList?: string[]; + /** + * Skip rate limiting for requests that have been successful. + * Defaults to false. + */ + skipSuccessfulRequests?: boolean; + /** + * Skip rate limiting for requests that have failed. + * Defaults to false. + */ + skipFailedRequests?: boolean; + }; + /** * Configuration related to URL reading, used for example for reading catalog info * files, scaffolder templates, and techdocs content. diff --git a/packages/backend-defaults/package.json b/packages/backend-defaults/package.json index 3c74bad2d5..c10cb9f11f 100644 --- a/packages/backend-defaults/package.json +++ b/packages/backend-defaults/package.json @@ -168,6 +168,7 @@ "cron": "^3.0.0", "express": "^4.17.1", "express-promise-router": "^4.1.0", + "express-rate-limit": "^7.4.0", "fs-extra": "^11.2.0", "git-url-parse": "^15.0.0", "helmet": "^6.0.0", @@ -187,6 +188,7 @@ "pg": "^8.11.3", "pg-connection-string": "^2.3.0", "pg-format": "^1.0.4", + "rate-limit-redis": "^4.2.0", "raw-body": "^2.4.1", "selfsigned": "^2.0.0", "tar": "^6.1.12", diff --git a/packages/backend-defaults/report-rootHttpRouter.api.md b/packages/backend-defaults/report-rootHttpRouter.api.md index 826d592e8b..91e760b7ea 100644 --- a/packages/backend-defaults/report-rootHttpRouter.api.md +++ b/packages/backend-defaults/report-rootHttpRouter.api.md @@ -20,6 +20,7 @@ import { RootHttpRouterService } from '@backstage/backend-plugin-api'; import { Router } from 'express'; import type { Server } from 'node:http'; import { ServiceFactory } from '@backstage/backend-plugin-api'; +import type { Store } from 'express-rate-limit'; // @public (undocumented) export function createHealthRouter(options: { @@ -93,6 +94,7 @@ export class MiddlewareFactory { helmet(): RequestHandler; logging(): RequestHandler; notFound(): RequestHandler; + rateLimit(): RequestHandler; } // @public @@ -109,6 +111,15 @@ export interface MiddlewareFactoryOptions { logger: LoggerService; } +// @public +export class RateLimitStoreFactory { + constructor(config: Config); + // (undocumented) + create(): Store | undefined; + // (undocumented) + redis(): Store; +} + // @public export function readCorsOptions(config?: Config): CorsOptions; diff --git a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/MiddlewareFactory.ts b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/MiddlewareFactory.ts index 1ff9bd15d1..561a1999c9 100644 --- a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/MiddlewareFactory.ts +++ b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/MiddlewareFactory.ts @@ -15,15 +15,15 @@ */ import { - RootConfigService, LoggerService, + RootConfigService, } from '@backstage/backend-plugin-api'; import { - Request, - Response, ErrorRequestHandler, NextFunction, + Request, RequestHandler, + Response, } from 'express'; import cors from 'cors'; import helmet from 'helmet'; @@ -37,12 +37,19 @@ import { InputError, NotAllowedError, NotFoundError, + NotImplementedError, NotModifiedError, - ServiceUnavailableError, serializeError, + ServiceUnavailableError, } from '@backstage/errors'; -import { NotImplementedError } from '@backstage/errors'; import { applyInternalErrorFilter } from './applyInternalErrorFilter'; +import { DraftHeadersVersion, rateLimit } from 'express-rate-limit'; +import { + durationToMilliseconds, + HumanDuration, + JsonValue, +} from '@backstage/types'; +import { RateLimitStoreFactory } from './RateLimitStoreFactory'; type LogMeta = { date: string; @@ -227,6 +234,75 @@ export class MiddlewareFactory { return cors(readCorsOptions(this.#config.getOptionalConfig('backend'))); } + /** + * Returns a middleware that implements rate limiting. + * + * @remarks + * + * Rate limiting is a common technique to prevent abuse of APIs. This middleware is + * configured using the config key `backend.rateLimit`. + * + * @returns An Express request handler + */ + rateLimit(): RequestHandler { + const rateLimitOptions = + this.#config.getOptionalConfig('backend.rateLimit'); + const enabled = rateLimitOptions?.getOptionalBoolean('enabled') ?? false; + if (!rateLimitOptions || !enabled) { + return (_req: Request, _res: Response, next: NextFunction) => { + next(); + }; + } + + const window = rateLimitOptions.getOptional( + 'window', + ); + let windowMs: number | undefined; + if (window !== undefined) { + if (typeof window === 'number') { + windowMs = window; + } else if (typeof window === 'object' && !Array.isArray(window)) { + windowMs = durationToMilliseconds(window); + } else { + throw new Error( + `Invalid configuration backend.rateLimit.window: ${window}, expected milliseconds number or HumanDuration object`, + ); + } + } + + const ipAllowList = rateLimitOptions.getOptionalStringArray( + 'ipAllowList', + ) ?? ['127.0.0.1']; + + return rateLimit({ + windowMs, + limit: rateLimitOptions.getOptionalNumber('limit'), + message: rateLimitOptions.getOptional('message'), + statusCode: rateLimitOptions.getOptionalNumber('statusCode'), + skipSuccessfulRequests: rateLimitOptions.getOptionalBoolean( + 'skipSuccessfulRequests', + ), + skipFailedRequests: + rateLimitOptions.getOptionalBoolean('skipFailedRequests'), + legacyHeaders: rateLimitOptions.getOptionalBoolean('legacyHeaders'), + standardHeaders: rateLimitOptions.getOptionalString( + 'standardHeaders', + ) as DraftHeadersVersion, + passOnStoreError: rateLimitOptions.getOptionalBoolean('passOnStoreError'), + keyGenerator(req, _res): string { + if (!req.ip) { + return req.socket.remoteAddress!; + } + + return req.ip.replace(/:\d+[^:]*$/, ''); + }, + skip: (req, _res) => { + return Boolean(req.ip && ipAllowList.includes(req.ip)); + }, + store: new RateLimitStoreFactory(this.#config).create(), + }); + } + /** * Express middleware to handle errors during request processing. * diff --git a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.test.ts b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.test.ts new file mode 100644 index 0000000000..105cf0cac6 --- /dev/null +++ b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.test.ts @@ -0,0 +1,84 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { mockServices } from '@backstage/backend-test-utils'; +import { RateLimitStoreFactory } from './RateLimitStoreFactory'; +import { RedisStore } from 'rate-limit-redis'; +import KeyvRedis from '@keyv/redis'; + +jest.mock('@keyv/redis'); +(KeyvRedis as jest.Mocked).mockImplementation(() => { + return { + redis: { + call: jest.fn().mockResolvedValue('OK'), + }, + }; +}); + +describe('CacheRateLimitStoreFactory', () => { + it('should return redis store with auto configuration', () => { + const config = mockServices.rootConfig({ + data: { + backend: { + cache: { + store: 'redis', + }, + rateLimit: { + store: undefined, + }, + }, + }, + }); + const factory = new RateLimitStoreFactory(config); + const store = factory.create(); + expect(store).toBeInstanceOf(RedisStore); + }); + + it('should return undefined store with auto configuration if redis is not available', () => { + const config = mockServices.rootConfig({ + data: { + backend: { + cache: { + store: 'memory', + }, + database: { + client: 'sqlite3', + }, + rateLimit: { + store: undefined, + }, + }, + }, + }); + const factory = new RateLimitStoreFactory(config); + const store = factory.create(); + expect(store).toBeUndefined(); + }); + + it('should return redis store if configured explicitly', () => { + const config = mockServices.rootConfig({ + data: { + backend: { + rateLimit: { + store: 'redis', + }, + }, + }, + }); + const factory = new RateLimitStoreFactory(config); + const store = factory.create(); + expect(store).toBeInstanceOf(RedisStore); + }); +}); diff --git a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.ts b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.ts new file mode 100644 index 0000000000..00e5440ebe --- /dev/null +++ b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.ts @@ -0,0 +1,69 @@ +/* + * Copyright 2024 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { Config } from '@backstage/config'; +import type { Store } from 'express-rate-limit'; +import KeyvRedis from '@keyv/redis'; +import { RedisStore } from 'rate-limit-redis'; + +/** + * Creates a store for `express-rate-limit` based on the configuration. + * + * @public + */ +export class RateLimitStoreFactory { + constructor(private readonly config: Config) {} + + create(): Store | undefined { + const storeType = this.config.getOptionalString('backend.rateLimit.store'); + if (!storeType) { + return this.auto(); + } + switch (storeType) { + case 'redis': + return this.redis(); + default: + throw new Error( + `Invalid 'backend.rateLimit.store' provided: ${storeType}`, + ); + } + } + + private auto(): Store | undefined { + const cacheStore = + this.config.getOptionalString('backend.cache.store') || 'memory'; + // Use redis as primary if available + if (cacheStore === 'redis') { + return this.redis(); + } + + // Fallback to undefined (memory) + return undefined; + } + + redis(): Store { + const connectionString = + this.config.getOptionalString('backend.cache.connection') || ''; + const useRedisSets = + this.config.getOptionalBoolean('backend.cache.useRedisSets') ?? true; + const keyv = new KeyvRedis(connectionString, { + useRedisSets, + }); + return new RedisStore({ + // Keyv uses ioredis under the hood + sendCommand: (...args: string[]) => keyv.redis.call(...args), + }); + } +} diff --git a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/index.ts b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/index.ts index 4a9ec14cf8..5c63d887bb 100644 --- a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/index.ts +++ b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/index.ts @@ -28,3 +28,4 @@ export type { HttpServerCertificateOptions, HttpServerOptions, } from './types'; +export { RateLimitStoreFactory } from './RateLimitStoreFactory'; diff --git a/packages/backend-defaults/src/entrypoints/rootHttpRouter/rootHttpRouterServiceFactory.ts b/packages/backend-defaults/src/entrypoints/rootHttpRouter/rootHttpRouterServiceFactory.ts index da7fcfe3f9..7d0d778b0c 100644 --- a/packages/backend-defaults/src/entrypoints/rootHttpRouter/rootHttpRouterServiceFactory.ts +++ b/packages/backend-defaults/src/entrypoints/rootHttpRouter/rootHttpRouterServiceFactory.ts @@ -15,13 +15,13 @@ */ import { - RootConfigService, coreServices, createServiceFactory, LifecycleService, LoggerService, + RootConfigService, } from '@backstage/backend-plugin-api'; -import express, { RequestHandler, Express } from 'express'; +import express, { Express, RequestHandler } from 'express'; import type { Server } from 'node:http'; import { createHttpServer, @@ -117,6 +117,7 @@ const rootHttpRouterServiceFactoryWithOptions = ( if (trustProxy !== undefined) { app.set('trust proxy', trustProxy); } + app.use(middleware.rateLimit()); app.use(middleware.helmet()); app.use(middleware.cors()); app.use(middleware.compression()); diff --git a/yarn.lock b/yarn.lock index d8736591ea..0a73d77844 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3335,7 +3335,16 @@ __metadata: languageName: node linkType: hard -"@babel/runtime@npm:^7.0.0, @babel/runtime@npm:^7.1.2, @babel/runtime@npm:^7.10.1, @babel/runtime@npm:^7.12.1, @babel/runtime@npm:^7.12.5, @babel/runtime@npm:^7.13.10, @babel/runtime@npm:^7.17.8, @babel/runtime@npm:^7.18.3, @babel/runtime@npm:^7.18.6, @babel/runtime@npm:^7.20.13, @babel/runtime@npm:^7.20.6, @babel/runtime@npm:^7.21.0, @babel/runtime@npm:^7.23.9, @babel/runtime@npm:^7.26.10, @babel/runtime@npm:^7.3.1, @babel/runtime@npm:^7.4.4, @babel/runtime@npm:^7.5.5, @babel/runtime@npm:^7.6.0, @babel/runtime@npm:^7.7.6, @babel/runtime@npm:^7.8.3, @babel/runtime@npm:^7.8.4, @babel/runtime@npm:^7.8.7, @babel/runtime@npm:^7.9.2": +"@babel/runtime@npm:^7.0.0, @babel/runtime@npm:^7.1.2, @babel/runtime@npm:^7.10.1, @babel/runtime@npm:^7.12.1, @babel/runtime@npm:^7.12.5, @babel/runtime@npm:^7.13.10, @babel/runtime@npm:^7.17.8, @babel/runtime@npm:^7.18.3, @babel/runtime@npm:^7.18.6, @babel/runtime@npm:^7.20.13, @babel/runtime@npm:^7.20.6, @babel/runtime@npm:^7.21.0, @babel/runtime@npm:^7.23.9, @babel/runtime@npm:^7.3.1, @babel/runtime@npm:^7.4.4, @babel/runtime@npm:^7.5.5, @babel/runtime@npm:^7.6.0, @babel/runtime@npm:^7.7.6, @babel/runtime@npm:^7.8.3, @babel/runtime@npm:^7.8.4, @babel/runtime@npm:^7.8.7, @babel/runtime@npm:^7.9.2": + version: 7.26.7 + resolution: "@babel/runtime@npm:7.26.7" + dependencies: + regenerator-runtime: "npm:^0.14.0" + checksum: 10/c7a661a6836b332d9d2e047cba77ba1862c1e4f78cec7146db45808182ef7636d8a7170be9797e5d8fd513180bffb9fa16f6ca1c69341891efec56113cf22bfc + languageName: node + linkType: hard + +"@babel/runtime@npm:^7.26.10": version: 7.27.0 resolution: "@babel/runtime@npm:7.27.0" dependencies: @@ -3344,7 +3353,18 @@ __metadata: languageName: node linkType: hard -"@babel/template@npm:^7.22.5, @babel/template@npm:^7.24.7, @babel/template@npm:^7.25.9, @babel/template@npm:^7.27.0, @babel/template@npm:^7.3.3": +"@babel/template@npm:^7.22.5, @babel/template@npm:^7.24.7, @babel/template@npm:^7.25.9, @babel/template@npm:^7.3.3": + version: 7.25.9 + resolution: "@babel/template@npm:7.25.9" + dependencies: + "@babel/code-frame": "npm:^7.25.9" + "@babel/parser": "npm:^7.25.9" + "@babel/types": "npm:^7.25.9" + checksum: 10/e861180881507210150c1335ad94aff80fd9e9be6202e1efa752059c93224e2d5310186ddcdd4c0f0b0fc658ce48cb47823f15142b5c00c8456dde54f5de80b2 + languageName: node + linkType: hard + +"@babel/template@npm:^7.27.0": version: 7.27.0 resolution: "@babel/template@npm:7.27.0" dependencies: @@ -3370,7 +3390,17 @@ __metadata: languageName: node linkType: hard -"@babel/types@npm:^7.0.0, @babel/types@npm:^7.18.9, @babel/types@npm:^7.20.0, @babel/types@npm:^7.20.7, @babel/types@npm:^7.22.10, @babel/types@npm:^7.22.5, @babel/types@npm:^7.24.7, @babel/types@npm:^7.24.8, @babel/types@npm:^7.25.9, @babel/types@npm:^7.26.0, @babel/types@npm:^7.27.0, @babel/types@npm:^7.3.3, @babel/types@npm:^7.4.4": +"@babel/types@npm:^7.0.0, @babel/types@npm:^7.18.9, @babel/types@npm:^7.20.0, @babel/types@npm:^7.20.7, @babel/types@npm:^7.22.10, @babel/types@npm:^7.22.5, @babel/types@npm:^7.24.7, @babel/types@npm:^7.24.8, @babel/types@npm:^7.25.9, @babel/types@npm:^7.26.0, @babel/types@npm:^7.3.3, @babel/types@npm:^7.4.4": + version: 7.26.0 + resolution: "@babel/types@npm:7.26.0" + dependencies: + "@babel/helper-string-parser": "npm:^7.25.9" + "@babel/helper-validator-identifier": "npm:^7.25.9" + checksum: 10/40780741ecec886ed9edae234b5eb4976968cc70d72b4e5a40d55f83ff2cc457de20f9b0f4fe9d858350e43dab0ea496e7ef62e2b2f08df699481a76df02cd6e + languageName: node + linkType: hard + +"@babel/types@npm:^7.27.0": version: 7.27.0 resolution: "@babel/types@npm:7.27.0" dependencies: @@ -3602,6 +3632,7 @@ __metadata: cron: "npm:^3.0.0" express: "npm:^4.17.1" express-promise-router: "npm:^4.1.0" + express-rate-limit: "npm:^7.4.0" fs-extra: "npm:^11.2.0" git-url-parse: "npm:^15.0.0" helmet: "npm:^6.0.0" @@ -3624,6 +3655,7 @@ __metadata: pg: "npm:^8.11.3" pg-connection-string: "npm:^2.3.0" pg-format: "npm:^1.0.4" + rate-limit-redis: "npm:^4.2.0" raw-body: "npm:^2.4.1" selfsigned: "npm:^2.0.0" supertest: "npm:^7.0.0" @@ -19193,6 +19225,13 @@ __metadata: languageName: node linkType: hard +"@swc/core-darwin-arm64@npm:1.10.6": + version: 1.10.6 + resolution: "@swc/core-darwin-arm64@npm:1.10.6" + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: hard + "@swc/core-darwin-arm64@npm:1.11.24": version: 1.11.24 resolution: "@swc/core-darwin-arm64@npm:1.11.24" @@ -19200,6 +19239,13 @@ __metadata: languageName: node linkType: hard +"@swc/core-darwin-x64@npm:1.10.6": + version: 1.10.6 + resolution: "@swc/core-darwin-x64@npm:1.10.6" + conditions: os=darwin & cpu=x64 + languageName: node + linkType: hard + "@swc/core-darwin-x64@npm:1.11.24": version: 1.11.24 resolution: "@swc/core-darwin-x64@npm:1.11.24" @@ -19207,6 +19253,13 @@ __metadata: languageName: node linkType: hard +"@swc/core-linux-arm-gnueabihf@npm:1.10.6": + version: 1.10.6 + resolution: "@swc/core-linux-arm-gnueabihf@npm:1.10.6" + conditions: os=linux & cpu=arm + languageName: node + linkType: hard + "@swc/core-linux-arm-gnueabihf@npm:1.11.24": version: 1.11.24 resolution: "@swc/core-linux-arm-gnueabihf@npm:1.11.24" @@ -19214,6 +19267,13 @@ __metadata: languageName: node linkType: hard +"@swc/core-linux-arm64-gnu@npm:1.10.6": + version: 1.10.6 + resolution: "@swc/core-linux-arm64-gnu@npm:1.10.6" + conditions: os=linux & cpu=arm64 & libc=glibc + languageName: node + linkType: hard + "@swc/core-linux-arm64-gnu@npm:1.11.24": version: 1.11.24 resolution: "@swc/core-linux-arm64-gnu@npm:1.11.24" @@ -19221,6 +19281,13 @@ __metadata: languageName: node linkType: hard +"@swc/core-linux-arm64-musl@npm:1.10.6": + version: 1.10.6 + resolution: "@swc/core-linux-arm64-musl@npm:1.10.6" + conditions: os=linux & cpu=arm64 & libc=musl + languageName: node + linkType: hard + "@swc/core-linux-arm64-musl@npm:1.11.24": version: 1.11.24 resolution: "@swc/core-linux-arm64-musl@npm:1.11.24" @@ -19228,6 +19295,13 @@ __metadata: languageName: node linkType: hard +"@swc/core-linux-x64-gnu@npm:1.10.6": + version: 1.10.6 + resolution: "@swc/core-linux-x64-gnu@npm:1.10.6" + conditions: os=linux & cpu=x64 & libc=glibc + languageName: node + linkType: hard + "@swc/core-linux-x64-gnu@npm:1.11.24": version: 1.11.24 resolution: "@swc/core-linux-x64-gnu@npm:1.11.24" @@ -19235,6 +19309,13 @@ __metadata: languageName: node linkType: hard +"@swc/core-linux-x64-musl@npm:1.10.6": + version: 1.10.6 + resolution: "@swc/core-linux-x64-musl@npm:1.10.6" + conditions: os=linux & cpu=x64 & libc=musl + languageName: node + linkType: hard + "@swc/core-linux-x64-musl@npm:1.11.24": version: 1.11.24 resolution: "@swc/core-linux-x64-musl@npm:1.11.24" @@ -19242,6 +19323,13 @@ __metadata: languageName: node linkType: hard +"@swc/core-win32-arm64-msvc@npm:1.10.6": + version: 1.10.6 + resolution: "@swc/core-win32-arm64-msvc@npm:1.10.6" + conditions: os=win32 & cpu=arm64 + languageName: node + linkType: hard + "@swc/core-win32-arm64-msvc@npm:1.11.24": version: 1.11.24 resolution: "@swc/core-win32-arm64-msvc@npm:1.11.24" @@ -19249,6 +19337,13 @@ __metadata: languageName: node linkType: hard +"@swc/core-win32-ia32-msvc@npm:1.10.6": + version: 1.10.6 + resolution: "@swc/core-win32-ia32-msvc@npm:1.10.6" + conditions: os=win32 & cpu=ia32 + languageName: node + linkType: hard + "@swc/core-win32-ia32-msvc@npm:1.11.24": version: 1.11.24 resolution: "@swc/core-win32-ia32-msvc@npm:1.11.24" @@ -19256,6 +19351,13 @@ __metadata: languageName: node linkType: hard +"@swc/core-win32-x64-msvc@npm:1.10.6": + version: 1.10.6 + resolution: "@swc/core-win32-x64-msvc@npm:1.10.6" + conditions: os=win32 & cpu=x64 + languageName: node + linkType: hard + "@swc/core-win32-x64-msvc@npm:1.11.24": version: 1.11.24 resolution: "@swc/core-win32-x64-msvc@npm:1.11.24" @@ -19263,7 +19365,7 @@ __metadata: languageName: node linkType: hard -"@swc/core@npm:^1.10.8, @swc/core@npm:^1.3.46": +"@swc/core@npm:^1.10.8": version: 1.11.24 resolution: "@swc/core@npm:1.11.24" dependencies: @@ -19309,6 +19411,52 @@ __metadata: languageName: node linkType: hard +"@swc/core@npm:^1.3.46": + version: 1.10.6 + resolution: "@swc/core@npm:1.10.6" + dependencies: + "@swc/core-darwin-arm64": "npm:1.10.6" + "@swc/core-darwin-x64": "npm:1.10.6" + "@swc/core-linux-arm-gnueabihf": "npm:1.10.6" + "@swc/core-linux-arm64-gnu": "npm:1.10.6" + "@swc/core-linux-arm64-musl": "npm:1.10.6" + "@swc/core-linux-x64-gnu": "npm:1.10.6" + "@swc/core-linux-x64-musl": "npm:1.10.6" + "@swc/core-win32-arm64-msvc": "npm:1.10.6" + "@swc/core-win32-ia32-msvc": "npm:1.10.6" + "@swc/core-win32-x64-msvc": "npm:1.10.6" + "@swc/counter": "npm:^0.1.3" + "@swc/types": "npm:^0.1.17" + peerDependencies: + "@swc/helpers": "*" + dependenciesMeta: + "@swc/core-darwin-arm64": + optional: true + "@swc/core-darwin-x64": + optional: true + "@swc/core-linux-arm-gnueabihf": + optional: true + "@swc/core-linux-arm64-gnu": + optional: true + "@swc/core-linux-arm64-musl": + optional: true + "@swc/core-linux-x64-gnu": + optional: true + "@swc/core-linux-x64-musl": + optional: true + "@swc/core-win32-arm64-msvc": + optional: true + "@swc/core-win32-ia32-msvc": + optional: true + "@swc/core-win32-x64-msvc": + optional: true + peerDependenciesMeta: + "@swc/helpers": + optional: true + checksum: 10/51eccbba6ee8a41f57a6ba4213ec05859434f52fd6698f508c92a8bb467afda56a1e95b07985faf059b27cb28e77d479340de7210a8616976ea82f774a6d86a3 + languageName: node + linkType: hard + "@swc/counter@npm:^0.1.3": version: 0.1.3 resolution: "@swc/counter@npm:0.1.3" @@ -19338,6 +19486,15 @@ __metadata: languageName: node linkType: hard +"@swc/types@npm:^0.1.17": + version: 0.1.17 + resolution: "@swc/types@npm:0.1.17" + dependencies: + "@swc/counter": "npm:^0.1.3" + checksum: 10/ddef1ad5bfead3acdfc41f14e79ba43a99200eb325afbad5716058dbe36358b0513400e9f22aff32432be84a98ae93df95a20b94192f69b8687144270e4eaa18 + languageName: node + linkType: hard + "@swc/types@npm:^0.1.21": version: 0.1.21 resolution: "@swc/types@npm:0.1.21" @@ -20659,7 +20816,16 @@ __metadata: languageName: node linkType: hard -"@types/node@npm:*, @types/node@npm:>=12, @types/node@npm:>=12.0.0, @types/node@npm:>=13.7.0, @types/node@npm:>=18.0.0, @types/node@npm:^22.0.0": +"@types/node@npm:*, @types/node@npm:>=13.7.0, @types/node@npm:^22.0.0": + version: 22.10.5 + resolution: "@types/node@npm:22.10.5" + dependencies: + undici-types: "npm:~6.20.0" + checksum: 10/a5366961ffa9921e8f15435bc18ea9f8b7a7bb6b3d92dd5e93ebcd25e8af65708872bd8e6fee274b4655bab9ca80fbff9f0e42b5b53857790f13cf68cf4cbbfc + languageName: node + linkType: hard + +"@types/node@npm:>=12, @types/node@npm:>=12.0.0, @types/node@npm:>=18.0.0": version: 22.13.10 resolution: "@types/node@npm:22.13.10" dependencies: @@ -24011,13 +24177,20 @@ __metadata: languageName: node linkType: hard -"async@npm:^3.2.2, async@npm:^3.2.3, async@npm:^3.2.4, async@npm:^3.2.6": +"async@npm:^3.2.2, async@npm:^3.2.6": version: 3.2.6 resolution: "async@npm:3.2.6" checksum: 10/cb6e0561a3c01c4b56a799cc8bab6ea5fef45f069ab32500b6e19508db270ef2dffa55e5aed5865c5526e9907b1f8be61b27530823b411ffafb5e1538c86c368 languageName: node linkType: hard +"async@npm:^3.2.3, async@npm:^3.2.4": + version: 3.2.4 + resolution: "async@npm:3.2.4" + checksum: 10/bebb5dc2258c45b83fa1d3be179ae0eb468e1646a62d443c8d60a45e84041b28fccebe1e2d1f234bfc3dcad44e73dcdbf4ba63d98327c9f6556e3dbd47c2ae8b + languageName: node + linkType: hard + "asynckit@npm:^0.4.0": version: 0.4.0 resolution: "asynckit@npm:0.4.0" @@ -24509,13 +24682,20 @@ __metadata: languageName: node linkType: hard -"before-after-hook@npm:^2.1.0, before-after-hook@npm:^2.2.0": +"before-after-hook@npm:^2.1.0": version: 2.2.3 resolution: "before-after-hook@npm:2.2.3" checksum: 10/e676f769dbc4abcf4b3317db2fd2badb4a92c0710e0a7da12cf14b59c3482d4febf835ad7de7874499060fd4e13adf0191628e504728b3c5bb4ec7a878c09940 languageName: node linkType: hard +"before-after-hook@npm:^2.2.0": + version: 2.2.2 + resolution: "before-after-hook@npm:2.2.2" + checksum: 10/34c190def503f771f8811db0bd0c62b35301fe6059c8d847664633ce0548e8253e2661104ba66c71a85548746ba87d5ff2ebf5278c1f3ad367d111ffc9a26bb4 + languageName: node + linkType: hard + "better-opn@npm:^3.0.2": version: 3.0.2 resolution: "better-opn@npm:3.0.2" @@ -29865,6 +30045,15 @@ __metadata: languageName: node linkType: hard +"express-rate-limit@npm:^7.4.0": + version: 7.4.0 + resolution: "express-rate-limit@npm:7.4.0" + peerDependencies: + express: 4 || 5 || ^5.0.0-beta.1 + checksum: 10/33178c652bb1472aad2022194b5cd7963bd3e74d3eaf5e49eb1491a968fdce54551cc76b097ac10d3a1646d62cec2e6f2405ccef5ef5b60152a0c4a148749a4d + languageName: node + linkType: hard + "express-session@npm:^1.17.1, express-session@npm:^1.17.3": version: 1.18.1 resolution: "express-session@npm:1.18.1" @@ -31040,7 +31229,25 @@ __metadata: languageName: node linkType: hard -"get-intrinsic@npm:^1.1.3, get-intrinsic@npm:^1.2.1, get-intrinsic@npm:^1.2.4, get-intrinsic@npm:^1.2.5, get-intrinsic@npm:^1.2.6, get-intrinsic@npm:^1.3.0": +"get-intrinsic@npm:^1.1.3, get-intrinsic@npm:^1.2.4, get-intrinsic@npm:^1.2.5, get-intrinsic@npm:^1.2.6": + version: 1.2.6 + resolution: "get-intrinsic@npm:1.2.6" + dependencies: + call-bind-apply-helpers: "npm:^1.0.1" + dunder-proto: "npm:^1.0.0" + es-define-property: "npm:^1.0.1" + es-errors: "npm:^1.3.0" + es-object-atoms: "npm:^1.0.0" + function-bind: "npm:^1.1.2" + gopd: "npm:^1.2.0" + has-symbols: "npm:^1.1.0" + hasown: "npm:^2.0.2" + math-intrinsics: "npm:^1.0.0" + checksum: 10/a1ffae6d7893a6fa0f4d1472adbc85095edd6b3b0943ead97c3738539cecb19d422ff4d48009eed8c3c27ad678c2b1e38a83b1a1e96b691d13ed8ecefca1068d + languageName: node + linkType: hard + +"get-intrinsic@npm:^1.2.1, get-intrinsic@npm:^1.3.0": version: 1.3.0 resolution: "get-intrinsic@npm:1.3.0" dependencies: @@ -36734,7 +36941,7 @@ __metadata: languageName: node linkType: hard -"math-intrinsics@npm:^1.1.0": +"math-intrinsics@npm:^1.0.0, math-intrinsics@npm:^1.1.0": version: 1.1.0 resolution: "math-intrinsics@npm:1.1.0" checksum: 10/11df2eda46d092a6035479632e1ec865b8134bdfc4bd9e571a656f4191525404f13a283a515938c3a8de934dbfd9c09674d9da9fa831e6eb7e22b50b197d2edd @@ -41835,6 +42042,15 @@ __metadata: languageName: node linkType: hard +"rate-limit-redis@npm:^4.2.0": + version: 4.2.0 + resolution: "rate-limit-redis@npm:4.2.0" + peerDependencies: + express-rate-limit: ">= 6" + checksum: 10/22adc67918ca906f613b45f9dcfd039f543d363921979d21ba56be5f3288c6e9973c9e4bb4ec59810fc6b3abb20defd572c102f607a8c3b4d273d5e09b63839f + languageName: node + linkType: hard + "rate-limiter-flexible@npm:^4.0.1": version: 4.0.1 resolution: "rate-limiter-flexible@npm:4.0.1" From 7bd8af72ad11eafda24b8c5ac5397ac6c825c170 Mon Sep 17 00:00:00 2001 From: Heikki Hellgren Date: Tue, 17 Sep 2024 13:38:41 +0300 Subject: [PATCH 02/12] feat(backend): add support for trustProxy in backend config this is required for the rate limiting to work behind proxy. closes #24169 Signed-off-by: Heikki Hellgren --- .changeset/sour-comics-attend.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/sour-comics-attend.md diff --git a/.changeset/sour-comics-attend.md b/.changeset/sour-comics-attend.md new file mode 100644 index 0000000000..729fc59ba7 --- /dev/null +++ b/.changeset/sour-comics-attend.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-defaults': patch +--- + +Add configuration variable for `express` trust proxy setting From 279c15cb5dc3d6151fc829e60cd3b0e471dee8f8 Mon Sep 17 00:00:00 2001 From: Heikki Hellgren Date: Tue, 17 Sep 2024 17:19:45 +0300 Subject: [PATCH 03/12] fix: skip trust proxy validation in rate limiting Signed-off-by: Heikki Hellgren --- .changeset/famous-terms-rescue.md | 5 +- packages/backend-defaults/config.d.ts | 106 +++++++----------- .../report-rootHttpRouter.api.md | 10 -- .../rootHttpRouter/http/MiddlewareFactory.ts | 43 ++++--- .../http/RateLimitStoreFactory.test.ts | 50 +++------ .../http/RateLimitStoreFactory.ts | 40 ++----- .../entrypoints/rootHttpRouter/http/index.ts | 1 - 7 files changed, 91 insertions(+), 164 deletions(-) diff --git a/.changeset/famous-terms-rescue.md b/.changeset/famous-terms-rescue.md index 073f3a02ba..78630eb532 100644 --- a/.changeset/famous-terms-rescue.md +++ b/.changeset/famous-terms-rescue.md @@ -9,7 +9,6 @@ Rate limiting can be turned on by adding the following configuration to `app-con ```yaml backend: rateLimit: - enabled: true - windowMs: 60000 - limit: 100 + window: 60000 + incomingRequestLimit: 100 ``` diff --git a/packages/backend-defaults/config.d.ts b/packages/backend-defaults/config.d.ts index 7866b17c29..49a51ec2f5 100644 --- a/packages/backend-defaults/config.d.ts +++ b/packages/backend-defaults/config.d.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { HumanDuration, JsonValue } from '@backstage/types'; +import { HumanDuration } from '@backstage/types'; export interface Config { app: { @@ -793,68 +793,48 @@ export interface Config { /** * Rate limiting options */ - rateLimit?: { - /** - * Store to use for rate limiting. If not defined, the store will be automatically - * decided based on `backend.cache.store` value. If - * redis is not available, the store will be memory. - */ - store?: 'memory' | 'redis'; - /** - * Rate limiting enabled. Defaults to false. - */ - enabled?: boolean; - /** - * Time frame in milliseconds or as human duration for which requests are checked/remembered. - * Defaults to 6000ms. - */ - window?: number | HumanDuration; - /** - * The maximum number of connections to allow during the `window` before rate limiting the client. - * Defaults to 5. - */ - limit?: number; - /** - * The response body to send back when a client is rate limited. - * Defaults to 'Too many requests, please try again later.'. - */ - message?: JsonValue; - /** - * The HTTP status code to send back when a client is rate limited. - * Defaults to 429. - */ - statusCode?: number; - /** - * Whether to send the legacy rate limit headers for the limit. - * Defaults to true. - */ - legacyHeaders?: boolean; - /** - * Whether to enable support for headers conforming the RateLimit header fields for HTTP - * standardization. Defaults to undefined. - */ - standardHeaders?: 'draft-6' | 'draft-7'; - /** - * Whether to pass requests in case of store failure. - * Defaults to false. - */ - passOnStoreError?: boolean; - /** - * List of allowed IP addresses that are not rate limited. - * Defaults to [127.0.0.1]. - */ - ipAllowList?: string[]; - /** - * Skip rate limiting for requests that have been successful. - * Defaults to false. - */ - skipSuccessfulRequests?: boolean; - /** - * Skip rate limiting for requests that have failed. - * Defaults to false. - */ - skipFailedRequests?: boolean; - }; + rateLimit?: + | false + | { + store?: + | { + client: 'redis'; + connection: string; + } + | { + client: 'memory'; + }; + /** + * Time frame in milliseconds or as human duration for which requests are checked/remembered. + * Defaults to 6000ms. + */ + window?: number | HumanDuration; + /** + * The maximum number of connections to allow during the `window` before rate limiting the client. + * Defaults to 5. + */ + incomingRequestLimit?: number; + /** + * Whether to pass requests in case of store failure. + * Defaults to false. + */ + passOnStoreError?: boolean; + /** + * List of allowed IP addresses that are not rate limited. + * Defaults to [127.0.0.1, 0:0:0:0:0:0:0:1, ::1]. + */ + ipAllowList?: string[]; + /** + * Skip rate limiting for requests that have been successful. + * Defaults to false. + */ + skipSuccessfulRequests?: boolean; + /** + * Skip rate limiting for requests that have failed. + * Defaults to false. + */ + skipFailedRequests?: boolean; + }; /** * Configuration related to URL reading, used for example for reading catalog info diff --git a/packages/backend-defaults/report-rootHttpRouter.api.md b/packages/backend-defaults/report-rootHttpRouter.api.md index 91e760b7ea..84424be39d 100644 --- a/packages/backend-defaults/report-rootHttpRouter.api.md +++ b/packages/backend-defaults/report-rootHttpRouter.api.md @@ -20,7 +20,6 @@ import { RootHttpRouterService } from '@backstage/backend-plugin-api'; import { Router } from 'express'; import type { Server } from 'node:http'; import { ServiceFactory } from '@backstage/backend-plugin-api'; -import type { Store } from 'express-rate-limit'; // @public (undocumented) export function createHealthRouter(options: { @@ -111,15 +110,6 @@ export interface MiddlewareFactoryOptions { logger: LoggerService; } -// @public -export class RateLimitStoreFactory { - constructor(config: Config); - // (undocumented) - create(): Store | undefined; - // (undocumented) - redis(): Store; -} - // @public export function readCorsOptions(config?: Config): CorsOptions; diff --git a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/MiddlewareFactory.ts b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/MiddlewareFactory.ts index 561a1999c9..853e5aa8f6 100644 --- a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/MiddlewareFactory.ts +++ b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/MiddlewareFactory.ts @@ -43,12 +43,9 @@ import { ServiceUnavailableError, } from '@backstage/errors'; import { applyInternalErrorFilter } from './applyInternalErrorFilter'; -import { DraftHeadersVersion, rateLimit } from 'express-rate-limit'; -import { - durationToMilliseconds, - HumanDuration, - JsonValue, -} from '@backstage/types'; +import { rateLimit } from 'express-rate-limit'; +import { Config } from '@backstage/config'; +import { durationToMilliseconds, HumanDuration } from '@backstage/types'; import { RateLimitStoreFactory } from './RateLimitStoreFactory'; type LogMeta = { @@ -245,15 +242,15 @@ export class MiddlewareFactory { * @returns An Express request handler */ rateLimit(): RequestHandler { - const rateLimitOptions = - this.#config.getOptionalConfig('backend.rateLimit'); - const enabled = rateLimitOptions?.getOptionalBoolean('enabled') ?? false; - if (!rateLimitOptions || !enabled) { + const conf = this.#config.getOptional( + 'backend.rateLimit', + ); + if (!conf || typeof conf !== 'object') { return (_req: Request, _res: Response, next: NextFunction) => { next(); }; } - + const rateLimitOptions = conf as Config; const window = rateLimitOptions.getOptional( 'window', ); @@ -272,32 +269,34 @@ export class MiddlewareFactory { const ipAllowList = rateLimitOptions.getOptionalStringArray( 'ipAllowList', - ) ?? ['127.0.0.1']; + ) ?? ['127.0.0.1', '0:0:0:0:0:0:0:1', '::1']; return rateLimit({ windowMs, - limit: rateLimitOptions.getOptionalNumber('limit'), - message: rateLimitOptions.getOptional('message'), - statusCode: rateLimitOptions.getOptionalNumber('statusCode'), + limit: rateLimitOptions.getOptionalNumber('incomingRequestLimit'), skipSuccessfulRequests: rateLimitOptions.getOptionalBoolean( 'skipSuccessfulRequests', ), skipFailedRequests: rateLimitOptions.getOptionalBoolean('skipFailedRequests'), - legacyHeaders: rateLimitOptions.getOptionalBoolean('legacyHeaders'), - standardHeaders: rateLimitOptions.getOptionalString( - 'standardHeaders', - ) as DraftHeadersVersion, passOnStoreError: rateLimitOptions.getOptionalBoolean('passOnStoreError'), keyGenerator(req, _res): string { if (!req.ip) { return req.socket.remoteAddress!; } - - return req.ip.replace(/:\d+[^:]*$/, ''); + return req.ip; }, skip: (req, _res) => { - return Boolean(req.ip && ipAllowList.includes(req.ip)); + return ( + Boolean(req.ip && ipAllowList.includes(req.ip)) || + Boolean( + req.socket.remoteAddress && + ipAllowList.includes(req.socket.remoteAddress), + ) + ); + }, + validate: { + trustProxy: false, }, store: new RateLimitStoreFactory(this.#config).create(), }); diff --git a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.test.ts b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.test.ts index 105cf0cac6..4fdc513c81 100644 --- a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.test.ts +++ b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.test.ts @@ -13,49 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { mockServices } from '@backstage/backend-test-utils'; +import { mockServices, TestCaches } from '@backstage/backend-test-utils'; import { RateLimitStoreFactory } from './RateLimitStoreFactory'; import { RedisStore } from 'rate-limit-redis'; -import KeyvRedis from '@keyv/redis'; - -jest.mock('@keyv/redis'); -(KeyvRedis as jest.Mocked).mockImplementation(() => { - return { - redis: { - call: jest.fn().mockResolvedValue('OK'), - }, - }; -}); describe('CacheRateLimitStoreFactory', () => { - it('should return redis store with auto configuration', () => { - const config = mockServices.rootConfig({ - data: { - backend: { - cache: { - store: 'redis', - }, - rateLimit: { - store: undefined, - }, - }, - }, - }); - const factory = new RateLimitStoreFactory(config); - const store = factory.create(); - expect(store).toBeInstanceOf(RedisStore); - }); + const caches = TestCaches.create(); + + afterEach(jest.clearAllMocks); it('should return undefined store with auto configuration if redis is not available', () => { const config = mockServices.rootConfig({ data: { backend: { - cache: { - store: 'memory', - }, - database: { - client: 'sqlite3', - }, rateLimit: { store: undefined, }, @@ -67,12 +37,20 @@ describe('CacheRateLimitStoreFactory', () => { expect(store).toBeUndefined(); }); - it('should return redis store if configured explicitly', () => { + it('should return redis store if configured explicitly', async () => { + if (!caches.supports('REDIS_7')) { + return; + } + const conf = await caches.init('REDIS_7'); + const config = mockServices.rootConfig({ data: { backend: { rateLimit: { - store: 'redis', + store: { + client: 'redis', + connection: conf.connection, + }, }, }, }, diff --git a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.ts b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.ts index 00e5440ebe..85eb3c4d86 100644 --- a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.ts +++ b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.ts @@ -21,46 +21,28 @@ import { RedisStore } from 'rate-limit-redis'; /** * Creates a store for `express-rate-limit` based on the configuration. * - * @public + * @internal */ export class RateLimitStoreFactory { constructor(private readonly config: Config) {} create(): Store | undefined { - const storeType = this.config.getOptionalString('backend.rateLimit.store'); - if (!storeType) { - return this.auto(); + const store = this.config.getOptionalConfig('backend.rateLimit.store'); + if (!store) { + return undefined; } - switch (storeType) { + const client = store.getString('client'); + switch (client) { case 'redis': - return this.redis(); + return this.redis(store); default: - throw new Error( - `Invalid 'backend.rateLimit.store' provided: ${storeType}`, - ); + return undefined; } } - private auto(): Store | undefined { - const cacheStore = - this.config.getOptionalString('backend.cache.store') || 'memory'; - // Use redis as primary if available - if (cacheStore === 'redis') { - return this.redis(); - } - - // Fallback to undefined (memory) - return undefined; - } - - redis(): Store { - const connectionString = - this.config.getOptionalString('backend.cache.connection') || ''; - const useRedisSets = - this.config.getOptionalBoolean('backend.cache.useRedisSets') ?? true; - const keyv = new KeyvRedis(connectionString, { - useRedisSets, - }); + redis(storeConfig: Config): Store { + const connectionString = storeConfig.getString('connection'); + const keyv = new KeyvRedis(connectionString); return new RedisStore({ // Keyv uses ioredis under the hood sendCommand: (...args: string[]) => keyv.redis.call(...args), diff --git a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/index.ts b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/index.ts index 5c63d887bb..4a9ec14cf8 100644 --- a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/index.ts +++ b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/index.ts @@ -28,4 +28,3 @@ export type { HttpServerCertificateOptions, HttpServerOptions, } from './types'; -export { RateLimitStoreFactory } from './RateLimitStoreFactory'; From d6bd7a540d8ab2b3eee0bcce0bcb426514cf9226 Mon Sep 17 00:00:00 2001 From: Heikki Hellgren Date: Tue, 26 Nov 2024 14:33:30 +0200 Subject: [PATCH 04/12] fix: review findings Signed-off-by: Heikki Hellgren --- .changeset/famous-terms-rescue.md | 3 +- app-config.yaml | 6 ++ packages/backend-defaults/config.d.ts | 8 +-- packages/backend-defaults/package.json | 2 +- .../rootHttpRouter/http/MiddlewareFactory.ts | 62 +++++++++++-------- .../http/RateLimitStoreFactory.test.ts | 35 ++++++----- .../http/RateLimitStoreFactory.ts | 16 ++--- .../rootHttpRouterServiceFactory.ts | 2 +- yarn.lock | 12 ++-- 9 files changed, 83 insertions(+), 63 deletions(-) diff --git a/.changeset/famous-terms-rescue.md b/.changeset/famous-terms-rescue.md index 78630eb532..9f38986934 100644 --- a/.changeset/famous-terms-rescue.md +++ b/.changeset/famous-terms-rescue.md @@ -4,11 +4,12 @@ Added new rate limit middleware to allow rate limiting requests to the backend +If you are using the `configure` callback of the root HTTP router service and do NOT call `applyDefaults()` inside it, please see [the relevant changes](https://github.com/backstage/backstage/pull/26725/files#diff-86ad1b6a694dd250823aee39d410428dd837c9d9a04ca8c33bd1081fbe3f22af) that were made, to see if you want to apply them as well to your custom configuration. Rate limiting can be turned on by adding the following configuration to `app-config.yaml`: ```yaml backend: rateLimit: - window: 60000 + window: 6000ms incomingRequestLimit: 100 ``` diff --git a/app-config.yaml b/app-config.yaml index eaee401d09..df80c7add0 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -36,6 +36,12 @@ backend: # keys: # - secret: ${BACKEND_SECRET} + # Used for testing rate limiting locally + # rateLimit: + # windowMs: 60000 + # incomingRequestLimit: 1 + # ipAllowList: [] + auth: # TODO: once plugins have been migrated we can remove this, but right now it # is require for the backend-next to work in this repo diff --git a/packages/backend-defaults/config.d.ts b/packages/backend-defaults/config.d.ts index 49a51ec2f5..6cd1aa183a 100644 --- a/packages/backend-defaults/config.d.ts +++ b/packages/backend-defaults/config.d.ts @@ -791,10 +791,10 @@ export interface Config { }; /** - * Rate limiting options + * Rate limiting options. Defining this as `true` will enable rate limiting with default values. */ rateLimit?: - | false + | true | { store?: | { @@ -806,9 +806,9 @@ export interface Config { }; /** * Time frame in milliseconds or as human duration for which requests are checked/remembered. - * Defaults to 6000ms. + * Defaults to one minute. */ - window?: number | HumanDuration; + window?: string | HumanDuration; /** * The maximum number of connections to allow during the `window` before rate limiting the client. * Defaults to 5. diff --git a/packages/backend-defaults/package.json b/packages/backend-defaults/package.json index c10cb9f11f..b1f9513f2c 100644 --- a/packages/backend-defaults/package.json +++ b/packages/backend-defaults/package.json @@ -168,7 +168,7 @@ "cron": "^3.0.0", "express": "^4.17.1", "express-promise-router": "^4.1.0", - "express-rate-limit": "^7.4.0", + "express-rate-limit": "^7.5.0", "fs-extra": "^11.2.0", "git-url-parse": "^15.0.0", "helmet": "^6.0.0", diff --git a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/MiddlewareFactory.ts b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/MiddlewareFactory.ts index 853e5aa8f6..3dddc99974 100644 --- a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/MiddlewareFactory.ts +++ b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/MiddlewareFactory.ts @@ -44,8 +44,8 @@ import { } from '@backstage/errors'; import { applyInternalErrorFilter } from './applyInternalErrorFilter'; import { rateLimit } from 'express-rate-limit'; -import { Config } from '@backstage/config'; -import { durationToMilliseconds, HumanDuration } from '@backstage/types'; +import { readDurationFromConfig } from '@backstage/config'; +import { durationToMilliseconds } from '@backstage/types'; import { RateLimitStoreFactory } from './RateLimitStoreFactory'; type LogMeta = { @@ -242,44 +242,50 @@ export class MiddlewareFactory { * @returns An Express request handler */ rateLimit(): RequestHandler { - const conf = this.#config.getOptional( - 'backend.rateLimit', - ); - if (!conf || typeof conf !== 'object') { + const enabled = this.#config.has('backend.rateLimit'); + if (!enabled) { return (_req: Request, _res: Response, next: NextFunction) => { next(); }; } - const rateLimitOptions = conf as Config; - const window = rateLimitOptions.getOptional( - 'window', - ); - let windowMs: number | undefined; - if (window !== undefined) { - if (typeof window === 'number') { - windowMs = window; - } else if (typeof window === 'object' && !Array.isArray(window)) { - windowMs = durationToMilliseconds(window); - } else { - throw new Error( - `Invalid configuration backend.rateLimit.window: ${window}, expected milliseconds number or HumanDuration object`, - ); - } + + const useDefaults = this.#config.getOptional('backend.rateLimit') === true; + const rateLimitOptions = useDefaults + ? undefined + : this.#config.getOptionalConfig('backend.rateLimit'); + + let windowMs: number = 60000; + if (rateLimitOptions && rateLimitOptions.has('window')) { + const windowDuration = readDurationFromConfig(rateLimitOptions, { + key: 'window', + }); + windowMs = durationToMilliseconds(windowDuration); } - const ipAllowList = rateLimitOptions.getOptionalStringArray( + const ipAllowList = rateLimitOptions?.getOptionalStringArray( 'ipAllowList', ) ?? ['127.0.0.1', '0:0:0:0:0:0:0:1', '::1']; return rateLimit({ windowMs, - limit: rateLimitOptions.getOptionalNumber('incomingRequestLimit'), - skipSuccessfulRequests: rateLimitOptions.getOptionalBoolean( + limit: rateLimitOptions?.getOptionalNumber('incomingRequestLimit'), + skipSuccessfulRequests: rateLimitOptions?.getOptionalBoolean( 'skipSuccessfulRequests', ), + message: { + error: { + name: 'Error', + message: `Too many requests, please try again later`, + }, + response: { + statusCode: 429, + }, + }, + statusCode: 429, skipFailedRequests: - rateLimitOptions.getOptionalBoolean('skipFailedRequests'), - passOnStoreError: rateLimitOptions.getOptionalBoolean('passOnStoreError'), + rateLimitOptions?.getOptionalBoolean('skipFailedRequests'), + passOnStoreError: + rateLimitOptions?.getOptionalBoolean('passOnStoreError'), keyGenerator(req, _res): string { if (!req.ip) { return req.socket.remoteAddress!; @@ -298,7 +304,9 @@ export class MiddlewareFactory { validate: { trustProxy: false, }, - store: new RateLimitStoreFactory(this.#config).create(), + store: useDefaults + ? undefined + : RateLimitStoreFactory.create(this.#config), }); } diff --git a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.test.ts b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.test.ts index 4fdc513c81..c1f690474e 100644 --- a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.test.ts +++ b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.test.ts @@ -13,13 +13,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { mockServices, TestCaches } from '@backstage/backend-test-utils'; +import { mockServices } from '@backstage/backend-test-utils'; import { RateLimitStoreFactory } from './RateLimitStoreFactory'; -import { RedisStore } from 'rate-limit-redis'; + +jest.mock('@keyv/redis', () => { + const Actual = jest.requireActual('@keyv/redis'); + return { + ...Actual, + __esModule: true, + default: jest.fn(() => { + return { + getClient: jest.fn(() => ({ + sendCommand: jest.fn().mockReturnValue('mock'), + })), + }; + }), + }; +}); describe('CacheRateLimitStoreFactory', () => { - const caches = TestCaches.create(); - afterEach(jest.clearAllMocks); it('should return undefined store with auto configuration if redis is not available', () => { @@ -32,31 +44,24 @@ describe('CacheRateLimitStoreFactory', () => { }, }, }); - const factory = new RateLimitStoreFactory(config); - const store = factory.create(); + const store = RateLimitStoreFactory.create(config); expect(store).toBeUndefined(); }); it('should return redis store if configured explicitly', async () => { - if (!caches.supports('REDIS_7')) { - return; - } - const conf = await caches.init('REDIS_7'); - const config = mockServices.rootConfig({ data: { backend: { rateLimit: { store: { client: 'redis', - connection: conf.connection, + connection: 'redis://localhost:6379', }, }, }, }, }); - const factory = new RateLimitStoreFactory(config); - const store = factory.create(); - expect(store).toBeInstanceOf(RedisStore); + const store = RateLimitStoreFactory.create(config); + expect(store).not.toBeUndefined(); }); }); diff --git a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.ts b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.ts index 85eb3c4d86..1275095aca 100644 --- a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.ts +++ b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.ts @@ -15,7 +15,6 @@ */ import { Config } from '@backstage/config'; import type { Store } from 'express-rate-limit'; -import KeyvRedis from '@keyv/redis'; import { RedisStore } from 'rate-limit-redis'; /** @@ -24,10 +23,8 @@ import { RedisStore } from 'rate-limit-redis'; * @internal */ export class RateLimitStoreFactory { - constructor(private readonly config: Config) {} - - create(): Store | undefined { - const store = this.config.getOptionalConfig('backend.rateLimit.store'); + static create(config: Config): Store | undefined { + const store = config.getOptionalConfig('backend.rateLimit.store'); if (!store) { return undefined; } @@ -40,12 +37,15 @@ export class RateLimitStoreFactory { } } - redis(storeConfig: Config): Store { + private static redis(storeConfig: Config): Store { const connectionString = storeConfig.getString('connection'); + const KeyvRedis = require('@keyv/redis').default; const keyv = new KeyvRedis(connectionString); return new RedisStore({ - // Keyv uses ioredis under the hood - sendCommand: (...args: string[]) => keyv.redis.call(...args), + sendCommand: async (...args: string[]) => { + const client = await keyv.getClient(); + return client.sendCommand(args); + }, }); } } diff --git a/packages/backend-defaults/src/entrypoints/rootHttpRouter/rootHttpRouterServiceFactory.ts b/packages/backend-defaults/src/entrypoints/rootHttpRouter/rootHttpRouterServiceFactory.ts index 7d0d778b0c..3b50515742 100644 --- a/packages/backend-defaults/src/entrypoints/rootHttpRouter/rootHttpRouterServiceFactory.ts +++ b/packages/backend-defaults/src/entrypoints/rootHttpRouter/rootHttpRouterServiceFactory.ts @@ -117,11 +117,11 @@ const rootHttpRouterServiceFactoryWithOptions = ( if (trustProxy !== undefined) { app.set('trust proxy', trustProxy); } - app.use(middleware.rateLimit()); app.use(middleware.helmet()); app.use(middleware.cors()); app.use(middleware.compression()); app.use(middleware.logging()); + app.use(middleware.rateLimit()); app.use(healthRouter); app.use(routes); app.use(middleware.notFound()); diff --git a/yarn.lock b/yarn.lock index 0a73d77844..aebb875a10 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3632,7 +3632,7 @@ __metadata: cron: "npm:^3.0.0" express: "npm:^4.17.1" express-promise-router: "npm:^4.1.0" - express-rate-limit: "npm:^7.4.0" + express-rate-limit: "npm:^7.5.0" fs-extra: "npm:^11.2.0" git-url-parse: "npm:^15.0.0" helmet: "npm:^6.0.0" @@ -30045,12 +30045,12 @@ __metadata: languageName: node linkType: hard -"express-rate-limit@npm:^7.4.0": - version: 7.4.0 - resolution: "express-rate-limit@npm:7.4.0" +"express-rate-limit@npm:^7.5.0": + version: 7.5.0 + resolution: "express-rate-limit@npm:7.5.0" peerDependencies: - express: 4 || 5 || ^5.0.0-beta.1 - checksum: 10/33178c652bb1472aad2022194b5cd7963bd3e74d3eaf5e49eb1491a968fdce54551cc76b097ac10d3a1646d62cec2e6f2405ccef5ef5b60152a0c4a148749a4d + express: ^4.11 || 5 || ^5.0.0-beta.1 + checksum: 10/eff34c83bf586789933a332a339b66649e2cca95c8e977d193aa8bead577d3182ac9f0e9c26f39389287539b8038890ff023f910b54ebb506a26a2ce135b92ca languageName: node linkType: hard From c05a6982eabffaddc1915faa720bffe65ac1777e Mon Sep 17 00:00:00 2001 From: Hellgren Heikki Date: Tue, 4 Feb 2025 08:06:37 +0200 Subject: [PATCH 05/12] feat: support postgres store for rate limiting Signed-off-by: Hellgren Heikki --- .changeset/famous-terms-rescue.md | 4 +- app-config.yaml | 2 +- packages/backend-defaults/config.d.ts | 16 +++ packages/backend-defaults/package.json | 1 + .../http/RateLimitStoreFactory.test.ts | 21 ++- .../http/RateLimitStoreFactory.ts | 15 ++ yarn.lock | 130 +++++++++++++++++- 7 files changed, 184 insertions(+), 5 deletions(-) diff --git a/.changeset/famous-terms-rescue.md b/.changeset/famous-terms-rescue.md index 9f38986934..68fd1f3591 100644 --- a/.changeset/famous-terms-rescue.md +++ b/.changeset/famous-terms-rescue.md @@ -4,12 +4,12 @@ Added new rate limit middleware to allow rate limiting requests to the backend -If you are using the `configure` callback of the root HTTP router service and do NOT call `applyDefaults()` inside it, please see [the relevant changes](https://github.com/backstage/backstage/pull/26725/files#diff-86ad1b6a694dd250823aee39d410428dd837c9d9a04ca8c33bd1081fbe3f22af) that were made, to see if you want to apply them as well to your custom configuration. +If you are using the `configure` callback of the root HTTP router service and do NOT call `applyDefaults()` inside it, please see [the relevant changes](https://github.com/backstage/backstage/pull/28708/files#diff-86ad1b6a694dd250823aee39d410428dd837c9d9a04ca8c33bd1081fbe3f22af) that were made, to see if you want to apply them as well to your custom configuration. Rate limiting can be turned on by adding the following configuration to `app-config.yaml`: ```yaml backend: rateLimit: - window: 6000ms + window: 6s incomingRequestLimit: 100 ``` diff --git a/app-config.yaml b/app-config.yaml index df80c7add0..89926357c2 100644 --- a/app-config.yaml +++ b/app-config.yaml @@ -38,7 +38,7 @@ backend: # Used for testing rate limiting locally # rateLimit: - # windowMs: 60000 + # windowMs: 1m # incomingRequestLimit: 1 # ipAllowList: [] diff --git a/packages/backend-defaults/config.d.ts b/packages/backend-defaults/config.d.ts index 6cd1aa183a..67183896bb 100644 --- a/packages/backend-defaults/config.d.ts +++ b/packages/backend-defaults/config.d.ts @@ -801,6 +801,22 @@ export interface Config { client: 'redis'; connection: string; } + | { + client: 'postgres'; + connection: + | string + | { + /** + * @visibility secret + */ + password?: string; + /** + * Other connection settings + * @see https://node-postgres.com/apis/client + */ + [key: string]: unknown; + }; + } | { client: 'memory'; }; diff --git a/packages/backend-defaults/package.json b/packages/backend-defaults/package.json index b1f9513f2c..bca883c44e 100644 --- a/packages/backend-defaults/package.json +++ b/packages/backend-defaults/package.json @@ -130,6 +130,7 @@ "test": "backstage-cli package test" }, "dependencies": { + "@acpr/rate-limit-postgresql": "^1.4.1", "@aws-sdk/abort-controller": "^3.347.0", "@aws-sdk/client-codecommit": "^3.350.0", "@aws-sdk/client-s3": "^3.350.0", diff --git a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.test.ts b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.test.ts index c1f690474e..6fe4a2d327 100644 --- a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.test.ts +++ b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.test.ts @@ -15,6 +15,8 @@ */ import { mockServices } from '@backstage/backend-test-utils'; import { RateLimitStoreFactory } from './RateLimitStoreFactory'; +import { RedisStore } from 'rate-limit-redis'; +import { PostgresStore } from '@acpr/rate-limit-postgresql'; jest.mock('@keyv/redis', () => { const Actual = jest.requireActual('@keyv/redis'); @@ -62,6 +64,23 @@ describe('CacheRateLimitStoreFactory', () => { }, }); const store = RateLimitStoreFactory.create(config); - expect(store).not.toBeUndefined(); + expect(store).toBeInstanceOf(RedisStore); + }); + + it('should return postgres store if configured explicitly', async () => { + const config = mockServices.rootConfig({ + data: { + backend: { + rateLimit: { + store: { + client: 'postgres', + connection: 'postgres://localhost:5432', + }, + }, + }, + }, + }); + const store = RateLimitStoreFactory.create(config); + expect(store).toBeInstanceOf(PostgresStore); }); }); diff --git a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.ts b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.ts index 1275095aca..3905655587 100644 --- a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.ts +++ b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.ts @@ -16,6 +16,8 @@ import { Config } from '@backstage/config'; import type { Store } from 'express-rate-limit'; import { RedisStore } from 'rate-limit-redis'; +import { parsePgConnectionString } from '../../database/connectors/postgres.ts'; +import { PostgresStore } from '@acpr/rate-limit-postgresql'; /** * Creates a store for `express-rate-limit` based on the configuration. @@ -32,6 +34,9 @@ export class RateLimitStoreFactory { switch (client) { case 'redis': return this.redis(store); + case 'postgres': + return this.postgres(store); + case 'memory': default: return undefined; } @@ -48,4 +53,14 @@ export class RateLimitStoreFactory { }, }); } + + private static postgres(storeConfig: Config): Store { + const connection = storeConfig.get('connection') as any; + const isConnectionString = + typeof connection === 'string' || connection instanceof String; + const connectionOptions = isConnectionString + ? parsePgConnectionString(connection as string) + : connection; + return new PostgresStore(connectionOptions, 'rl'); + } } diff --git a/yarn.lock b/yarn.lock index aebb875a10..66f819db26 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12,6 +12,20 @@ __metadata: languageName: node linkType: hard +"@acpr/rate-limit-postgresql@npm:^1.4.1": + version: 1.4.1 + resolution: "@acpr/rate-limit-postgresql@npm:1.4.1" + dependencies: + "@types/pg-pool": "npm:2.0.3" + pg: "npm:8.11.3" + pg-pool: "npm:3.6.1" + postgres-migrations: "npm:5.3.0" + peerDependencies: + express-rate-limit: ">=6.0.0" + checksum: 10/9295f86890ea10f0be24a211f100cfe9dde40df20d8328be36a66736e36ee7043dc6fcae785e39bea19de3f43ad344f3e0fa3f9d40bc8d89d38bf6ce457bcc28 + languageName: node + linkType: hard + "@adobe/css-tools@npm:^4.4.0": version: 4.4.0 resolution: "@adobe/css-tools@npm:4.4.0" @@ -3580,6 +3594,7 @@ __metadata: version: 0.0.0-use.local resolution: "@backstage/backend-defaults@workspace:packages/backend-defaults" dependencies: + "@acpr/rate-limit-postgresql": "npm:^1.4.1" "@aws-sdk/abort-controller": "npm:^3.347.0" "@aws-sdk/client-codecommit": "npm:^3.350.0" "@aws-sdk/client-s3": "npm:^3.350.0" @@ -20987,6 +21002,15 @@ __metadata: languageName: node linkType: hard +"@types/pg-pool@npm:2.0.3": + version: 2.0.3 + resolution: "@types/pg-pool@npm:2.0.3" + dependencies: + "@types/pg": "npm:*" + checksum: 10/9ea0bcdbdd09c9de6f774e59465189e552ee094901724278082c41ba6287e7fddffb9ba4b4107c242bba4e8f8a1f0016e6a1eb0c6ca306d43c08b5ddd7f34549 + languageName: node + linkType: hard + "@types/pg-pool@npm:2.0.6": version: 2.0.6 resolution: "@types/pg-pool@npm:2.0.6" @@ -25149,6 +25173,13 @@ __metadata: languageName: node linkType: hard +"buffer-writer@npm:2.0.0": + version: 2.0.0 + resolution: "buffer-writer@npm:2.0.0" + checksum: 10/fdca8e28c55704de7af2f41c8f875293de69ad22005d5041d54aa916d125cead00afa969bc09e4702ae6b66e098409958c06bebfc97fcf8fa4ea5afcae088cd9 + languageName: node + linkType: hard + "buffer-xor@npm:^1.0.3": version: 1.0.3 resolution: "buffer-xor@npm:1.0.3" @@ -39963,6 +39994,13 @@ __metadata: languageName: node linkType: hard +"packet-reader@npm:1.0.0": + version: 1.0.0 + resolution: "packet-reader@npm:1.0.0" + checksum: 10/8504cc8c32672380867e933516a029b1d4dd784c139213c85c9042ffc1162de48ec914f8c71260a9311518694cf5d0be11c67357f4b536129d2ea42aa7257ec0 + languageName: node + linkType: hard + "pacote@npm:^12.0.0, pacote@npm:^12.0.2": version: 12.0.3 resolution: "pacote@npm:12.0.3" @@ -40492,7 +40530,7 @@ __metadata: languageName: node linkType: hard -"pg-connection-string@npm:^2.3.0, pg-connection-string@npm:^2.5.0, pg-connection-string@npm:^2.7.0": +"pg-connection-string@npm:^2.3.0, pg-connection-string@npm:^2.5.0, pg-connection-string@npm:^2.6.2, pg-connection-string@npm:^2.7.0": version: 2.7.0 resolution: "pg-connection-string@npm:2.7.0" checksum: 10/68015a8874b7ca5dad456445e4114af3d2602bac2fdb8069315ecad0ff9660ec93259b9af7186606529ac4f6f72a06831e6f20897a689b16cc7fda7ca0e247fd @@ -40520,6 +40558,24 @@ __metadata: languageName: node linkType: hard +"pg-pool@npm:3.6.1": + version: 3.6.1 + resolution: "pg-pool@npm:3.6.1" + peerDependencies: + pg: ">=8.0" + checksum: 10/5d1b02b959e6c849004d8f3d2222c48d3b3b67b7b1eb5f2e5819ed9412129ea6b0f0376bc74ddf197973c99575d325cbb3f64a8017ab520535c011329b12fffb + languageName: node + linkType: hard + +"pg-pool@npm:^3.6.1, pg-pool@npm:^3.7.0": + version: 3.7.0 + resolution: "pg-pool@npm:3.7.0" + peerDependencies: + pg: ">=8.0" + checksum: 10/a07a4f9e26eec9d7ac3597dc7b3469c62983edff9a321dbb7acbe1bbc7f5e9b2d33438e277d4cf8145071f3d63c7ebdc287a539fd69dfb8cdddb15b33eefe1a2 + languageName: node + linkType: hard + "pg-pool@npm:^3.8.0": version: 3.8.0 resolution: "pg-pool@npm:3.8.0" @@ -40536,6 +40592,13 @@ __metadata: languageName: node linkType: hard +"pg-protocol@npm:^1.6.0, pg-protocol@npm:^1.7.0": + version: 1.7.0 + resolution: "pg-protocol@npm:1.7.0" + checksum: 10/ffffdf74426c9357b57050f1c191e84447c0e8b2a701b3ab302ac7dd0eb27b862d92e5e3b2d38876a1051de83547eb9165d6a58b3a8e90bb050dae97f9993d54 + languageName: node + linkType: hard + "pg-types@npm:^2.1.0, pg-types@npm:^2.2.0": version: 2.2.0 resolution: "pg-types@npm:2.2.0" @@ -40564,6 +40627,30 @@ __metadata: languageName: node linkType: hard +"pg@npm:8.11.3": + version: 8.11.3 + resolution: "pg@npm:8.11.3" + dependencies: + buffer-writer: "npm:2.0.0" + packet-reader: "npm:1.0.0" + pg-cloudflare: "npm:^1.1.1" + pg-connection-string: "npm:^2.6.2" + pg-pool: "npm:^3.6.1" + pg-protocol: "npm:^1.6.0" + pg-types: "npm:^2.1.0" + pgpass: "npm:1.x" + peerDependencies: + pg-native: ">=3.0.1" + dependenciesMeta: + pg-cloudflare: + optional: true + peerDependenciesMeta: + pg-native: + optional: true + checksum: 10/f15f29c8e17723ee1da72abdf400cbed2c04602c58c93687f3f0068e71df2a6fb62b9a3543e13da21b10a0494f4c5b4cfc8d6cd8396617b76c4cbfd6ddab17e7 + languageName: node + linkType: hard + "pg@npm:^8.11.3, pg@npm:^8.9.0": version: 8.14.1 resolution: "pg@npm:8.14.1" @@ -40586,6 +40673,28 @@ __metadata: languageName: node linkType: hard +"pg@npm:^8.6.0": + version: 8.13.1 + resolution: "pg@npm:8.13.1" + dependencies: + pg-cloudflare: "npm:^1.1.1" + pg-connection-string: "npm:^2.7.0" + pg-pool: "npm:^3.7.0" + pg-protocol: "npm:^1.7.0" + pg-types: "npm:^2.1.0" + pgpass: "npm:1.x" + peerDependencies: + pg-native: ">=3.0.1" + dependenciesMeta: + pg-cloudflare: + optional: true + peerDependenciesMeta: + pg-native: + optional: true + checksum: 10/542aa49fcb37657cf5f779b4a31fe6eb336e683445ecca38e267eeb0ca85d873ffe51f04794f9f9e184187e9f74bf7895e932a0fa9507132ac0dfc76c7c73451 + languageName: node + linkType: hard + "pgpass@npm:1.x": version: 1.0.2 resolution: "pgpass@npm:1.0.2" @@ -41353,6 +41462,18 @@ __metadata: languageName: node linkType: hard +"postgres-migrations@npm:5.3.0": + version: 5.3.0 + resolution: "postgres-migrations@npm:5.3.0" + dependencies: + pg: "npm:^8.6.0" + sql-template-strings: "npm:^2.2.2" + bin: + pg-validate-migrations: dist/bin/validate.js + checksum: 10/520d95f01144f88689d5c0a7575743c4f99536935deb1ffff7b3765883a688c4f001d98e8b493ca9b342cd2609593970c3d2198b41fade648f102008e3607226 + languageName: node + linkType: hard + "postgres-range@npm:^1.1.1": version: 1.1.3 resolution: "postgres-range@npm:1.1.3" @@ -45172,6 +45293,13 @@ __metadata: languageName: node linkType: hard +"sql-template-strings@npm:^2.2.2": + version: 2.2.2 + resolution: "sql-template-strings@npm:2.2.2" + checksum: 10/594378a44acbaf3db8a4067137c0c315d0656fcc1b6b8fa76c760d032c1970bf6ede2b31690a3bdc6482d86cbff8b202bb14f6528aa1d9d6bf19d48b03ba2744 + languageName: node + linkType: hard + "sqlstring@npm:^2.3.2": version: 2.3.2 resolution: "sqlstring@npm:2.3.2" From 6633c138b583146db5574feac33f4b1712f43d36 Mon Sep 17 00:00:00 2001 From: Hellgren Heikki Date: Thu, 27 Feb 2025 11:40:11 +0200 Subject: [PATCH 06/12] fix: review findings + docs update Signed-off-by: Hellgren Heikki --- .../core-services/root-http-router.md | 5 + packages/backend-defaults/config.d.ts | 20 +-- packages/backend-defaults/package.json | 1 - .../http/RateLimitStoreFactory.test.ts | 18 --- .../http/RateLimitStoreFactory.ts | 18 +-- yarn.lock | 130 +----------------- 6 files changed, 10 insertions(+), 182 deletions(-) diff --git a/docs/backend-system/core-services/root-http-router.md b/docs/backend-system/core-services/root-http-router.md index 7a268e27fc..bdaf7bba23 100644 --- a/docs/backend-system/core-services/root-http-router.md +++ b/docs/backend-system/core-services/root-http-router.md @@ -121,6 +121,11 @@ backend.add( app.use(middleware.cors()); app.use(middleware.compression()); + // Optional rate limiting middleware + app.use(middleware.rateLimit()); + // If you are using rate limiting behind a proxy, you should set the `trust proxy` setting to true + app.set('trust proxy', true); + app.use(healthRouter); // you can add you your own middleware in here diff --git a/packages/backend-defaults/config.d.ts b/packages/backend-defaults/config.d.ts index 67183896bb..f67680d1e2 100644 --- a/packages/backend-defaults/config.d.ts +++ b/packages/backend-defaults/config.d.ts @@ -798,27 +798,11 @@ export interface Config { | { store?: | { - client: 'redis'; + type: 'redis'; connection: string; } | { - client: 'postgres'; - connection: - | string - | { - /** - * @visibility secret - */ - password?: string; - /** - * Other connection settings - * @see https://node-postgres.com/apis/client - */ - [key: string]: unknown; - }; - } - | { - client: 'memory'; + type: 'memory'; }; /** * Time frame in milliseconds or as human duration for which requests are checked/remembered. diff --git a/packages/backend-defaults/package.json b/packages/backend-defaults/package.json index bca883c44e..b1f9513f2c 100644 --- a/packages/backend-defaults/package.json +++ b/packages/backend-defaults/package.json @@ -130,7 +130,6 @@ "test": "backstage-cli package test" }, "dependencies": { - "@acpr/rate-limit-postgresql": "^1.4.1", "@aws-sdk/abort-controller": "^3.347.0", "@aws-sdk/client-codecommit": "^3.350.0", "@aws-sdk/client-s3": "^3.350.0", diff --git a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.test.ts b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.test.ts index 6fe4a2d327..f1044979e8 100644 --- a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.test.ts +++ b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.test.ts @@ -16,7 +16,6 @@ import { mockServices } from '@backstage/backend-test-utils'; import { RateLimitStoreFactory } from './RateLimitStoreFactory'; import { RedisStore } from 'rate-limit-redis'; -import { PostgresStore } from '@acpr/rate-limit-postgresql'; jest.mock('@keyv/redis', () => { const Actual = jest.requireActual('@keyv/redis'); @@ -66,21 +65,4 @@ describe('CacheRateLimitStoreFactory', () => { const store = RateLimitStoreFactory.create(config); expect(store).toBeInstanceOf(RedisStore); }); - - it('should return postgres store if configured explicitly', async () => { - const config = mockServices.rootConfig({ - data: { - backend: { - rateLimit: { - store: { - client: 'postgres', - connection: 'postgres://localhost:5432', - }, - }, - }, - }, - }); - const store = RateLimitStoreFactory.create(config); - expect(store).toBeInstanceOf(PostgresStore); - }); }); diff --git a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.ts b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.ts index 3905655587..0a24a8792c 100644 --- a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.ts +++ b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.ts @@ -16,8 +16,6 @@ import { Config } from '@backstage/config'; import type { Store } from 'express-rate-limit'; import { RedisStore } from 'rate-limit-redis'; -import { parsePgConnectionString } from '../../database/connectors/postgres.ts'; -import { PostgresStore } from '@acpr/rate-limit-postgresql'; /** * Creates a store for `express-rate-limit` based on the configuration. @@ -30,12 +28,10 @@ export class RateLimitStoreFactory { if (!store) { return undefined; } - const client = store.getString('client'); - switch (client) { + const type = store.getString('type'); + switch (type) { case 'redis': return this.redis(store); - case 'postgres': - return this.postgres(store); case 'memory': default: return undefined; @@ -53,14 +49,4 @@ export class RateLimitStoreFactory { }, }); } - - private static postgres(storeConfig: Config): Store { - const connection = storeConfig.get('connection') as any; - const isConnectionString = - typeof connection === 'string' || connection instanceof String; - const connectionOptions = isConnectionString - ? parsePgConnectionString(connection as string) - : connection; - return new PostgresStore(connectionOptions, 'rl'); - } } diff --git a/yarn.lock b/yarn.lock index 66f819db26..aebb875a10 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12,20 +12,6 @@ __metadata: languageName: node linkType: hard -"@acpr/rate-limit-postgresql@npm:^1.4.1": - version: 1.4.1 - resolution: "@acpr/rate-limit-postgresql@npm:1.4.1" - dependencies: - "@types/pg-pool": "npm:2.0.3" - pg: "npm:8.11.3" - pg-pool: "npm:3.6.1" - postgres-migrations: "npm:5.3.0" - peerDependencies: - express-rate-limit: ">=6.0.0" - checksum: 10/9295f86890ea10f0be24a211f100cfe9dde40df20d8328be36a66736e36ee7043dc6fcae785e39bea19de3f43ad344f3e0fa3f9d40bc8d89d38bf6ce457bcc28 - languageName: node - linkType: hard - "@adobe/css-tools@npm:^4.4.0": version: 4.4.0 resolution: "@adobe/css-tools@npm:4.4.0" @@ -3594,7 +3580,6 @@ __metadata: version: 0.0.0-use.local resolution: "@backstage/backend-defaults@workspace:packages/backend-defaults" dependencies: - "@acpr/rate-limit-postgresql": "npm:^1.4.1" "@aws-sdk/abort-controller": "npm:^3.347.0" "@aws-sdk/client-codecommit": "npm:^3.350.0" "@aws-sdk/client-s3": "npm:^3.350.0" @@ -21002,15 +20987,6 @@ __metadata: languageName: node linkType: hard -"@types/pg-pool@npm:2.0.3": - version: 2.0.3 - resolution: "@types/pg-pool@npm:2.0.3" - dependencies: - "@types/pg": "npm:*" - checksum: 10/9ea0bcdbdd09c9de6f774e59465189e552ee094901724278082c41ba6287e7fddffb9ba4b4107c242bba4e8f8a1f0016e6a1eb0c6ca306d43c08b5ddd7f34549 - languageName: node - linkType: hard - "@types/pg-pool@npm:2.0.6": version: 2.0.6 resolution: "@types/pg-pool@npm:2.0.6" @@ -25173,13 +25149,6 @@ __metadata: languageName: node linkType: hard -"buffer-writer@npm:2.0.0": - version: 2.0.0 - resolution: "buffer-writer@npm:2.0.0" - checksum: 10/fdca8e28c55704de7af2f41c8f875293de69ad22005d5041d54aa916d125cead00afa969bc09e4702ae6b66e098409958c06bebfc97fcf8fa4ea5afcae088cd9 - languageName: node - linkType: hard - "buffer-xor@npm:^1.0.3": version: 1.0.3 resolution: "buffer-xor@npm:1.0.3" @@ -39994,13 +39963,6 @@ __metadata: languageName: node linkType: hard -"packet-reader@npm:1.0.0": - version: 1.0.0 - resolution: "packet-reader@npm:1.0.0" - checksum: 10/8504cc8c32672380867e933516a029b1d4dd784c139213c85c9042ffc1162de48ec914f8c71260a9311518694cf5d0be11c67357f4b536129d2ea42aa7257ec0 - languageName: node - linkType: hard - "pacote@npm:^12.0.0, pacote@npm:^12.0.2": version: 12.0.3 resolution: "pacote@npm:12.0.3" @@ -40530,7 +40492,7 @@ __metadata: languageName: node linkType: hard -"pg-connection-string@npm:^2.3.0, pg-connection-string@npm:^2.5.0, pg-connection-string@npm:^2.6.2, pg-connection-string@npm:^2.7.0": +"pg-connection-string@npm:^2.3.0, pg-connection-string@npm:^2.5.0, pg-connection-string@npm:^2.7.0": version: 2.7.0 resolution: "pg-connection-string@npm:2.7.0" checksum: 10/68015a8874b7ca5dad456445e4114af3d2602bac2fdb8069315ecad0ff9660ec93259b9af7186606529ac4f6f72a06831e6f20897a689b16cc7fda7ca0e247fd @@ -40558,24 +40520,6 @@ __metadata: languageName: node linkType: hard -"pg-pool@npm:3.6.1": - version: 3.6.1 - resolution: "pg-pool@npm:3.6.1" - peerDependencies: - pg: ">=8.0" - checksum: 10/5d1b02b959e6c849004d8f3d2222c48d3b3b67b7b1eb5f2e5819ed9412129ea6b0f0376bc74ddf197973c99575d325cbb3f64a8017ab520535c011329b12fffb - languageName: node - linkType: hard - -"pg-pool@npm:^3.6.1, pg-pool@npm:^3.7.0": - version: 3.7.0 - resolution: "pg-pool@npm:3.7.0" - peerDependencies: - pg: ">=8.0" - checksum: 10/a07a4f9e26eec9d7ac3597dc7b3469c62983edff9a321dbb7acbe1bbc7f5e9b2d33438e277d4cf8145071f3d63c7ebdc287a539fd69dfb8cdddb15b33eefe1a2 - languageName: node - linkType: hard - "pg-pool@npm:^3.8.0": version: 3.8.0 resolution: "pg-pool@npm:3.8.0" @@ -40592,13 +40536,6 @@ __metadata: languageName: node linkType: hard -"pg-protocol@npm:^1.6.0, pg-protocol@npm:^1.7.0": - version: 1.7.0 - resolution: "pg-protocol@npm:1.7.0" - checksum: 10/ffffdf74426c9357b57050f1c191e84447c0e8b2a701b3ab302ac7dd0eb27b862d92e5e3b2d38876a1051de83547eb9165d6a58b3a8e90bb050dae97f9993d54 - languageName: node - linkType: hard - "pg-types@npm:^2.1.0, pg-types@npm:^2.2.0": version: 2.2.0 resolution: "pg-types@npm:2.2.0" @@ -40627,30 +40564,6 @@ __metadata: languageName: node linkType: hard -"pg@npm:8.11.3": - version: 8.11.3 - resolution: "pg@npm:8.11.3" - dependencies: - buffer-writer: "npm:2.0.0" - packet-reader: "npm:1.0.0" - pg-cloudflare: "npm:^1.1.1" - pg-connection-string: "npm:^2.6.2" - pg-pool: "npm:^3.6.1" - pg-protocol: "npm:^1.6.0" - pg-types: "npm:^2.1.0" - pgpass: "npm:1.x" - peerDependencies: - pg-native: ">=3.0.1" - dependenciesMeta: - pg-cloudflare: - optional: true - peerDependenciesMeta: - pg-native: - optional: true - checksum: 10/f15f29c8e17723ee1da72abdf400cbed2c04602c58c93687f3f0068e71df2a6fb62b9a3543e13da21b10a0494f4c5b4cfc8d6cd8396617b76c4cbfd6ddab17e7 - languageName: node - linkType: hard - "pg@npm:^8.11.3, pg@npm:^8.9.0": version: 8.14.1 resolution: "pg@npm:8.14.1" @@ -40673,28 +40586,6 @@ __metadata: languageName: node linkType: hard -"pg@npm:^8.6.0": - version: 8.13.1 - resolution: "pg@npm:8.13.1" - dependencies: - pg-cloudflare: "npm:^1.1.1" - pg-connection-string: "npm:^2.7.0" - pg-pool: "npm:^3.7.0" - pg-protocol: "npm:^1.7.0" - pg-types: "npm:^2.1.0" - pgpass: "npm:1.x" - peerDependencies: - pg-native: ">=3.0.1" - dependenciesMeta: - pg-cloudflare: - optional: true - peerDependenciesMeta: - pg-native: - optional: true - checksum: 10/542aa49fcb37657cf5f779b4a31fe6eb336e683445ecca38e267eeb0ca85d873ffe51f04794f9f9e184187e9f74bf7895e932a0fa9507132ac0dfc76c7c73451 - languageName: node - linkType: hard - "pgpass@npm:1.x": version: 1.0.2 resolution: "pgpass@npm:1.0.2" @@ -41462,18 +41353,6 @@ __metadata: languageName: node linkType: hard -"postgres-migrations@npm:5.3.0": - version: 5.3.0 - resolution: "postgres-migrations@npm:5.3.0" - dependencies: - pg: "npm:^8.6.0" - sql-template-strings: "npm:^2.2.2" - bin: - pg-validate-migrations: dist/bin/validate.js - checksum: 10/520d95f01144f88689d5c0a7575743c4f99536935deb1ffff7b3765883a688c4f001d98e8b493ca9b342cd2609593970c3d2198b41fade648f102008e3607226 - languageName: node - linkType: hard - "postgres-range@npm:^1.1.1": version: 1.1.3 resolution: "postgres-range@npm:1.1.3" @@ -45293,13 +45172,6 @@ __metadata: languageName: node linkType: hard -"sql-template-strings@npm:^2.2.2": - version: 2.2.2 - resolution: "sql-template-strings@npm:2.2.2" - checksum: 10/594378a44acbaf3db8a4067137c0c315d0656fcc1b6b8fa76c760d032c1970bf6ede2b31690a3bdc6482d86cbff8b202bb14f6528aa1d9d6bf19d48b03ba2744 - languageName: node - linkType: hard - "sqlstring@npm:^2.3.2": version: 2.3.2 resolution: "sqlstring@npm:2.3.2" From a68bbc54c02524bf679247802601c376127c6b88 Mon Sep 17 00:00:00 2001 From: Hellgren Heikki Date: Thu, 27 Feb 2025 15:02:43 +0200 Subject: [PATCH 07/12] feat: allow plugin specific rate limiting Signed-off-by: Hellgren Heikki --- packages/backend-defaults/config.d.ts | 40 +++++++++ .../http/createRateLimitMiddleware.ts | 40 +++++++++ .../httpRouter/httpRouterServiceFactory.ts | 7 +- .../rootHttpRouter/http/MiddlewareFactory.ts | 64 +++------------ .../RateLimitStoreFactory.test.ts | 4 +- .../http => lib}/RateLimitStoreFactory.ts | 0 .../src/lib/rateLimitMiddleware.ts | 81 +++++++++++++++++++ 7 files changed, 180 insertions(+), 56 deletions(-) create mode 100644 packages/backend-defaults/src/entrypoints/httpRouter/http/createRateLimitMiddleware.ts rename packages/backend-defaults/src/{entrypoints/rootHttpRouter/http => lib}/RateLimitStoreFactory.test.ts (95%) rename packages/backend-defaults/src/{entrypoints/rootHttpRouter/http => lib}/RateLimitStoreFactory.ts (100%) create mode 100644 packages/backend-defaults/src/lib/rateLimitMiddleware.ts diff --git a/packages/backend-defaults/config.d.ts b/packages/backend-defaults/config.d.ts index f67680d1e2..167a8935f9 100644 --- a/packages/backend-defaults/config.d.ts +++ b/packages/backend-defaults/config.d.ts @@ -804,6 +804,11 @@ export interface Config { | { type: 'memory'; }; + /** + * Enable/disable global rate limiting. If this is disabled, plugin specific rate limiting must be + * used. + */ + global?: boolean; /** * Time frame in milliseconds or as human duration for which requests are checked/remembered. * Defaults to one minute. @@ -834,6 +839,41 @@ export interface Config { * Defaults to false. */ skipFailedRequests?: boolean; + /** Plugin specific rate limiting configuration */ + plugin?: { + [pluginId: string]: { + /** + * Time frame in milliseconds or as human duration for which requests are checked/remembered. + * Defaults to one minute. + */ + window?: string | HumanDuration; + /** + * The maximum number of connections to allow during the `window` before rate limiting the client. + * Defaults to 5. + */ + incomingRequestLimit?: number; + /** + * Whether to pass requests in case of store failure. + * Defaults to false. + */ + passOnStoreError?: boolean; + /** + * List of allowed IP addresses that are not rate limited. + * Defaults to [127.0.0.1, 0:0:0:0:0:0:0:1, ::1]. + */ + ipAllowList?: string[]; + /** + * Skip rate limiting for requests that have been successful. + * Defaults to false. + */ + skipSuccessfulRequests?: boolean; + /** + * Skip rate limiting for requests that have failed. + * Defaults to false. + */ + skipFailedRequests?: boolean; + }; + }; }; /** diff --git a/packages/backend-defaults/src/entrypoints/httpRouter/http/createRateLimitMiddleware.ts b/packages/backend-defaults/src/entrypoints/httpRouter/http/createRateLimitMiddleware.ts new file mode 100644 index 0000000000..0e9239650b --- /dev/null +++ b/packages/backend-defaults/src/entrypoints/httpRouter/http/createRateLimitMiddleware.ts @@ -0,0 +1,40 @@ +/* + * Copyright 2025 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { NextFunction, Request, Response } from 'express'; +import { RateLimitStoreFactory } from '../../../lib/RateLimitStoreFactory.ts'; +import { Config } from '@backstage/config'; +import { rateLimitMiddleware } from '../../../lib/rateLimitMiddleware.ts'; + +export const createRateLimitMiddleware = (options: { + pluginId: string; + config: Config; +}) => { + const { pluginId, config } = options; + const configKey = `backend.rateLimit.${pluginId}`; + const enabled = config.has(configKey); + if (!enabled) { + return (_req: Request, _res: Response, next: NextFunction) => { + next(); + }; + } + + const rateLimitOptions = config.getConfig(configKey); + + return rateLimitMiddleware({ + store: RateLimitStoreFactory.create(config), + config: rateLimitOptions, + }); +}; diff --git a/packages/backend-defaults/src/entrypoints/httpRouter/httpRouterServiceFactory.ts b/packages/backend-defaults/src/entrypoints/httpRouter/httpRouterServiceFactory.ts index 1f2b989018..0fc70a07f0 100644 --- a/packages/backend-defaults/src/entrypoints/httpRouter/httpRouterServiceFactory.ts +++ b/packages/backend-defaults/src/entrypoints/httpRouter/httpRouterServiceFactory.ts @@ -22,12 +22,13 @@ import { HttpRouterServiceAuthPolicy, } from '@backstage/backend-plugin-api'; import { - createLifecycleMiddleware, + createAuthIntegrationRouter, createCookieAuthRefreshMiddleware, createCredentialsBarrier, - createAuthIntegrationRouter, + createLifecycleMiddleware, } from './http'; import { MiddlewareFactory } from '../rootHttpRouter'; +import { createRateLimitMiddleware } from './http/createRateLimitMiddleware.ts'; /** * HTTP route registration for plugins. @@ -61,6 +62,8 @@ export const httpRouterServiceFactory = createServiceFactory({ }) { const router = PromiseRouter(); + router.use(createRateLimitMiddleware({ pluginId: plugin.getId(), config })); + rootHttpRouter.use(`/api/${plugin.getId()}`, router); const credentialsBarrier = createCredentialsBarrier({ diff --git a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/MiddlewareFactory.ts b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/MiddlewareFactory.ts index 3dddc99974..9a76ec5a59 100644 --- a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/MiddlewareFactory.ts +++ b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/MiddlewareFactory.ts @@ -43,10 +43,8 @@ import { ServiceUnavailableError, } from '@backstage/errors'; import { applyInternalErrorFilter } from './applyInternalErrorFilter'; -import { rateLimit } from 'express-rate-limit'; -import { readDurationFromConfig } from '@backstage/config'; -import { durationToMilliseconds } from '@backstage/types'; -import { RateLimitStoreFactory } from './RateLimitStoreFactory'; +import { RateLimitStoreFactory } from '../../../lib/RateLimitStoreFactory.ts'; +import { rateLimitMiddleware } from '../../../lib/rateLimitMiddleware.ts'; type LogMeta = { date: string; @@ -254,59 +252,21 @@ export class MiddlewareFactory { ? undefined : this.#config.getOptionalConfig('backend.rateLimit'); - let windowMs: number = 60000; - if (rateLimitOptions && rateLimitOptions.has('window')) { - const windowDuration = readDurationFromConfig(rateLimitOptions, { - key: 'window', - }); - windowMs = durationToMilliseconds(windowDuration); + // Global rate limiting disabled + if ( + rateLimitOptions && + rateLimitOptions.getOptionalBoolean('global') === false + ) { + return (_req: Request, _res: Response, next: NextFunction) => { + next(); + }; } - const ipAllowList = rateLimitOptions?.getOptionalStringArray( - 'ipAllowList', - ) ?? ['127.0.0.1', '0:0:0:0:0:0:0:1', '::1']; - - return rateLimit({ - windowMs, - limit: rateLimitOptions?.getOptionalNumber('incomingRequestLimit'), - skipSuccessfulRequests: rateLimitOptions?.getOptionalBoolean( - 'skipSuccessfulRequests', - ), - message: { - error: { - name: 'Error', - message: `Too many requests, please try again later`, - }, - response: { - statusCode: 429, - }, - }, - statusCode: 429, - skipFailedRequests: - rateLimitOptions?.getOptionalBoolean('skipFailedRequests'), - passOnStoreError: - rateLimitOptions?.getOptionalBoolean('passOnStoreError'), - keyGenerator(req, _res): string { - if (!req.ip) { - return req.socket.remoteAddress!; - } - return req.ip; - }, - skip: (req, _res) => { - return ( - Boolean(req.ip && ipAllowList.includes(req.ip)) || - Boolean( - req.socket.remoteAddress && - ipAllowList.includes(req.socket.remoteAddress), - ) - ); - }, - validate: { - trustProxy: false, - }, + return rateLimitMiddleware({ store: useDefaults ? undefined : RateLimitStoreFactory.create(this.#config), + config: rateLimitOptions, }); } diff --git a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.test.ts b/packages/backend-defaults/src/lib/RateLimitStoreFactory.test.ts similarity index 95% rename from packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.test.ts rename to packages/backend-defaults/src/lib/RateLimitStoreFactory.test.ts index f1044979e8..02a7f30417 100644 --- a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.test.ts +++ b/packages/backend-defaults/src/lib/RateLimitStoreFactory.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ import { mockServices } from '@backstage/backend-test-utils'; -import { RateLimitStoreFactory } from './RateLimitStoreFactory'; +import { RateLimitStoreFactory } from './RateLimitStoreFactory.ts'; import { RedisStore } from 'rate-limit-redis'; jest.mock('@keyv/redis', () => { @@ -55,7 +55,7 @@ describe('CacheRateLimitStoreFactory', () => { backend: { rateLimit: { store: { - client: 'redis', + type: 'redis', connection: 'redis://localhost:6379', }, }, diff --git a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.ts b/packages/backend-defaults/src/lib/RateLimitStoreFactory.ts similarity index 100% rename from packages/backend-defaults/src/entrypoints/rootHttpRouter/http/RateLimitStoreFactory.ts rename to packages/backend-defaults/src/lib/RateLimitStoreFactory.ts diff --git a/packages/backend-defaults/src/lib/rateLimitMiddleware.ts b/packages/backend-defaults/src/lib/rateLimitMiddleware.ts new file mode 100644 index 0000000000..57163b5908 --- /dev/null +++ b/packages/backend-defaults/src/lib/rateLimitMiddleware.ts @@ -0,0 +1,81 @@ +/* + * Copyright 2025 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { RequestHandler } from 'express'; +import { rateLimit, Store } from 'express-rate-limit'; +import { Config, readDurationFromConfig } from '@backstage/config'; +import { durationToMilliseconds } from '@backstage/types'; + +export const rateLimitMiddleware = (options: { + store?: Store; + config?: Config; +}): RequestHandler => { + const { store, config } = options; + let windowMs: number = 60000; + if (config && config.has('window')) { + const windowDuration = readDurationFromConfig(config, { + key: 'window', + }); + windowMs = durationToMilliseconds(windowDuration); + } + const limit = config?.getOptionalNumber('incomingRequestLimit'); + const ipAllowList = config?.getOptionalStringArray('ipAllowList') ?? [ + '127.0.0.1', + '0:0:0:0:0:0:0:1', + '::1', + ]; + const skipSuccessfulRequests = config?.getOptionalBoolean( + 'skipSuccessfulRequests', + ); + const skipFailedRequests = config?.getOptionalBoolean('skipFailedRequests'); + const passOnStoreError = config?.getOptionalBoolean('passOnStoreError'); + + return rateLimit({ + windowMs, + limit, + skipSuccessfulRequests, + message: { + error: { + name: 'Error', + message: `Too many requests, please try again later`, + }, + response: { + statusCode: 429, + }, + }, + statusCode: 429, + skipFailedRequests, + passOnStoreError: passOnStoreError, + keyGenerator(req, _res): string { + if (!req.ip) { + return req.socket.remoteAddress!; + } + return req.ip; + }, + skip: (req, _res) => { + return ( + Boolean(req.ip && ipAllowList.includes(req.ip)) || + Boolean( + req.socket.remoteAddress && + ipAllowList.includes(req.socket.remoteAddress), + ) + ); + }, + validate: { + trustProxy: false, + }, + store, + }); +}; From eddd61a600044cab080ebae0ef85265f7d27149d Mon Sep 17 00:00:00 2001 From: Hellgren Heikki Date: Thu, 27 Feb 2025 15:13:41 +0200 Subject: [PATCH 08/12] fix: use prefix for plugin rate limit redis store Signed-off-by: Hellgren Heikki --- .../httpRouter/http/createRateLimitMiddleware.ts | 2 +- .../rootHttpRouter/http/MiddlewareFactory.ts | 2 +- .../src/lib/RateLimitStoreFactory.test.ts | 4 ++-- .../src/lib/RateLimitStoreFactory.ts | 14 ++++++++++---- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/packages/backend-defaults/src/entrypoints/httpRouter/http/createRateLimitMiddleware.ts b/packages/backend-defaults/src/entrypoints/httpRouter/http/createRateLimitMiddleware.ts index 0e9239650b..377d7fa301 100644 --- a/packages/backend-defaults/src/entrypoints/httpRouter/http/createRateLimitMiddleware.ts +++ b/packages/backend-defaults/src/entrypoints/httpRouter/http/createRateLimitMiddleware.ts @@ -34,7 +34,7 @@ export const createRateLimitMiddleware = (options: { const rateLimitOptions = config.getConfig(configKey); return rateLimitMiddleware({ - store: RateLimitStoreFactory.create(config), + store: RateLimitStoreFactory.create({ config, prefix: pluginId }), config: rateLimitOptions, }); }; diff --git a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/MiddlewareFactory.ts b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/MiddlewareFactory.ts index 9a76ec5a59..80a10fe0eb 100644 --- a/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/MiddlewareFactory.ts +++ b/packages/backend-defaults/src/entrypoints/rootHttpRouter/http/MiddlewareFactory.ts @@ -265,7 +265,7 @@ export class MiddlewareFactory { return rateLimitMiddleware({ store: useDefaults ? undefined - : RateLimitStoreFactory.create(this.#config), + : RateLimitStoreFactory.create({ config: this.#config }), config: rateLimitOptions, }); } diff --git a/packages/backend-defaults/src/lib/RateLimitStoreFactory.test.ts b/packages/backend-defaults/src/lib/RateLimitStoreFactory.test.ts index 02a7f30417..684c204cb8 100644 --- a/packages/backend-defaults/src/lib/RateLimitStoreFactory.test.ts +++ b/packages/backend-defaults/src/lib/RateLimitStoreFactory.test.ts @@ -45,7 +45,7 @@ describe('CacheRateLimitStoreFactory', () => { }, }, }); - const store = RateLimitStoreFactory.create(config); + const store = RateLimitStoreFactory.create({ config }); expect(store).toBeUndefined(); }); @@ -62,7 +62,7 @@ describe('CacheRateLimitStoreFactory', () => { }, }, }); - const store = RateLimitStoreFactory.create(config); + const store = RateLimitStoreFactory.create({ config }); expect(store).toBeInstanceOf(RedisStore); }); }); diff --git a/packages/backend-defaults/src/lib/RateLimitStoreFactory.ts b/packages/backend-defaults/src/lib/RateLimitStoreFactory.ts index 0a24a8792c..9df6f03225 100644 --- a/packages/backend-defaults/src/lib/RateLimitStoreFactory.ts +++ b/packages/backend-defaults/src/lib/RateLimitStoreFactory.ts @@ -23,7 +23,11 @@ import { RedisStore } from 'rate-limit-redis'; * @internal */ export class RateLimitStoreFactory { - static create(config: Config): Store | undefined { + static create(options: { + config: Config; + prefix?: string; + }): Store | undefined { + const { config, prefix } = options; const store = config.getOptionalConfig('backend.rateLimit.store'); if (!store) { return undefined; @@ -31,18 +35,20 @@ export class RateLimitStoreFactory { const type = store.getString('type'); switch (type) { case 'redis': - return this.redis(store); + return this.redis({ store, prefix }); case 'memory': default: return undefined; } } - private static redis(storeConfig: Config): Store { - const connectionString = storeConfig.getString('connection'); + private static redis(options: { store: Config; prefix?: string }): Store { + const { store, prefix } = options; + const connectionString = store.getString('connection'); const KeyvRedis = require('@keyv/redis').default; const keyv = new KeyvRedis(connectionString); return new RedisStore({ + prefix, sendCommand: async (...args: string[]) => { const client = await keyv.getClient(); return client.sendCommand(args); From 25ffb8684cf1c2d9dbd6abeda169cfacd308d6a0 Mon Sep 17 00:00:00 2001 From: Hellgren Heikki Date: Thu, 27 Feb 2025 15:15:13 +0200 Subject: [PATCH 09/12] chore: add mention about plugin rate limit to changeset Signed-off-by: Hellgren Heikki --- .changeset/famous-terms-rescue.md | 12 +++++ .changeset/sour-comics-attend.md | 5 --- .../core-services/http-router.md | 44 +++++++++++++++++++ .../http/createRateLimitMiddleware.ts | 2 +- 4 files changed, 57 insertions(+), 6 deletions(-) delete mode 100644 .changeset/sour-comics-attend.md diff --git a/.changeset/famous-terms-rescue.md b/.changeset/famous-terms-rescue.md index 68fd1f3591..d323bf0867 100644 --- a/.changeset/famous-terms-rescue.md +++ b/.changeset/famous-terms-rescue.md @@ -13,3 +13,15 @@ backend: window: 6s incomingRequestLimit: 100 ``` + +Plugin specific rate limiting can be configured by adding the following configuration to `app-config.yaml`: + +```yaml +backend: + rateLimit: + global: false # This will disable the global rate limiting + plugin: + catalog: + window: 6s + incomingRequestLimit: 100 +``` diff --git a/.changeset/sour-comics-attend.md b/.changeset/sour-comics-attend.md deleted file mode 100644 index 729fc59ba7..0000000000 --- a/.changeset/sour-comics-attend.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/backend-defaults': patch ---- - -Add configuration variable for `express` trust proxy setting diff --git a/docs/backend-system/core-services/http-router.md b/docs/backend-system/core-services/http-router.md index d7658d9a67..87462eac5a 100644 --- a/docs/backend-system/core-services/http-router.md +++ b/docs/backend-system/core-services/http-router.md @@ -68,6 +68,50 @@ For those routes you will also have to specify `allowLimitedAccess: true` when using the [`auth`](./auth.md) and [`httpAuth`](./http-auth.md) services to access the incoming credentials. +## Rate limiting + +Rate limiting allows you to limit the amount of requests users can send to you backend. +This is useful for blocking various network attacks, such as DDOS, if your instance does not +have additional firewall configured to handle this. + +To configure the default rate limiting, add the following to your config: + +```yaml +backend: + rateLimit: true +``` + +You can additionally configure the rate limiting parameters also by plugin: + +```yaml +backend: + rateLimit: + global: true # Enables or disables rate limit for all plugins + window: 6s # Time window for rate limiting for single client + incomingRequestLimit: 100 # Number of requests to accept from one client during time window + ipAllowList: ['127.0.0.1'] # IPs to bypass rate limiting + skipSuccesfulRequests: false # Rate limit successful requests + skipFailedRequests: false # Rate limit failed requests + plugin: + # Plugin specific rate limiting + catalog: + window: 3s + incomingRequestLimit: 50 +``` + +By default, the rate limiting is per instance and the request counts are stored into memory. +If you want to share this information across all your backstage instances, you have to configure +the rate limiting store: + +```yaml +backend: + rateLimit: + global: true + store: + type: redis + connection: redis://127.0.0.1:16379 +``` + ## Configuring the service For more advanced customization, there are several APIs from the `@backstage/backend-defaults/httpRouter` package that allow you to customize the implementation of the config service. The default implementation uses all of the middleware exported from `@backstage/backend-defaults/httpRouter`, including `createLifecycleMiddleware`, `createAuthIntegrationRouter`, `createCredentialsBarrier` and `createCookieAuthRefreshMiddleware`. You can use these to create your own `httpRouter` service implementation, for example - here's how you would add a custom health check route to all plugins: diff --git a/packages/backend-defaults/src/entrypoints/httpRouter/http/createRateLimitMiddleware.ts b/packages/backend-defaults/src/entrypoints/httpRouter/http/createRateLimitMiddleware.ts index 377d7fa301..b139b212d2 100644 --- a/packages/backend-defaults/src/entrypoints/httpRouter/http/createRateLimitMiddleware.ts +++ b/packages/backend-defaults/src/entrypoints/httpRouter/http/createRateLimitMiddleware.ts @@ -23,7 +23,7 @@ export const createRateLimitMiddleware = (options: { config: Config; }) => { const { pluginId, config } = options; - const configKey = `backend.rateLimit.${pluginId}`; + const configKey = `backend.rateLimit.plugin.${pluginId}`; const enabled = config.has(configKey); if (!enabled) { return (_req: Request, _res: Response, next: NextFunction) => { From b994af48e0dd38dc101264d910137f3074043de4 Mon Sep 17 00:00:00 2001 From: Hellgren Heikki Date: Wed, 14 May 2025 21:39:34 +0300 Subject: [PATCH 10/12] chore: yarn.lock fix Signed-off-by: Hellgren Heikki --- yarn.lock | 214 +++--------------------------------------------------- 1 file changed, 9 insertions(+), 205 deletions(-) diff --git a/yarn.lock b/yarn.lock index aebb875a10..6cf0fb2716 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3335,16 +3335,7 @@ __metadata: languageName: node linkType: hard -"@babel/runtime@npm:^7.0.0, @babel/runtime@npm:^7.1.2, @babel/runtime@npm:^7.10.1, @babel/runtime@npm:^7.12.1, @babel/runtime@npm:^7.12.5, @babel/runtime@npm:^7.13.10, @babel/runtime@npm:^7.17.8, @babel/runtime@npm:^7.18.3, @babel/runtime@npm:^7.18.6, @babel/runtime@npm:^7.20.13, @babel/runtime@npm:^7.20.6, @babel/runtime@npm:^7.21.0, @babel/runtime@npm:^7.23.9, @babel/runtime@npm:^7.3.1, @babel/runtime@npm:^7.4.4, @babel/runtime@npm:^7.5.5, @babel/runtime@npm:^7.6.0, @babel/runtime@npm:^7.7.6, @babel/runtime@npm:^7.8.3, @babel/runtime@npm:^7.8.4, @babel/runtime@npm:^7.8.7, @babel/runtime@npm:^7.9.2": - version: 7.26.7 - resolution: "@babel/runtime@npm:7.26.7" - dependencies: - regenerator-runtime: "npm:^0.14.0" - checksum: 10/c7a661a6836b332d9d2e047cba77ba1862c1e4f78cec7146db45808182ef7636d8a7170be9797e5d8fd513180bffb9fa16f6ca1c69341891efec56113cf22bfc - languageName: node - linkType: hard - -"@babel/runtime@npm:^7.26.10": +"@babel/runtime@npm:^7.0.0, @babel/runtime@npm:^7.1.2, @babel/runtime@npm:^7.10.1, @babel/runtime@npm:^7.12.1, @babel/runtime@npm:^7.12.5, @babel/runtime@npm:^7.13.10, @babel/runtime@npm:^7.17.8, @babel/runtime@npm:^7.18.3, @babel/runtime@npm:^7.18.6, @babel/runtime@npm:^7.20.13, @babel/runtime@npm:^7.20.6, @babel/runtime@npm:^7.21.0, @babel/runtime@npm:^7.23.9, @babel/runtime@npm:^7.26.10, @babel/runtime@npm:^7.3.1, @babel/runtime@npm:^7.4.4, @babel/runtime@npm:^7.5.5, @babel/runtime@npm:^7.6.0, @babel/runtime@npm:^7.7.6, @babel/runtime@npm:^7.8.3, @babel/runtime@npm:^7.8.4, @babel/runtime@npm:^7.8.7, @babel/runtime@npm:^7.9.2": version: 7.27.0 resolution: "@babel/runtime@npm:7.27.0" dependencies: @@ -3353,18 +3344,7 @@ __metadata: languageName: node linkType: hard -"@babel/template@npm:^7.22.5, @babel/template@npm:^7.24.7, @babel/template@npm:^7.25.9, @babel/template@npm:^7.3.3": - version: 7.25.9 - resolution: "@babel/template@npm:7.25.9" - dependencies: - "@babel/code-frame": "npm:^7.25.9" - "@babel/parser": "npm:^7.25.9" - "@babel/types": "npm:^7.25.9" - checksum: 10/e861180881507210150c1335ad94aff80fd9e9be6202e1efa752059c93224e2d5310186ddcdd4c0f0b0fc658ce48cb47823f15142b5c00c8456dde54f5de80b2 - languageName: node - linkType: hard - -"@babel/template@npm:^7.27.0": +"@babel/template@npm:^7.22.5, @babel/template@npm:^7.24.7, @babel/template@npm:^7.25.9, @babel/template@npm:^7.27.0, @babel/template@npm:^7.3.3": version: 7.27.0 resolution: "@babel/template@npm:7.27.0" dependencies: @@ -3390,17 +3370,7 @@ __metadata: languageName: node linkType: hard -"@babel/types@npm:^7.0.0, @babel/types@npm:^7.18.9, @babel/types@npm:^7.20.0, @babel/types@npm:^7.20.7, @babel/types@npm:^7.22.10, @babel/types@npm:^7.22.5, @babel/types@npm:^7.24.7, @babel/types@npm:^7.24.8, @babel/types@npm:^7.25.9, @babel/types@npm:^7.26.0, @babel/types@npm:^7.3.3, @babel/types@npm:^7.4.4": - version: 7.26.0 - resolution: "@babel/types@npm:7.26.0" - dependencies: - "@babel/helper-string-parser": "npm:^7.25.9" - "@babel/helper-validator-identifier": "npm:^7.25.9" - checksum: 10/40780741ecec886ed9edae234b5eb4976968cc70d72b4e5a40d55f83ff2cc457de20f9b0f4fe9d858350e43dab0ea496e7ef62e2b2f08df699481a76df02cd6e - languageName: node - linkType: hard - -"@babel/types@npm:^7.27.0": +"@babel/types@npm:^7.0.0, @babel/types@npm:^7.18.9, @babel/types@npm:^7.20.0, @babel/types@npm:^7.20.7, @babel/types@npm:^7.22.10, @babel/types@npm:^7.22.5, @babel/types@npm:^7.24.7, @babel/types@npm:^7.24.8, @babel/types@npm:^7.25.9, @babel/types@npm:^7.26.0, @babel/types@npm:^7.27.0, @babel/types@npm:^7.3.3, @babel/types@npm:^7.4.4": version: 7.27.0 resolution: "@babel/types@npm:7.27.0" dependencies: @@ -19225,13 +19195,6 @@ __metadata: languageName: node linkType: hard -"@swc/core-darwin-arm64@npm:1.10.6": - version: 1.10.6 - resolution: "@swc/core-darwin-arm64@npm:1.10.6" - conditions: os=darwin & cpu=arm64 - languageName: node - linkType: hard - "@swc/core-darwin-arm64@npm:1.11.24": version: 1.11.24 resolution: "@swc/core-darwin-arm64@npm:1.11.24" @@ -19239,13 +19202,6 @@ __metadata: languageName: node linkType: hard -"@swc/core-darwin-x64@npm:1.10.6": - version: 1.10.6 - resolution: "@swc/core-darwin-x64@npm:1.10.6" - conditions: os=darwin & cpu=x64 - languageName: node - linkType: hard - "@swc/core-darwin-x64@npm:1.11.24": version: 1.11.24 resolution: "@swc/core-darwin-x64@npm:1.11.24" @@ -19253,13 +19209,6 @@ __metadata: languageName: node linkType: hard -"@swc/core-linux-arm-gnueabihf@npm:1.10.6": - version: 1.10.6 - resolution: "@swc/core-linux-arm-gnueabihf@npm:1.10.6" - conditions: os=linux & cpu=arm - languageName: node - linkType: hard - "@swc/core-linux-arm-gnueabihf@npm:1.11.24": version: 1.11.24 resolution: "@swc/core-linux-arm-gnueabihf@npm:1.11.24" @@ -19267,13 +19216,6 @@ __metadata: languageName: node linkType: hard -"@swc/core-linux-arm64-gnu@npm:1.10.6": - version: 1.10.6 - resolution: "@swc/core-linux-arm64-gnu@npm:1.10.6" - conditions: os=linux & cpu=arm64 & libc=glibc - languageName: node - linkType: hard - "@swc/core-linux-arm64-gnu@npm:1.11.24": version: 1.11.24 resolution: "@swc/core-linux-arm64-gnu@npm:1.11.24" @@ -19281,13 +19223,6 @@ __metadata: languageName: node linkType: hard -"@swc/core-linux-arm64-musl@npm:1.10.6": - version: 1.10.6 - resolution: "@swc/core-linux-arm64-musl@npm:1.10.6" - conditions: os=linux & cpu=arm64 & libc=musl - languageName: node - linkType: hard - "@swc/core-linux-arm64-musl@npm:1.11.24": version: 1.11.24 resolution: "@swc/core-linux-arm64-musl@npm:1.11.24" @@ -19295,13 +19230,6 @@ __metadata: languageName: node linkType: hard -"@swc/core-linux-x64-gnu@npm:1.10.6": - version: 1.10.6 - resolution: "@swc/core-linux-x64-gnu@npm:1.10.6" - conditions: os=linux & cpu=x64 & libc=glibc - languageName: node - linkType: hard - "@swc/core-linux-x64-gnu@npm:1.11.24": version: 1.11.24 resolution: "@swc/core-linux-x64-gnu@npm:1.11.24" @@ -19309,13 +19237,6 @@ __metadata: languageName: node linkType: hard -"@swc/core-linux-x64-musl@npm:1.10.6": - version: 1.10.6 - resolution: "@swc/core-linux-x64-musl@npm:1.10.6" - conditions: os=linux & cpu=x64 & libc=musl - languageName: node - linkType: hard - "@swc/core-linux-x64-musl@npm:1.11.24": version: 1.11.24 resolution: "@swc/core-linux-x64-musl@npm:1.11.24" @@ -19323,13 +19244,6 @@ __metadata: languageName: node linkType: hard -"@swc/core-win32-arm64-msvc@npm:1.10.6": - version: 1.10.6 - resolution: "@swc/core-win32-arm64-msvc@npm:1.10.6" - conditions: os=win32 & cpu=arm64 - languageName: node - linkType: hard - "@swc/core-win32-arm64-msvc@npm:1.11.24": version: 1.11.24 resolution: "@swc/core-win32-arm64-msvc@npm:1.11.24" @@ -19337,13 +19251,6 @@ __metadata: languageName: node linkType: hard -"@swc/core-win32-ia32-msvc@npm:1.10.6": - version: 1.10.6 - resolution: "@swc/core-win32-ia32-msvc@npm:1.10.6" - conditions: os=win32 & cpu=ia32 - languageName: node - linkType: hard - "@swc/core-win32-ia32-msvc@npm:1.11.24": version: 1.11.24 resolution: "@swc/core-win32-ia32-msvc@npm:1.11.24" @@ -19351,13 +19258,6 @@ __metadata: languageName: node linkType: hard -"@swc/core-win32-x64-msvc@npm:1.10.6": - version: 1.10.6 - resolution: "@swc/core-win32-x64-msvc@npm:1.10.6" - conditions: os=win32 & cpu=x64 - languageName: node - linkType: hard - "@swc/core-win32-x64-msvc@npm:1.11.24": version: 1.11.24 resolution: "@swc/core-win32-x64-msvc@npm:1.11.24" @@ -19365,7 +19265,7 @@ __metadata: languageName: node linkType: hard -"@swc/core@npm:^1.10.8": +"@swc/core@npm:^1.10.8, @swc/core@npm:^1.3.46": version: 1.11.24 resolution: "@swc/core@npm:1.11.24" dependencies: @@ -19411,52 +19311,6 @@ __metadata: languageName: node linkType: hard -"@swc/core@npm:^1.3.46": - version: 1.10.6 - resolution: "@swc/core@npm:1.10.6" - dependencies: - "@swc/core-darwin-arm64": "npm:1.10.6" - "@swc/core-darwin-x64": "npm:1.10.6" - "@swc/core-linux-arm-gnueabihf": "npm:1.10.6" - "@swc/core-linux-arm64-gnu": "npm:1.10.6" - "@swc/core-linux-arm64-musl": "npm:1.10.6" - "@swc/core-linux-x64-gnu": "npm:1.10.6" - "@swc/core-linux-x64-musl": "npm:1.10.6" - "@swc/core-win32-arm64-msvc": "npm:1.10.6" - "@swc/core-win32-ia32-msvc": "npm:1.10.6" - "@swc/core-win32-x64-msvc": "npm:1.10.6" - "@swc/counter": "npm:^0.1.3" - "@swc/types": "npm:^0.1.17" - peerDependencies: - "@swc/helpers": "*" - dependenciesMeta: - "@swc/core-darwin-arm64": - optional: true - "@swc/core-darwin-x64": - optional: true - "@swc/core-linux-arm-gnueabihf": - optional: true - "@swc/core-linux-arm64-gnu": - optional: true - "@swc/core-linux-arm64-musl": - optional: true - "@swc/core-linux-x64-gnu": - optional: true - "@swc/core-linux-x64-musl": - optional: true - "@swc/core-win32-arm64-msvc": - optional: true - "@swc/core-win32-ia32-msvc": - optional: true - "@swc/core-win32-x64-msvc": - optional: true - peerDependenciesMeta: - "@swc/helpers": - optional: true - checksum: 10/51eccbba6ee8a41f57a6ba4213ec05859434f52fd6698f508c92a8bb467afda56a1e95b07985faf059b27cb28e77d479340de7210a8616976ea82f774a6d86a3 - languageName: node - linkType: hard - "@swc/counter@npm:^0.1.3": version: 0.1.3 resolution: "@swc/counter@npm:0.1.3" @@ -19486,15 +19340,6 @@ __metadata: languageName: node linkType: hard -"@swc/types@npm:^0.1.17": - version: 0.1.17 - resolution: "@swc/types@npm:0.1.17" - dependencies: - "@swc/counter": "npm:^0.1.3" - checksum: 10/ddef1ad5bfead3acdfc41f14e79ba43a99200eb325afbad5716058dbe36358b0513400e9f22aff32432be84a98ae93df95a20b94192f69b8687144270e4eaa18 - languageName: node - linkType: hard - "@swc/types@npm:^0.1.21": version: 0.1.21 resolution: "@swc/types@npm:0.1.21" @@ -20816,16 +20661,7 @@ __metadata: languageName: node linkType: hard -"@types/node@npm:*, @types/node@npm:>=13.7.0, @types/node@npm:^22.0.0": - version: 22.10.5 - resolution: "@types/node@npm:22.10.5" - dependencies: - undici-types: "npm:~6.20.0" - checksum: 10/a5366961ffa9921e8f15435bc18ea9f8b7a7bb6b3d92dd5e93ebcd25e8af65708872bd8e6fee274b4655bab9ca80fbff9f0e42b5b53857790f13cf68cf4cbbfc - languageName: node - linkType: hard - -"@types/node@npm:>=12, @types/node@npm:>=12.0.0, @types/node@npm:>=18.0.0": +"@types/node@npm:*, @types/node@npm:>=12, @types/node@npm:>=12.0.0, @types/node@npm:>=13.7.0, @types/node@npm:>=18.0.0, @types/node@npm:^22.0.0": version: 22.13.10 resolution: "@types/node@npm:22.13.10" dependencies: @@ -24177,20 +24013,13 @@ __metadata: languageName: node linkType: hard -"async@npm:^3.2.2, async@npm:^3.2.6": +"async@npm:^3.2.2, async@npm:^3.2.3, async@npm:^3.2.4, async@npm:^3.2.6": version: 3.2.6 resolution: "async@npm:3.2.6" checksum: 10/cb6e0561a3c01c4b56a799cc8bab6ea5fef45f069ab32500b6e19508db270ef2dffa55e5aed5865c5526e9907b1f8be61b27530823b411ffafb5e1538c86c368 languageName: node linkType: hard -"async@npm:^3.2.3, async@npm:^3.2.4": - version: 3.2.4 - resolution: "async@npm:3.2.4" - checksum: 10/bebb5dc2258c45b83fa1d3be179ae0eb468e1646a62d443c8d60a45e84041b28fccebe1e2d1f234bfc3dcad44e73dcdbf4ba63d98327c9f6556e3dbd47c2ae8b - languageName: node - linkType: hard - "asynckit@npm:^0.4.0": version: 0.4.0 resolution: "asynckit@npm:0.4.0" @@ -24682,20 +24511,13 @@ __metadata: languageName: node linkType: hard -"before-after-hook@npm:^2.1.0": +"before-after-hook@npm:^2.1.0, before-after-hook@npm:^2.2.0": version: 2.2.3 resolution: "before-after-hook@npm:2.2.3" checksum: 10/e676f769dbc4abcf4b3317db2fd2badb4a92c0710e0a7da12cf14b59c3482d4febf835ad7de7874499060fd4e13adf0191628e504728b3c5bb4ec7a878c09940 languageName: node linkType: hard -"before-after-hook@npm:^2.2.0": - version: 2.2.2 - resolution: "before-after-hook@npm:2.2.2" - checksum: 10/34c190def503f771f8811db0bd0c62b35301fe6059c8d847664633ce0548e8253e2661104ba66c71a85548746ba87d5ff2ebf5278c1f3ad367d111ffc9a26bb4 - languageName: node - linkType: hard - "better-opn@npm:^3.0.2": version: 3.0.2 resolution: "better-opn@npm:3.0.2" @@ -31229,25 +31051,7 @@ __metadata: languageName: node linkType: hard -"get-intrinsic@npm:^1.1.3, get-intrinsic@npm:^1.2.4, get-intrinsic@npm:^1.2.5, get-intrinsic@npm:^1.2.6": - version: 1.2.6 - resolution: "get-intrinsic@npm:1.2.6" - dependencies: - call-bind-apply-helpers: "npm:^1.0.1" - dunder-proto: "npm:^1.0.0" - es-define-property: "npm:^1.0.1" - es-errors: "npm:^1.3.0" - es-object-atoms: "npm:^1.0.0" - function-bind: "npm:^1.1.2" - gopd: "npm:^1.2.0" - has-symbols: "npm:^1.1.0" - hasown: "npm:^2.0.2" - math-intrinsics: "npm:^1.0.0" - checksum: 10/a1ffae6d7893a6fa0f4d1472adbc85095edd6b3b0943ead97c3738539cecb19d422ff4d48009eed8c3c27ad678c2b1e38a83b1a1e96b691d13ed8ecefca1068d - languageName: node - linkType: hard - -"get-intrinsic@npm:^1.2.1, get-intrinsic@npm:^1.3.0": +"get-intrinsic@npm:^1.1.3, get-intrinsic@npm:^1.2.1, get-intrinsic@npm:^1.2.4, get-intrinsic@npm:^1.2.5, get-intrinsic@npm:^1.2.6, get-intrinsic@npm:^1.3.0": version: 1.3.0 resolution: "get-intrinsic@npm:1.3.0" dependencies: @@ -36941,7 +36745,7 @@ __metadata: languageName: node linkType: hard -"math-intrinsics@npm:^1.0.0, math-intrinsics@npm:^1.1.0": +"math-intrinsics@npm:^1.1.0": version: 1.1.0 resolution: "math-intrinsics@npm:1.1.0" checksum: 10/11df2eda46d092a6035479632e1ec865b8134bdfc4bd9e571a656f4191525404f13a283a515938c3a8de934dbfd9c09674d9da9fa831e6eb7e22b50b197d2edd From 4990bc2063ff0eacd9391e12425e7b44cc0d21e7 Mon Sep 17 00:00:00 2001 From: Hellgren Heikki Date: Wed, 14 May 2025 22:05:47 +0300 Subject: [PATCH 11/12] docs: improve documentation a bit Signed-off-by: Hellgren Heikki --- .../core-services/http-router.md | 19 ++++++++++++++++++- .../core-services/root-http-router.md | 4 ++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/docs/backend-system/core-services/http-router.md b/docs/backend-system/core-services/http-router.md index 87462eac5a..f53f7d9cd8 100644 --- a/docs/backend-system/core-services/http-router.md +++ b/docs/backend-system/core-services/http-router.md @@ -81,7 +81,7 @@ backend: rateLimit: true ``` -You can additionally configure the rate limiting parameters also by plugin: +You can additionally configure the rate limiting parameters, also by plugin: ```yaml backend: @@ -112,6 +112,17 @@ backend: connection: redis://127.0.0.1:16379 ``` +If your instance is working behind a proxy, you have to configure the backend to trust the proxy +for the rate limiting being able to distinguish clients. + +```yaml +backend: + trustProxy: true +``` + +For more information about the trust proxy configuration and available options, +please refer to [express documentation](https://expressjs.com/en/guide/behind-proxies.html). + ## Configuring the service For more advanced customization, there are several APIs from the `@backstage/backend-defaults/httpRouter` package that allow you to customize the implementation of the config service. The default implementation uses all of the middleware exported from `@backstage/backend-defaults/httpRouter`, including `createLifecycleMiddleware`, `createAuthIntegrationRouter`, `createCredentialsBarrier` and `createCookieAuthRefreshMiddleware`. You can use these to create your own `httpRouter` service implementation, for example - here's how you would add a custom health check route to all plugins: @@ -122,6 +133,7 @@ import { createCookieAuthRefreshMiddleware, createCredentialsBarrier, createAuthIntegrationRouter, + createRateLimitMiddleware, } from '@backstage/backend-defaults/httpRouter'; import { createServiceFactory } from '@backstage/backend-plugin-api'; @@ -149,6 +161,11 @@ backend.add( }) { const router = PromiseRouter(); + // Optional rate limiting middleware + router.use( + createRateLimitMiddleware({ pluginId: plugin.getId(), config }), + ); + rootHttpRouter.use(`/api/${plugin.getId()}`, router); const credentialsBarrier = createCredentialsBarrier({ diff --git a/docs/backend-system/core-services/root-http-router.md b/docs/backend-system/core-services/root-http-router.md index bdaf7bba23..8a6f86fab8 100644 --- a/docs/backend-system/core-services/root-http-router.md +++ b/docs/backend-system/core-services/root-http-router.md @@ -40,6 +40,10 @@ createBackendPlugin({ }); ``` +## Rate limiting + +Please refer to the [HTTP Router documentation](./http-router.md#rate-limiting). + ## Configuring the service ### Via `app-config.yaml` From 1011968ae676ed891584e01f79554dc94c8e1002 Mon Sep 17 00:00:00 2001 From: Hellgren Heikki Date: Thu, 12 Jun 2025 08:39:02 +0300 Subject: [PATCH 12/12] docs: fix doc proposals Signed-off-by: Hellgren Heikki --- docs/backend-system/core-services/http-router.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/backend-system/core-services/http-router.md b/docs/backend-system/core-services/http-router.md index f53f7d9cd8..5e065152a0 100644 --- a/docs/backend-system/core-services/http-router.md +++ b/docs/backend-system/core-services/http-router.md @@ -71,10 +71,9 @@ access the incoming credentials. ## Rate limiting Rate limiting allows you to limit the amount of requests users can send to you backend. -This is useful for blocking various network attacks, such as DDOS, if your instance does not -have additional firewall configured to handle this. +This is useful for blocking various network attacks, such as DDOS. -To configure the default rate limiting, add the following to your config: +To enable rate limiting, add the following to your config: ```yaml backend: