paulononato/backstage
1
0
Fork 0
Code Issues Pull Requests Actions 21 Packages Projects Releases Wiki Activity

Merge pull request #28393 from backstage/yarn-plugin-snyk-lockfile-fixes

Browse Source 
yarn-plugin: include entries for both `backstage:` and `npm:` ranges in lockfile
This commit is contained in:
Vincenzo Scamporlino
2025-04-22 14:17:27 +02:00
committed by GitHub
parent 818fbbb96e 9f6d8c557f
commit 6c2c6cdbe8
14 changed files with 1449 additions and 378 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.changeset/wise-pillows-smile.md
+5
View File
@@ -0,0 +1,5 @@
---
'yarn-plugin-backstage': patch
---
Add both `npm:` and `backstage:` ranges to the lockfile to ensure compatibility with tools that parse the lockfile and ensure dependencies stay locked when building dist workspaces.
.yarn/patches/@yarnpkg-plugin-npm-npm-3.1.0-6533d0f5a1.patch
+11
View File
@@ -0,0 +1,11 @@
diff --git a/lib/NpmHttpFetcher.d.ts b/lib/NpmHttpFetcher.d.ts
index 015aeb8113776d0bc8d2d11154f02f0f2fd7d889..1398792d977fac7f10c7325a7a56d43914a26d17 100644
--- a/lib/NpmHttpFetcher.d.ts
+++ b/lib/NpmHttpFetcher.d.ts
@@ -9,5 +9,5 @@ export declare class NpmHttpFetcher implements Fetcher {
prefixPath: import("@yarnpkg/fslib").PortablePath;
checksum: string | null;
}>;
- fetchFromNetwork(locator: Locator, opts: FetchOptions): Promise<import("/home/runner/work/berry/berry/.yarn/__virtual__/@yarnpkg-libzip-virtual-4957b34c08/1/packages/yarnpkg-libzip").ZipFS>;
+ private fetchFromNetwork;
}
package.json
+2 -1
View File
@@ -106,7 +106,8 @@
"csstype@npm:^3.0.2": "3.0.9",
"csstype@npm:^3.1.2": "3.0.9",
"csstype@npm:^3.1.3": "3.0.9",
"jest-haste-map@^29.7.0": "patch:jest-haste-map@npm%3A29.7.0#./.yarn/patches/jest-haste-map-npm-29.7.0-e3be419eff.patch"
"jest-haste-map@^29.7.0": "patch:jest-haste-map@npm%3A29.7.0#./.yarn/patches/jest-haste-map-npm-29.7.0-e3be419eff.patch",
"@yarnpkg/plugin-npm@npm:^3.1.0": "patch:@yarnpkg/plugin-npm@npm%3A3.1.0#~/.yarn/patches/@yarnpkg-plugin-npm-npm-3.1.0-6533d0f5a1.patch"
},
"dependencies": {
"@backstage/errors": "workspace:^",
packages/yarn-plugin/README.md
+44
View File
@@ -38,3 +38,47 @@ The plugin can be manually tested in any repository running at least yarn 4.1.1.
Sadly it can't be manually tested directly in the Backstage monorepo - since
packages in this repository use `workspace:^` dependencies, there's no use case
for the yarn plugin.
## Architecture
This section is intended for people working directly on this package. It
describes the architecture of the plugin, and the means by which it manages npm
package versions.
The Backstage yarn plugin operates on `backstage:^` version ranges in
package.json files using the following three components:
### `reduceDependency` hook
_Converts `backstage:^` to `backstage:^::backstage=1.34.0&npm=1.2.3`_
This hook is called by yarn when resolving direct and indirect dependencies in
the workspace, and allows modifying the version range. The yarn plugin uses this
hook to parameterize `backstage:^` ranges with the current Backstage version and
the corresponding npm package version from the manifest. This uses the system
built into yarn for adding parameters to version ranges.
### `BackstageNpmResolver`
_Resolves the appropriate npm package for `backstage:^` ranges and adds the
`npm` range as a dependency_
The `BackstageNpmResolver` ensures that the lockfile contains entries for _both_
the `backstage:^` range, and the corresponding `npm:^<version>` range. Including
an entry for the `backstage:^` range means that tools that reconcile the
lockfile and package.json can match entries for Backstage packages together.
Including an entry for the corresponding `npm:` range ensures that dependencies
are not unlocked when switching between `backstage:^` and `npm:` ranges in the
lockfile, as happens when publishing the package or building a dist workspace
using `backstage-cli build-workspace`.
### `beforeWorkspacePacking` hook
_Replaces `backstage:^` ranges with the corresponding npm version ranges when
packing packages for publishing_
The yarn plugin is strictly optional, and intended to be opted-into in a
specific Backstage repository. As such, when publishing packages, all
`backstage:^` versions should be removed from the package.json and replaced with
the appropriate npm version ranges. This is handled by the
`beforeWorkspacePacking` hook.
packages/yarn-plugin/package.json
+6 -4
View File
@@ -32,17 +32,19 @@
"dependencies": {
"@backstage/cli-common": "workspace:^",
"@backstage/release-manifests": "workspace:^",
"@yarnpkg/core": "^4.0.3",
"@yarnpkg/fslib": "^3.0.2",
"@yarnpkg/plugin-pack": "^4.0.0",
"@yarnpkg/core": "^4.4.0",
"@yarnpkg/fslib": "^3.1.2",
"@yarnpkg/plugin-npm": "patch:@yarnpkg/plugin-npm@npm%3A3.1.0#~/.yarn/patches/@yarnpkg-plugin-npm-npm-3.1.0-6533d0f5a1.patch",
"@yarnpkg/plugin-pack": "^4.0.1",
"semver": "^7.6.0"
},
"devDependencies": {
"@backstage/backend-test-utils": "workspace:^",
"@backstage/cli": "workspace:^",
"@yarnpkg/builder": "^4.0.0",
"@yarnpkg/builder": "^4.2.1",
"fs-extra": "^11.2.0",
"nodemon": "^3.0.1",
"snyk-nodejs-lockfile-parser": "^1.58.14",
"yaml": "^2.0.0"
}
}
packages/yarn-plugin/src/handlers/reduceDependency.test.ts
+23 -12
View File
@@ -107,7 +107,7 @@ describe('reduceDependency', () => {
),
project,
),
).rejects.toThrow(/unexpected version selector/i);
).rejects.toThrow(/invalid backstage: version range/i);
},
);
@@ -127,20 +127,31 @@ describe('reduceDependency', () => {
);
});
it('replaces the range with the corresponding npm package range', async () => {
await expect(
reduceDependency(
structUtils.makeDescriptor(
structUtils.makeIdent('backstage', 'core'),
'backstage:^',
),
project,
),
).resolves.toEqual(
it('adds the current Backstage version as a parameter on the range', async () => {
const result = await reduceDependency(
structUtils.makeDescriptor(
structUtils.makeIdent('backstage', 'core'),
'npm:^6.7.8',
'backstage:^',
),
project,
);
await expect(
structUtils.parseRange(result.range).params?.backstage,
).toEqual('1.23.45');
});
it('adds the appropriate npm package version based on the Backstage manifest as a parameter on the range', async () => {
const result = await reduceDependency(
structUtils.makeDescriptor(
structUtils.makeIdent('backstage', 'core'),
'backstage:^',
),
project,
);
await expect(structUtils.parseRange(result.range).params?.npm).toEqual(
'6.7.8',
);
});
packages/yarn-plugin/src/handlers/reduceDependency.ts
+12 -6
View File
@@ -15,7 +15,7 @@
*/
import { Descriptor, Project, structUtils } from '@yarnpkg/core';
import { getPackageVersion } from '../util';
import { getCurrentBackstageVersion, getPackageVersion } from '../util';
import { PROTOCOL } from '../constants';
export const reduceDependency = async (
@@ -24,12 +24,18 @@ export const reduceDependency = async (
) => {
const range = structUtils.parseRange(dependency.range);
if (range.protocol === PROTOCOL) {
return structUtils.makeDescriptor(
dependency,
`npm:^${await getPackageVersion(dependency, project.configuration)}`,
if (range.protocol !== PROTOCOL) {
return dependency;
}
if (range.selector !== '^') {
throw new Error(
`Invalid backstage: version range found: ${dependency.range}`,
);
}
return dependency;
return structUtils.bindDescriptor(dependency, {
backstage: getCurrentBackstageVersion(),
npm: await getPackageVersion(dependency, project.configuration),
});
};
packages/yarn-plugin/src/index.test.ts
+27 -7
View File
@@ -18,6 +18,7 @@ import { join as joinPath } from 'path';
import { spawn, SpawnOptionsWithoutStdio } from 'child_process';
import fs from 'fs-extra';
import yaml from 'yaml';
import { buildDepTreeFromFiles } from 'snyk-nodejs-lockfile-parser';
import { findPaths } from '@backstage/cli-common';
import { createMockDirectory } from '@backstage/backend-test-utils';
@@ -66,10 +67,10 @@ const runYarnInstall = () =>
const nonWorkspaceEntries = (lockFileString: string = '') => {
const lockFile = yaml.parse(lockFileString);
const result: Record<string, string> = {};
const result = [];
for (const key of Object.keys(lockFile)) {
if (lockFile[key].version !== '0.0.0-use.local') {
result[key] = lockFile[key];
result.push(lockFile[key]);
}
}
@@ -231,15 +232,17 @@ describe('Backstage yarn plugin', () => {
'utf-8',
);
const lockFile = yaml.parse(lockFileContent);
const descriptors = Object.keys(yaml.parse(lockFileContent)).flatMap(
entry => entry.split(', '),
);
// Versions from old manifest no longer appear in lockfile
expect(lockFile['@backstage/cli-common@npm:^0.1.1']).toBeUndefined();
expect(lockFile['@backstage/config@npm:^0.1.2']).toBeUndefined();
expect(descriptors).not.toContain('@backstage/cli-common@npm:^0.1.1');
expect(descriptors).not.toContain('@backstage/config@npm:^0.1.2');
// Versions from new manifest have been added to lockfile
expect(lockFile['@backstage/cli-common@npm:^0.1.8']).toBeDefined();
expect(lockFile['@backstage/config@npm:^1.0.0']).toBeDefined();
expect(descriptors).toContain('@backstage/cli-common@npm:^0.1.8');
expect(descriptors).toContain('@backstage/config@npm:^1.0.0');
});
describe('after removing backstage:^ dependencies', () => {
@@ -301,6 +304,23 @@ describe('Backstage yarn plugin', () => {
});
},
);
it('successfully parses the lockfile with snyk-nodejs-lockfile-parser', async () => {
await expect(
buildDepTreeFromFiles(
mockDir.path,
'packages/b/package.json',
'yarn.lock',
),
).resolves.toEqual(
expect.objectContaining({
dependencies: expect.objectContaining({
'@backstage/cli-common': expect.anything(),
'@backstage/config': expect.anything(),
}),
}),
);
});
});
});
});
packages/yarn-plugin/src/index.ts
+2
View File
@@ -24,6 +24,7 @@
import { Plugin, Hooks, semverUtils, YarnVersion } from '@yarnpkg/core';
import { Hooks as PackHooks } from '@yarnpkg/plugin-pack';
import { beforeWorkspacePacking, reduceDependency } from './handlers';
import { BackstageNpmResolver } from './resolvers';
// All dependencies of the yarn plugin are bundled during the build. Chalk
// triples the size of the plugin bundle when included, so we avoid the
@@ -48,6 +49,7 @@ const plugin: Plugin<Hooks & PackHooks> = {
reduceDependency,
beforeWorkspacePacking,
},
resolvers: [BackstageNpmResolver],
};
export default plugin;
packages/yarn-plugin/src/resolvers/BackstageNpmResolver.test.ts
+167
View File
@@ -0,0 +1,167 @@
/*
* Copyright 2025 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { ResolveOptions, structUtils } from '@yarnpkg/core';
import { BackstageNpmResolver } from './BackstageNpmResolver';
import { NpmSemverResolver } from '@yarnpkg/plugin-npm';
const mockGetCandidates = jest.fn();
const mockGetSatisfying = jest.fn();
jest.mock('@yarnpkg/plugin-npm', () => {
return {
...jest.requireActual('@yarnpkg/plugin-npm'),
NpmSemverResolver: function MockNpmSemverResolver() {
return {
getCandidates: mockGetCandidates,
getSatisfying: mockGetSatisfying,
};
} as unknown as NpmSemverResolver,
};
});
/*
* Copyright 2025 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
describe('BackstageNpmResolver', () => {
const ident = structUtils.makeIdent('backstage', 'core');
afterEach(() => {
jest.clearAllMocks();
});
describe('getCandidates', () => {
it('should throw an error if the npm param is missing from the descriptor', async () => {
const resolver = new BackstageNpmResolver();
const descriptor = structUtils.bindDescriptor(
structUtils.makeDescriptor(ident, 'backstage:^'),
{ backstage: '1.0.0' },
);
await expect(
resolver.getCandidates(descriptor, undefined!, undefined!),
).rejects.toThrow(/Missing npm parameter/);
});
it('should call the npm resolver with the correct descriptor', async () => {
const resolver = new BackstageNpmResolver();
mockGetCandidates.mockResolvedValue([
structUtils.makeLocator(ident, '1.2.5'),
structUtils.makeLocator(ident, '1.2.4'),
structUtils.makeLocator(ident, '1.2.3'),
]);
const descriptor = structUtils.bindDescriptor(
structUtils.makeDescriptor(ident, 'backstage:^'),
{ backstage: '1.0.0', npm: '1.2.3' },
);
const deps = {};
const opts = {};
await expect(
resolver.getCandidates(descriptor, deps, opts as ResolveOptions),
).resolves.toEqual([
structUtils.makeLocator(ident, '1.2.5'),
structUtils.makeLocator(ident, '1.2.4'),
structUtils.makeLocator(ident, '1.2.3'),
]);
expect(mockGetCandidates).toHaveBeenCalledWith(
structUtils.makeDescriptor(descriptor, 'npm:^1.2.3'),
deps,
opts,
);
});
});
describe('getResolutionDependencies', () => {
it('should return the correct resolution dependencies', () => {
const resolver = new BackstageNpmResolver();
const descriptor = structUtils.bindDescriptor(
structUtils.makeDescriptor(ident, 'backstage:^'),
{ backstage: '1.0.0', npm: '1.2.3' },
);
expect(resolver.getResolutionDependencies(descriptor)).toEqual({
'@backstage/core': {
descriptorHash: expect.any(String),
identHash: descriptor.identHash,
name: 'core',
range: 'npm:^1.2.3',
scope: 'backstage',
},
});
});
});
describe('getSatisfying', () => {
it('should return the correct satisfying locator', async () => {
const resolver = new BackstageNpmResolver();
mockGetSatisfying.mockResolvedValue({
locators: [
structUtils.makeLocator(ident, '1.2.5'),
structUtils.makeLocator(ident, '1.2.4'),
structUtils.makeLocator(ident, '1.2.3'),
],
sorted: false,
});
const descriptor = structUtils.bindDescriptor(
structUtils.makeDescriptor(ident, 'backstage:^'),
{ backstage: '1.0.0', npm: '1.2.3' },
);
await expect(
resolver.getSatisfying(descriptor, {}, [], {} as ResolveOptions),
).resolves.toEqual({
locators: [
structUtils.makeLocator(ident, '1.2.5'),
structUtils.makeLocator(ident, '1.2.4'),
structUtils.makeLocator(ident, '1.2.3'),
],
sorted: false,
});
expect(mockGetSatisfying).toHaveBeenCalledWith(
{
descriptorHash: expect.any(String),
identHash: ident.identHash,
name: 'core',
range: 'npm:^1.2.3',
scope: 'backstage',
},
{},
[],
{} as ResolveOptions,
);
});
});
});
packages/yarn-plugin/src/resolvers/BackstageNpmResolver.ts
+152
View File
@@ -0,0 +1,152 @@
/*
* Copyright 2024 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {
structUtils,
Descriptor,
Locator,
Package,
Resolver,
ResolveOptions,
} from '@yarnpkg/core';
import { NpmSemverResolver } from '@yarnpkg/plugin-npm';
import { PROTOCOL } from '../constants';
export class BackstageNpmResolver implements Resolver {
static protocol = PROTOCOL;
/**
* Target only descriptors using the `backstage:` protocol
*/
supportsDescriptor = (descriptor: Descriptor) =>
descriptor.range.startsWith(BackstageNpmResolver.protocol);
/**
* We treat any `backstage:` descriptor as if it's targeting the npm package
* version from the manifest for the current version of Backstage, by pulling
* in the `NpmSemverResolver` and deferring to its `getCandidates` method.
*
* The version itself comes from the `npm` parameter on the incoming
* descriptor, which is set by the `reduceDependency` hook.
*/
async getCandidates(
descriptor: Descriptor,
dependencies: Record<string, Package>,
opts: ResolveOptions,
): Promise<Locator[]> {
const npmVersion = structUtils.parseRange(descriptor.range).params?.npm;
if (!npmVersion || Array.isArray(npmVersion)) {
throw new Error(
`Missing npm parameter on backstage: range "${descriptor.range}"`,
);
}
return new NpmSemverResolver().getCandidates(
structUtils.makeDescriptor(descriptor, `npm:^${npmVersion}`),
dependencies,
opts,
);
}
/**
* We insert the `npm:^<version>` descriptor as an additional dependency to
* ensure that dependencies remain locked when adding and removing the plugin
* from repositories. This is relevant for example when building packed
* production-like workspaces for testing using `backstage-cli
* build-workspace`.
*/
getResolutionDependencies(
descriptor: Descriptor,
): Record<string, Descriptor> {
const npmVersion = structUtils.parseRange(descriptor.range).params?.npm;
if (!npmVersion) {
throw new Error(
`Missing npm parameter on backstage: range "${descriptor.range}".`,
);
}
return {
[structUtils.stringifyIdent(descriptor)]: structUtils.makeDescriptor(
descriptor,
`npm:^${npmVersion}`,
),
};
}
/**
* This method is called when deduplicating locators. We first convert any
* `backstage:` locators into the corresponding `npm:` locator, and then defer
* to the implementation from `NpmSemverResolver`.
*/
async getSatisfying(
descriptor: Descriptor,
dependencies: Record<string, Package>,
locators: Array<Locator>,
opts: ResolveOptions,
) {
let npmDescriptor = descriptor;
const range = structUtils.parseRange(npmDescriptor.range);
if (range.protocol === PROTOCOL) {
const npmVersion = range.params?.npm;
npmDescriptor = structUtils.makeDescriptor(
descriptor,
`npm:^${npmVersion}`,
);
}
return new NpmSemverResolver().getSatisfying(
npmDescriptor,
dependencies,
locators,
opts,
);
}
/**
* Stub - no descriptor binding is needed in this resolver (note though that
* it does rely on the binding performed in the `reduceDependency` hook).
*/
bindDescriptor = (descriptor: Descriptor) => descriptor;
/**
* This plugin does not need to support any locators itself, since the
* `getCandidates` method will always convert `backstage:` versions into
* `npm:` versions which are resolved using the `NpmSemverResolver`.
*/
supportsLocator = () => false;
/**
* This method should never be called, since all emitted locators use the
* `npm:` protocol.
*/
shouldPersistResolution = () => {
throw new Error(
'Unreachable: BackstageNpmResolver should never persist resolution as it uses npm: protocol',
);
};
/**
* This method should never be called, since all emitted locators use the
* `npm:` protocol.
*/
resolve = async () => {
throw new Error(
'Unreachable: BackstageNpmResolver should never resolve as it uses npm: protocol',
);
};
}
packages/yarn-plugin/src/resolvers/index.ts
+16
View File
@@ -0,0 +1,16 @@
/*
* Copyright 2025 The Backstage Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export { BackstageNpmResolver } from './BackstageNpmResolver';
packages/yarn-plugin/src/util/index.ts
+1
View File
@@ -14,4 +14,5 @@
* limitations under the License.
*/
export { getCurrentBackstageVersion } from './getCurrentBackstageVersion';
export { getPackageVersion } from './getPackageVersion';
yarn.lock
+981 -348
View File
File diff suppressed because it is too large Load Diff