diff --git a/.changeset/twelve-peaches-develop.md b/.changeset/twelve-peaches-develop.md
new file mode 100644
index 0000000000..804c352ba4
--- /dev/null
+++ b/.changeset/twelve-peaches-develop.md
@@ -0,0 +1,5 @@
+---
+'@backstage/plugin-catalog-react': patch
+---
+
+Fix label related accessibility issues with `FavorityEntity`
diff --git a/plugins/catalog-react/src/components/FavoriteEntity/FavoriteEntity.test.tsx b/plugins/catalog-react/src/components/FavoriteEntity/FavoriteEntity.test.tsx
new file mode 100644
index 0000000000..d36700452a
--- /dev/null
+++ b/plugins/catalog-react/src/components/FavoriteEntity/FavoriteEntity.test.tsx
@@ -0,0 +1,106 @@
+/*
+ * Copyright 2024 The Backstage Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import { storageApiRef } from '@backstage/core-plugin-api';
+
+import { MockStarredEntitiesApi, starredEntitiesApiRef } from '../../apis';
+import { FavoriteEntity } from './FavoriteEntity';
+import { ComponentEntity } from '@backstage/catalog-model';
+import {
+  MockStorageApi,
+  renderInTestApp,
+  TestApiProvider,
+} from '@backstage/test-utils';
+import { screen } from '@testing-library/react';
+import userEvent from '@testing-library/user-event';
+
+const entity: ComponentEntity = {
+  apiVersion: 'backstage.io/v1alpha1',
+  kind: 'Component',
+  metadata: {
+    name: 'example',
+  },
+  spec: {
+    type: 'service',
+    lifecycle: 'experimental',
+    owner: 'user:default/john.doe_example.com',
+  },
+};
+
+const mockStorage = MockStorageApi.create();
+
+describe('<FavoriteEntity/>', () => {
+  it('should add to favorites', async () => {
+    await renderInTestApp(
+      <TestApiProvider
+        apis={[
+          [storageApiRef, mockStorage],
+          [starredEntitiesApiRef, new MockStarredEntitiesApi()],
+        ]}
+      >
+        <FavoriteEntity entity={entity} />
+      </TestApiProvider>,
+    );
+
+    const addToFavorite = screen.getByRole('button', {
+      name: 'Add to favorites',
+    });
+
+    // Should keep the label when hovering
+    await userEvent.hover(addToFavorite);
+    expect(addToFavorite).toBeInTheDocument();
+
+    await userEvent.click(addToFavorite);
+    expect(
+      screen.getByRole('button', {
+        name: 'Remove from favorites',
+      }),
+    ).toBeInTheDocument();
+  });
+
+  it('should remove from favorites', async () => {
+    const starredEntities = new MockStarredEntitiesApi();
+    await starredEntities.toggleStarred('component:default/example');
+
+    await renderInTestApp(
+      <TestApiProvider
+        apis={[
+          [storageApiRef, mockStorage],
+          [starredEntitiesApiRef, starredEntities],
+        ]}
+      >
+        <FavoriteEntity entity={entity} />
+      </TestApiProvider>,
+    );
+
+    const removeFromFavorites = screen.getByRole('button', {
+      name: 'Remove from favorites',
+    });
+
+    // Should keep the label when hovering
+    await userEvent.hover(removeFromFavorites);
+    expect(removeFromFavorites).toBeInTheDocument();
+
+    await userEvent.click(removeFromFavorites);
+
+    expect(
+      screen.getByRole('button', {
+        name: 'Add to favorites',
+      }),
+    ).toBeInTheDocument();
+  });
+});
diff --git a/plugins/catalog-react/src/components/FavoriteEntity/FavoriteEntity.tsx b/plugins/catalog-react/src/components/FavoriteEntity/FavoriteEntity.tsx
index 8848857976..ab674ff98e 100644
--- a/plugins/catalog-react/src/components/FavoriteEntity/FavoriteEntity.tsx
+++ b/plugins/catalog-react/src/components/FavoriteEntity/FavoriteEntity.tsx
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { Entity } from '@backstage/catalog-model';
+import { Entity, stringifyEntityRef } from '@backstage/catalog-model';
 import IconButton from '@material-ui/core/IconButton';
 import Tooltip from '@material-ui/core/Tooltip';
 import { withStyles } from '@material-ui/core/styles';
@@ -46,20 +46,24 @@ export const FavoriteEntity = (props: FavoriteEntityProps) => {
     props.entity,
   );
   const { t } = useTranslationRef(catalogReactTranslationRef);
+  const title = isStarredEntity
+    ? t('favoriteEntity.removeFromFavorites')
+    : t('favoriteEntity.addToFavorites');
+
+  const id = `favorite-${stringifyEntityRef(props.entity).replace(
+    /[^a-zA-Z0-9-_]/g,
+    '-',
+  )}`;
+
   return (
     <IconButton
-      aria-label="favorite"
+      aria-label={title}
+      id={id}
       color="inherit"
       {...props}
       onClick={() => toggleStarredEntity()}
     >
-      <Tooltip
-        title={
-          isStarredEntity
-            ? t('favoriteEntity.removeFromFavorites')
-            : t('favoriteEntity.addToFavorites')
-        }
-      >
+      <Tooltip id={id} title={title}>
         {isStarredEntity ? <YellowStar /> : <StarBorder />}
       </Tooltip>
     </IconButton>
diff --git a/plugins/scaffolder-react/src/next/components/TemplateCard/CardHeader.test.tsx b/plugins/scaffolder-react/src/next/components/TemplateCard/CardHeader.test.tsx
index 90157c3226..58033b718c 100644
--- a/plugins/scaffolder-react/src/next/components/TemplateCard/CardHeader.test.tsx
+++ b/plugins/scaffolder-react/src/next/components/TemplateCard/CardHeader.test.tsx
@@ -119,7 +119,7 @@ describe('CardHeader', () => {
       </TestApiProvider>,
     );
 
-    const favorite = getByRole('button', { name: 'favorite' });
+    const favorite = getByRole('button', { name: 'Add to favorites' });
 
     await fireEvent.click(favorite);