Fix dialog dismissal in Backstage UI (#33785)

Dismissing a dialog by clicking on the overlay doesn't work currently despite being enabled. This is because what appears to be the overlay in the UI is actually the modal content instead, as the classes are applied incorrectly. This PR fixes that by separating out the overlay from the Modal component and lifting each of the classes up one layer.

---------

Signed-off-by: James Brooks <jamesbrooks@spotify.com>

.changeset/neat-impalas-cross.md

@@ -0,0 +1,7 @@
+---
+'@backstage/ui': patch
+---
+
+Added `ModalOverlay` to `Dialog` so overlay styles are applied to the actual overlay rather than the modal content, and fixed dismissing via outside click in the process.
+
+**Affected components:** Dialog

packages/ui/report.api.md

@@ -1124,7 +1124,8 @@ export const DialogDefinition: {
   };
   readonly classNames: {
     readonly root: 'bui-DialogOverlay';
-    readonly dialog: 'bui-Dialog';
+    readonly container: 'bui-Dialog';
+    readonly inner: 'bui-DialogInner';
     readonly content: 'bui-DialogContent';
   };
   readonly propDefs: {

packages/ui/src/components/Dialog/Dialog.module.css

@@ -56,6 +56,13 @@
     max-width: calc(100vw - 3rem);
     height: var(--bui-dialog-height, auto);
     max-height: calc(100vh - 3rem);
+  }
+
+  .bui-DialogInner {
+    display: flex;
+    flex-direction: column;
+    flex: 1;
+    min-height: 0;
     outline: none;
   }
 

packages/ui/src/components/Dialog/Dialog.tsx

@@ -19,6 +19,7 @@ import {
   Dialog as RADialog,
   DialogTrigger as RADialogTrigger,
   Modal,
+  ModalOverlay,
   Heading,
 } from 'react-aria-components';
 import type {
@@ -58,20 +59,20 @@ export const DialogTrigger = (props: DialogTriggerProps) => {
 export const Dialog = forwardRef<React.ElementRef<typeof Modal>, DialogProps>(
   (props, ref) => {
     const { ownProps, restProps } = useDefinition(DialogDefinition, props, {
-      classNameTarget: 'dialog',
+      classNameTarget: 'container',
     });
     const { classes, children, width, height, style } = ownProps;
 
     return (
-      <Modal
-        ref={ref}
+      <ModalOverlay
         className={classes.root}
         isDismissable
         isKeyboardDismissDisabled={false}
         {...restProps}
       >
-        <RADialog
-          className={classes.dialog}
+        <Modal
+          ref={ref}
+          className={classes.container}
           style={{
             ['--bui-dialog-min-width' as keyof React.CSSProperties]:
               typeof width === 'number' ? `${width}px` : width || '400px',
@@ -84,13 +85,15 @@ export const Dialog = forwardRef<React.ElementRef<typeof Modal>, DialogProps>(
             ...style,
           }}
         >
-          <BgReset>
-            <Box bg="neutral" className={classes.content}>
-              {children}
-            </Box>
-          </BgReset>
-        </RADialog>
-      </Modal>
+          <RADialog className={classes.inner}>
+            <BgReset>
+              <Box bg="neutral" className={classes.content}>
+                {children}
+              </Box>
+            </BgReset>
+          </RADialog>
+        </Modal>
+      </ModalOverlay>
     );
   },
 );

packages/ui/src/components/Dialog/definition.ts

@@ -31,7 +31,8 @@ export const DialogDefinition = defineComponent<DialogOwnProps>()({
   styles,
   classNames: {
     root: 'bui-DialogOverlay',
-    dialog: 'bui-Dialog',
+    container: 'bui-Dialog',
+    inner: 'bui-DialogInner',
     content: 'bui-DialogContent',
   },
   propDefs: {