diff --git a/packages/cli/package.json b/packages/cli/package.json
index 5ac957d5f7..049f3f903a 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -78,10 +78,10 @@
     "rollup": "2.23.x",
     "rollup-plugin-dts": "1.4.11",
     "rollup-plugin-esbuild": "^2.0.0",
-    "rollup-plugin-image-files": "^1.4.2",
     "rollup-plugin-peer-deps-external": "^2.2.2",
     "rollup-plugin-postcss": "^3.1.1",
     "rollup-plugin-typescript2": "^0.27.3",
+    "rollup-pluginutils": "^2.8.2",
     "start-server-webpack-plugin": "^2.2.5",
     "style-loader": "^1.2.1",
     "sucrase": "^3.14.1",
diff --git a/packages/cli/src/lib/builder/config.ts b/packages/cli/src/lib/builder/config.ts
index afcedaae59..2566460494 100644
--- a/packages/cli/src/lib/builder/config.ts
+++ b/packages/cli/src/lib/builder/config.ts
@@ -22,13 +22,13 @@ import commonjs from '@rollup/plugin-commonjs';
 import resolve from '@rollup/plugin-node-resolve';
 import postcss from 'rollup-plugin-postcss';
 import esbuild from 'rollup-plugin-esbuild';
-import imageFiles from 'rollup-plugin-image-files';
 import svgr from '@svgr/rollup';
 import dts from 'rollup-plugin-dts';
 import json from '@rollup/plugin-json';
 import yaml from '@rollup/plugin-yaml';
 import { RollupOptions, OutputOptions } from 'rollup';
 
+import { forwardFileImports } from './plugins';
 import { BuildOptions, Output } from './types';
 import { paths } from '../paths';
 import { svgrTemplate } from '../svgrTemplate';
@@ -92,7 +92,7 @@ export const makeConfigs = async (
           exclude: [/\/[^/]+\.(?:stories|test)\.[^/]+$/],
         }),
         postcss(),
-        imageFiles({
+        forwardFileImports({
           exclude: /\.icon\.svg$/,
           include: [/\.svg$/, /\.png$/, /\.gif$/, /\.jpg$/, /\.jpeg$/],
         }),
diff --git a/packages/cli/src/lib/builder/plugins.ts b/packages/cli/src/lib/builder/plugins.ts
new file mode 100644
index 0000000000..48b6b65c61
--- /dev/null
+++ b/packages/cli/src/lib/builder/plugins.ts
@@ -0,0 +1,135 @@
+/*
+ * Copyright 2020 Spotify AB
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs-extra';
+import {
+  dirname,
+  resolve as resolvePath,
+  relative as relativePath,
+} from 'path';
+import { createFilter } from 'rollup-pluginutils';
+import { Plugin, InputOptions, OutputChunk } from 'rollup';
+
+type ForwardFileImportsOptions = {
+  include: Array<string | RegExp> | string | RegExp | null;
+  exclude?: Array<string | RegExp> | string | RegExp | null;
+};
+
+/**
+ * This rollup plugin leaves all encountered asset imports as-is, but
+ * copies the imported files into the output directory.
+ *
+ * For example `import ImageUrl from './my-image.png'` inside `src/MyComponent` will
+ * cause `src/MyComponent/my-image.png` to be copied to the output directory at the
+ * path `dist/MyComponent/my-image.png`. The import itself will stay, but be resolved,
+ * resulting in something like `import ImageUrl from './MyComponent/my-image.png'`
+ */
+export function forwardFileImports(options: ForwardFileImportsOptions): Plugin {
+  const filter = createFilter(options.include, options.exclude);
+
+  // We collect the absolute paths to all files we want to bundle into the
+  // output dir here. Resolving to relative paths in the output dir happens later.
+  const exportedFiles = new Set<string>();
+
+  // We keep track of output directories that we've already copied files
+  // into, so that we don't duplicate that work
+  const generatedFor = new Set<string>();
+
+  return {
+    name: 'forward-file-imports',
+    async generateBundle(outputOptions, bundle, isWrite) {
+      if (!isWrite) {
+        return;
+      }
+
+      const dir = outputOptions.dir || dirname(outputOptions.file!);
+      if (generatedFor.has(dir)) {
+        return;
+      }
+
+      for (const output of Object.values(bundle)) {
+        if (output.type !== 'chunk') {
+          continue;
+        }
+        const chunk = output as OutputChunk;
+
+        // This'll be an absolute path pointing to the initial index file of the
+        // build, and we use it to find the location of the `src` dir
+        if (!chunk.facadeModuleId) {
+          continue;
+        }
+        generatedFor.add(dir);
+
+        // We're assuming that the index file is at the root of the source dir, and
+        // that all assets exist within that dir.
+        const srcRoot = dirname(chunk.facadeModuleId);
+
+        // Copy all the files we found into the dist dir
+        await Promise.all(
+          Array.from(exportedFiles).map(async exportedFile => {
+            const outputPath = relativePath(srcRoot, exportedFile);
+            const targetFile = resolvePath(dir, outputPath);
+
+            await fs.ensureDir(dirname(targetFile));
+            await fs.copyFile(exportedFile, targetFile);
+          }),
+        );
+        return;
+      }
+    },
+    options(inputOptions) {
+      const origExternal = inputOptions.external;
+
+      // We decorate any existing `external` option with our own way of determining
+      // if a module should be external. The can't use `resolveId`, since asset files
+      // aren't passed there, might be some better way to do this though.
+      const external: InputOptions['external'] = (id, importer, isResolved) => {
+        // Call to inner external option
+        if (
+          typeof origExternal === 'function' &&
+          origExternal(id, importer, isResolved)
+        ) {
+          return true;
+        }
+
+        if (Array.isArray(origExternal) && origExternal.includes(id)) {
+          return true;
+        }
+
+        // The piece that we're adding
+        if (!filter(id)) {
+          return false;
+        }
+
+        // Sanity check, dunno if this can happen
+        if (!importer) {
+          throw new Error(`Unknown importer of file module ${id}`);
+        }
+
+        // Resolve relative imports to the full file URL, for deduping and copying later
+        const fullId = isResolved ? id : resolvePath(dirname(importer), id);
+        exportedFiles.add(fullId);
+
+        // Treating this module as external from here, meaning rollup won't try to
+        // put it in the output bundle, but still keep track of the relative imports
+        // as needed in the output code.
+        return true;
+      };
+
+      return { ...inputOptions, external };
+    },
+  };
+}
diff --git a/yarn.lock b/yarn.lock
index 9cee3c3602..e137f61f09 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -10764,7 +10764,7 @@ estraverse@^5.1.0:
   resolved "https://registry.npmjs.org/estraverse/-/estraverse-5.1.0.tgz#374309d39fd935ae500e7b92e8a6b4c720e59642"
   integrity sha512-FyohXK+R0vE+y1nHLoBM7ZTyqRpqAlhdZHCWIWEviFLiGB8b04H6bQs8G+XTthacvT8VuwvteiP7RJSxMs8UEw==
 
-estree-walker@^0.6.0, estree-walker@^0.6.1:
+estree-walker@^0.6.1:
   version "0.6.1"
   resolved "https://registry.npmjs.org/estree-walker/-/estree-walker-0.6.1.tgz#53049143f40c6eb918b23671d1fe3219f3a1b362"
   integrity sha512-SqmZANLWS0mnatqbSfRP5g8OXZC12Fgg1IwNtLsyHDzJizORW4khDfjPqJZsemPWBB2uqykUah5YpQ6epsqC/w==
@@ -20229,14 +20229,6 @@ rollup-plugin-esbuild@^2.0.0:
   dependencies:
     "@rollup/pluginutils" "^3.1.0"
 
-rollup-plugin-image-files@^1.4.2:
-  version "1.4.2"
-  resolved "https://registry.npmjs.org/rollup-plugin-image-files/-/rollup-plugin-image-files-1.4.2.tgz#0a329723bace95168a9a6efdacb51370c14fbfe5"
-  integrity sha512-tlydpQdGFssMWhPdWA9SFh4IGVSCzceNgJarJDID+km151IeIVzjATl8ZERNGS/QwMty25iammQqauxl1VUqDQ==
-  dependencies:
-    rollup "0.64.1"
-    rollup-pluginutils "2.4.1"
-
 rollup-plugin-peer-deps-external@^2.2.2:
   version "2.2.3"
   resolved "https://registry.npmjs.org/rollup-plugin-peer-deps-external/-/rollup-plugin-peer-deps-external-2.2.3.tgz#059a8aec1eefb48a475e9fcedc3b9e3deb521213"
@@ -20273,14 +20265,6 @@ rollup-plugin-typescript2@^0.27.3:
     resolve "1.17.0"
     tslib "2.0.1"
 
-rollup-pluginutils@2.4.1:
-  version "2.4.1"
-  resolved "https://registry.npmjs.org/rollup-pluginutils/-/rollup-pluginutils-2.4.1.tgz#de43ab54965bbf47843599a7f3adceb723de38db"
-  integrity sha512-wesMQ9/172IJDIW/lYWm0vW0LiKe5Ekjws481R7z9WTRtmO59cqyM/2uUlxvf6yzm/fElFmHUobeQOYz46dZJw==
-  dependencies:
-    estree-walker "^0.6.0"
-    micromatch "^3.1.10"
-
 rollup-pluginutils@^2.8.2:
   version "2.8.2"
   resolved "https://registry.npmjs.org/rollup-pluginutils/-/rollup-pluginutils-2.8.2.tgz#72f2af0748b592364dbd3389e600e5a9444a351e"
@@ -20288,14 +20272,6 @@ rollup-pluginutils@^2.8.2:
   dependencies:
     estree-walker "^0.6.1"
 
-rollup@0.64.1:
-  version "0.64.1"
-  resolved "https://registry.npmjs.org/rollup/-/rollup-0.64.1.tgz#9188ee368e5fcd43ffbc00ec414e72eeb5de87ba"
-  integrity sha512-+ThdVXrvonJdOTzyybMBipP0uz605Z8AnzWVY3rf+cSGnLO7uNkJBlN+9jXqWOomkvumXfm/esmBpA5d53qm7g==
-  dependencies:
-    "@types/estree" "0.0.39"
-    "@types/node" "*"
-
 rollup@2.23.x:
   version "2.23.0"
   resolved "https://registry.npmjs.org/rollup/-/rollup-2.23.0.tgz#b7ab1fee0c0e60132fd0553c4df1e9cdacfada9d"