wip

packages/excalidraw/actions/actionRemoveBackground.tsx

@@ -0,0 +1,113 @@
+import { generateIdFromFile, getDataURL } from "../data/blob";
+import { mutateElement } from "../element/mutateElement";
+import { isInitializedImageElement } from "../element/typeChecks";
+import type { InitializedExcalidrawImageElement } from "../element/types";
+import type { BinaryFileData } from "../types";
+import { register } from "./register";
+
+export const actionRemoveBackground = register({
+  name: "removeBackground",
+  label: "stats.fullTitle",
+  trackEvent: false,
+  viewMode: false,
+  async perform(elements, appState, _, app) {
+    const selectedElements = app.scene.getSelectedElements(appState);
+
+    if (
+      selectedElements.length > 0 &&
+      selectedElements.every(isInitializedImageElement)
+    ) {
+      const filesToProcess = selectedElements.reduce(
+        (
+          acc: Map<
+            BinaryFileData["id"],
+            {
+              file: BinaryFileData;
+              elements: InitializedExcalidrawImageElement[];
+            }
+          >,
+          imageElement,
+        ) => {
+          const file = app.files[imageElement.fileId];
+
+          if (file) {
+            const fileWithRemovedBackground = Object.values(app.files).find(
+              (_file) =>
+                _file.customData?.source === "backgroundRemoval" &&
+                _file.customData.parentFileId === file.id,
+            );
+
+            if (fileWithRemovedBackground) {
+              mutateElement(
+                imageElement,
+                { fileId: fileWithRemovedBackground.id },
+                false,
+              );
+            } else if (acc.has(file.id)) {
+              acc.get(file.id)!.elements.push(imageElement);
+            } else {
+              acc.set(file.id, { file, elements: [imageElement] });
+            }
+          }
+          return acc;
+        },
+        new Map(),
+      );
+
+      if (filesToProcess.size) {
+        const backgroundRemoval = await await import(
+          "@imgly/background-removal"
+        );
+
+        console.time("removeBackground");
+
+        for (const [, { file, elements }] of filesToProcess) {
+          const res = await backgroundRemoval.removeBackground(file.dataURL, {
+            // debug: true,
+            progress: (...args) => {
+              console.log("progress", args);
+            },
+            device: "gpu",
+            proxyToWorker: true,
+          });
+
+          const fileId = await generateIdFromFile(res);
+          const dataURL = await getDataURL(res);
+
+          for (const imageElement of elements) {
+            mutateElement(imageElement, { fileId }, false);
+          }
+
+          app.addFiles([
+            {
+              ...file,
+              id: fileId,
+              dataURL,
+              customData: {
+                source: "backgroundRemoval",
+                version: 1,
+                parentFileId: file.id,
+              },
+            },
+          ]);
+        }
+
+        console.timeEnd("removeBackground");
+      }
+
+      app.scene.triggerUpdate();
+    }
+    return false as false;
+  },
+  PanelComponent: ({ updateData }) => {
+    return (
+      <button
+        onClick={() => {
+          updateData();
+        }}
+      >
+        Remove background
+      </button>
+    );
+  },
+});

packages/excalidraw/actions/index.ts

@@ -86,3 +86,4 @@ export { actionUnbindText, actionBindText } from "./actionBoundText";
 export { actionLink } from "./actionLink";
 export { actionToggleElementLock } from "./actionElementLock";
 export { actionToggleLinearEditor } from "./actionLinearEditor";
+export { actionRemoveBackground } from "./actionRemoveBackground";

packages/excalidraw/actions/types.ts

@@ -136,7 +136,8 @@ export type ActionName =
   | "wrapTextInContainer"
   | "commandPalette"
   | "autoResize"
-  | "elementStats";
+  | "elementStats"
+  | "removeBackground";
 
 export type PanelComponentProps = {
   elements: readonly ExcalidrawElement[];

packages/excalidraw/components/Actions.tsx

@@ -25,6 +25,7 @@ import { hasStrokeColor } from "../scene/comparisons";
 import { trackEvent } from "../analytics";
 import {
   hasBoundTextElement,
+  isInitializedImageElement,
   isLinearElement,
   isTextElement,
 } from "../element/typeChecks";
@@ -125,6 +126,10 @@ export const SelectedShapeActions = ({
 
   return (
     <div className="panelColumn">
+      {targetElements.length > 0 &&
+        targetElements.every(isInitializedImageElement) && (
+          <div>{renderAction("removeBackground")}</div>
+        )}
       <div>
         {canChangeStrokeColor(appState, targetElements) &&
           renderAction("changeStrokeColor")}

packages/excalidraw/data/blob.ts

@@ -235,7 +235,9 @@ export const canvasToBlob = async (
 
 /** generates SHA-1 digest from supplied file (if not supported, falls back
     to a 40-char base64 random id) */
-export const generateIdFromFile = async (file: File): Promise<FileId> => {
+export const generateIdFromFile = async (
+  file: File | Blob,
+): Promise<FileId> => {
   try {
     const hashBuffer = await window.crypto.subtle.digest(
       "SHA-1",

packages/excalidraw/types.ts

@@ -108,6 +108,7 @@ export type BinaryFileData = {
    * Epoch timestamp in milliseconds.
    */
   lastRetrieved?: number;
+  customData?: Record<string, any>;
 };
 
 export type BinaryFileMetadata = Omit<BinaryFileData, "dataURL">;