Feature: Multi Point Arrows (#338)

* Add points to arrow on double click

* Use line generator instead of path to generate line segments

* Switch color of the circle when it is on an existing point in the segment

* Check point against both ends of the line segment to find collinearity

* Keep drawing the arrow based on mouse position until shape is changed

* Always select the arrow when in multi element mode

* Use curves instead of lines when drawing arrow points

* Add basic collision detection with some debug points

* Use roughjs shape when performing hit testing

* Draw proper handler rectangles for arrows

* Add argument to renderScene in export

* Globally resize all points on the arrow when bounds are resized

* Hide handler rectangles if an arrow has no size

- Allow continuing adding arrows when selected element is deleted

* Add dragging functionality to arrows

* Add SHIFT functionality to two point arrows

- Fix arrow positions when scrolling
- Revert the element back to selection when not in multi select mode

* Clean app state for export (JSON)

* Set curve options manually instead of using global options

- For some reason, this fixed the flickering issue in all shapes when arrows are rendered

* Set proper options for the arrow

* Increaase accuracy of hit testing arrows

- Additionally, skip testing if point is outside the domain of arrow and each curve

* Calculate bounding box of arrow based on roughjs curves

- Remove domain check per curve

* Change bounding box threshold to 10 and remove unnecessary code

* Fix handler rectangles for 2 and multi point arrows

- Fix margins of handler rectangles when using arrows
- Show handler rectangles in endpoints of 2-point arrows

* Remove unnecessary values from app state for export

* Use `resetTransform` instead of "retranslating" canvas space after each element rendering

* Allow resizing 2-point arrows

- Fix position of one of the handler rectangles

* refactor variable initialization

* Refactored to extract out mult-point generation to the abstracted function

* prevent dragging on arrow creation if under threshold

* Finalize selection during multi element mode when ENTER or ESC is clicked

* Set dragging element to null when finalizing

* Remove pathSegmentCircle from code

* Check if element is any "non-value" instead of NULL

* Show two points on any two point arrow and fix visibility of arrows during scroll

* Resume recording when done with drawing

- When deleting a multi select element, revert back to selection element type

* Resize arrow starting points perfectly

* Fix direction of arrow resize based for NW

* Resume recording history when there is more than one arrow

* Set dragging element to NULL when element is not locked

* Blur active element when finalizing

* Disable undo/redo for multielement, editingelement, and resizing element

- Allow undoing parts of the arrow

* Disable element visibility for arrow

* Use points array for arrow bounds when bezier curve shape is not available

Co-authored-by: David Luzar <luzar.david@gmail.com>
Co-authored-by: Preet <833927+pshihn@users.noreply.github.com>

src/element/bounds.ts

@@ -1,11 +1,16 @@
 import { ExcalidrawElement } from "./types";
 import { rotate } from "../math";
+import { Drawable } from "roughjs/bin/core";
+import { Point } from "roughjs/bin/geometry";
 
 // If the element is created from right to left, the width is going to be negative
 // This set of functions retrieves the absolute position of the 4 points.
 // We can't just always normalize it since we need to remember the fact that an arrow
 // is pointing left or right.
 export function getElementAbsoluteCoords(element: ExcalidrawElement) {
+  if (element.type === "arrow") {
+    return getArrowAbsoluteBounds(element);
+  }
   return [
     element.width >= 0 ? element.x : element.x + element.width, // x1
     element.height >= 0 ? element.y : element.y + element.height, // y1
@@ -29,11 +34,95 @@ export function getDiamondPoints(element: ExcalidrawElement) {
   return [topX, topY, rightX, rightY, bottomX, bottomY, leftX, leftY];
 }
 
+export function getArrowAbsoluteBounds(element: ExcalidrawElement) {
+  if (element.points.length < 2 || !element.shape) {
+    const { minX, minY, maxX, maxY } = element.points.reduce(
+      (limits, [x, y]) => {
+        limits.minY = Math.min(limits.minY, y);
+        limits.minX = Math.min(limits.minX, x);
+
+        limits.maxX = Math.max(limits.maxX, x);
+        limits.maxY = Math.max(limits.maxY, y);
+
+        return limits;
+      },
+      { minX: Infinity, minY: Infinity, maxX: -Infinity, maxY: -Infinity },
+    );
+    return [
+      minX + element.x,
+      minY + element.y,
+      maxX + element.x,
+      maxY + element.y,
+    ];
+  }
+
+  const shape = element.shape as Drawable[];
+
+  const ops = shape[1].sets[0].ops;
+
+  let currentP: Point = [0, 0];
+
+  const { minX, minY, maxX, maxY } = ops.reduce(
+    (limits, { op, data }) => {
+      // There are only four operation types:
+      // move, bcurveTo, lineTo, and curveTo
+      if (op === "move") {
+        // change starting point
+        currentP = data as Point;
+        // move operation does not draw anything; so, it always
+        // returns false
+      } else if (op === "bcurveTo") {
+        // create points from bezier curve
+        // bezier curve stores data as a flattened array of three positions
+        // [x1, y1, x2, y2, x3, y3]
+        const p1 = [data[0], data[1]] as Point;
+        const p2 = [data[2], data[3]] as Point;
+        const p3 = [data[4], data[5]] as Point;
+
+        const p0 = currentP;
+        currentP = p3;
+
+        const equation = (t: number, idx: number) =>
+          Math.pow(1 - t, 3) * p3[idx] +
+          3 * t * Math.pow(1 - t, 2) * p2[idx] +
+          3 * Math.pow(t, 2) * (1 - t) * p1[idx] +
+          p0[idx] * Math.pow(t, 3);
+
+        let t = 0;
+        while (t <= 1.0) {
+          const x = equation(t, 0);
+          const y = equation(t, 1);
+
+          limits.minY = Math.min(limits.minY, y);
+          limits.minX = Math.min(limits.minX, x);
+
+          limits.maxX = Math.max(limits.maxX, x);
+          limits.maxY = Math.max(limits.maxY, y);
+
+          t += 0.1;
+        }
+      } else if (op === "lineTo") {
+        // TODO: Implement this
+      } else if (op === "qcurveTo") {
+        // TODO: Implement this
+      }
+      return limits;
+    },
+    { minX: Infinity, minY: Infinity, maxX: -Infinity, maxY: -Infinity },
+  );
+
+  return [
+    minX + element.x,
+    minY + element.y,
+    maxX + element.x,
+    maxY + element.y,
+  ];
+}
+
 export function getArrowPoints(element: ExcalidrawElement) {
-  const x1 = 0;
-  const y1 = 0;
-  const x2 = element.width;
-  const y2 = element.height;
+  const points = element.points;
+  const [x1, y1] = points.length >= 2 ? points[points.length - 2] : [0, 0];
+  const [x2, y2] = points[points.length - 1];
 
   const size = 30; // pixels
   const distance = Math.hypot(x2 - x1, y2 - y1);
@@ -46,7 +135,7 @@ export function getArrowPoints(element: ExcalidrawElement) {
   const [x3, y3] = rotate(xs, ys, x2, y2, (-angle * Math.PI) / 180);
   const [x4, y4] = rotate(xs, ys, x2, y2, (angle * Math.PI) / 180);
 
-  return [x1, y1, x2, y2, x3, y3, x4, y4];
+  return [x2, y2, x3, y3, x4, y4];
 }
 
 export function getLinePoints(element: ExcalidrawElement) {

src/element/collision.ts

@@ -2,11 +2,13 @@ import { distanceBetweenPointAndSegment } from "../math";
 
 import { ExcalidrawElement } from "./types";
 import {
-  getArrowPoints,
   getDiamondPoints,
   getElementAbsoluteCoords,
   getLinePoints,
+  getArrowAbsoluteBounds,
 } from "./bounds";
+import { Point } from "roughjs/bin/geometry";
+import { Drawable, OpSet } from "roughjs/bin/core";
 
 function isElementDraggableFromInside(element: ExcalidrawElement): boolean {
   return element.backgroundColor !== "transparent" || element.isSelected;
@@ -145,18 +147,25 @@ export function hitTest(
         lineThreshold
     );
   } else if (element.type === "arrow") {
-    let [x1, y1, x2, y2, x3, y3, x4, y4] = getArrowPoints(element);
-    // The computation is done at the origin, we need to add a translation
-    x -= element.x;
-    y -= element.y;
+    if (!element.shape) {
+      return false;
+    }
+    const shape = element.shape as Drawable[];
+    // If shape does not consist of curve and two line segments
+    // for arrow shape, return false
+    if (shape.length < 3) return false;
 
+    const [x1, y1, x2, y2] = getArrowAbsoluteBounds(element);
+    if (x < x1 || y < y1 - 10 || x > x2 || y > y2 + 10) return false;
+
+    const relX = x - element.x;
+    const relY = y - element.y;
+
+    // hit test curve and lien segments for arrow
     return (
-      //    \
-      distanceBetweenPointAndSegment(x, y, x3, y3, x2, y2) < lineThreshold ||
-      // -----
-      distanceBetweenPointAndSegment(x, y, x1, y1, x2, y2) < lineThreshold ||
-      //    /
-      distanceBetweenPointAndSegment(x, y, x4, y4, x2, y2) < lineThreshold
+      hitTestRoughShape(shape[0].sets, relX, relY) ||
+      hitTestRoughShape(shape[1].sets, relX, relY) ||
+      hitTestRoughShape(shape[2].sets, relX, relY)
     );
   } else if (element.type === "line") {
     const [x1, y1, x2, y2] = getLinePoints(element);
@@ -176,3 +185,82 @@ export function hitTest(
     throw new Error("Unimplemented type " + element.type);
   }
 }
+
+const pointInBezierEquation = (
+  p0: Point,
+  p1: Point,
+  p2: Point,
+  p3: Point,
+  [mx, my]: Point,
+) => {
+  // B(t) = p0 * (1-t)^3 + 3p1 * t * (1-t)^2 + 3p2 * t^2 * (1-t) + p3 * t^3
+  const equation = (t: number, idx: number) =>
+    Math.pow(1 - t, 3) * p3[idx] +
+    3 * t * Math.pow(1 - t, 2) * p2[idx] +
+    3 * Math.pow(t, 2) * (1 - t) * p1[idx] +
+    p0[idx] * Math.pow(t, 3);
+
+  const epsilon = 20;
+  // go through t in increments of 0.01
+  let t = 0;
+  while (t <= 1.0) {
+    const tx = equation(t, 0);
+    const ty = equation(t, 1);
+
+    const diff = Math.sqrt(Math.pow(tx - mx, 2) + Math.pow(ty - my, 2));
+
+    if (diff < epsilon) {
+      return true;
+    }
+
+    t += 0.01;
+  }
+
+  return false;
+};
+
+const hitTestRoughShape = (opSet: OpSet[], x: number, y: number) => {
+  // read operations from first opSet
+  const ops = opSet[0].ops;
+
+  // set start position as (0,0) just in case
+  // move operation does not exist (unlikely but it is worth safekeeping it)
+  let currentP: Point = [0, 0];
+
+  return ops.some(({ op, data }, idx) => {
+    // There are only four operation types:
+    // move, bcurveTo, lineTo, and curveTo
+    if (op === "move") {
+      // change starting point
+      currentP = data as Point;
+      // move operation does not draw anything; so, it always
+      // returns false
+    } else if (op === "bcurveTo") {
+      // create points from bezier curve
+      // bezier curve stores data as a flattened array of three positions
+      // [x1, y1, x2, y2, x3, y3]
+      const p1 = [data[0], data[1]] as Point;
+      const p2 = [data[2], data[3]] as Point;
+      const p3 = [data[4], data[5]] as Point;
+
+      const p0 = currentP;
+      currentP = p3;
+
+      // check if points are on the curve
+      // cubic bezier curves require four parameters
+      // the first parameter is the last stored position (p0)
+      let retVal = pointInBezierEquation(p0, p1, p2, p3, [x, y]);
+
+      // set end point of bezier curve as the new starting point for
+      // upcoming operations as each operation is based on the last drawn
+      // position of the previous operation
+      return retVal;
+    } else if (op === "lineTo") {
+      // TODO: Implement this
+    } else if (op === "qcurveTo") {
+      // TODO: Implement this
+    }
+
+    return false;
+  });
+};

src/element/handlerRectangles.ts

@@ -1,5 +1,6 @@
 import { ExcalidrawElement } from "./types";
 import { SceneScroll } from "../scene/types";
+import { getArrowAbsoluteBounds } from "./bounds";
 
 type Sides = "n" | "s" | "w" | "e" | "nw" | "ne" | "sw" | "se";
 
@@ -7,18 +8,31 @@ export function handlerRectangles(
   element: ExcalidrawElement,
   { scrollX, scrollY }: SceneScroll,
 ) {
-  const elementX1 = element.x;
-  const elementX2 = element.x + element.width;
-  const elementY1 = element.y;
-  const elementY2 = element.y + element.height;
+  let elementX2 = 0;
+  let elementY2 = 0;
+  let elementX1 = Infinity;
+  let elementY1 = Infinity;
+  let marginX = -8;
+  let marginY = -8;
+
+  let minimumSize = 40;
+  if (element.type === "arrow") {
+    [elementX1, elementY1, elementX2, elementY2] = getArrowAbsoluteBounds(
+      element,
+    );
+  } else {
+    elementX1 = element.x;
+    elementX2 = element.x + element.width;
+    elementY1 = element.y;
+    elementY2 = element.y + element.height;
+
+    marginX = element.width < 0 ? 8 : -8;
+    marginY = element.height < 0 ? 8 : -8;
+  }
 
   const margin = 4;
-  const minimumSize = 40;
   const handlers = {} as { [T in Sides]: number[] };
 
-  const marginX = element.width < 0 ? 8 : -8;
-  const marginY = element.height < 0 ? 8 : -8;
-
   if (Math.abs(elementX2 - elementX1) > minimumSize) {
     handlers["n"] = [
       elementX1 + (elementX2 - elementX1) / 2 + scrollX - 4,
@@ -76,11 +90,58 @@ export function handlerRectangles(
     8,
   ]; // se
 
-  if (element.type === "arrow" || element.type === "line") {
+  if (element.type === "line") {
     return {
       nw: handlers.nw,
       se: handlers.se,
     } as typeof handlers;
+  } else if (element.type === "arrow") {
+    if (element.points.length === 2) {
+      // only check the last point because starting point is always (0,0)
+      const [, p1] = element.points;
+
+      if (p1[0] === 0 || p1[1] === 0) {
+        return {
+          nw: handlers.nw,
+          se: handlers.se,
+        } as typeof handlers;
+      }
+
+      if (p1[0] > 0 && p1[1] < 0) {
+        return {
+          ne: handlers.ne,
+          sw: handlers.sw,
+        } as typeof handlers;
+      }
+
+      if (p1[0] > 0 && p1[1] > 0) {
+        return {
+          nw: handlers.nw,
+          se: handlers.se,
+        } as typeof handlers;
+      }
+
+      if (p1[0] < 0 && p1[1] > 0) {
+        return {
+          ne: handlers.ne,
+          sw: handlers.sw,
+        } as typeof handlers;
+      }
+
+      if (p1[0] < 0 && p1[1] < 0) {
+        return {
+          nw: handlers.nw,
+          se: handlers.se,
+        } as typeof handlers;
+      }
+    }
+
+    return {
+      n: handlers.n,
+      s: handlers.s,
+      w: handlers.w,
+      e: handlers.e,
+    } as typeof handlers;
   }
 
   return handlers;

src/element/index.ts

@@ -5,6 +5,7 @@ export {
   getDiamondPoints,
   getArrowPoints,
   getLinePoints,
+  getArrowAbsoluteBounds,
 } from "./bounds";
 
 export { handlerRectangles } from "./handlerRectangles";

src/element/newElement.ts

@@ -1,6 +1,7 @@
 import { randomSeed } from "roughjs/bin/math";
 import nanoid from "nanoid";
 import { Drawable } from "roughjs/bin/core";
+import { Point } from "roughjs/bin/geometry";
 
 import { ExcalidrawElement, ExcalidrawTextElement } from "../element/types";
 import { measureText } from "../utils";
@@ -34,6 +35,7 @@ export function newElement(
     isSelected: false,
     seed: randomSeed(),
     shape: null as Drawable | Drawable[] | null,
+    points: [] as Point[],
   };
   return element;
 }

src/element/resizeTest.ts

@@ -17,6 +17,7 @@ export function resizeTest(
 
   const filter = Object.keys(handlers).filter(key => {
     const handler = handlers[key as HandlerRectanglesRet]!;
+    if (!handler) return false;
 
     return (
       x + scrollX >= handler[0] &&