From 98000be0e06ef4aba68c23dad6db9e1ff679c22d Mon Sep 17 00:00:00 2001
From: Ryan Di <ryan.weihao.di@gmail.com>
Date: Mon, 7 Apr 2025 23:57:14 +1000
Subject: [PATCH] nonzero enclosure

---
 packages/excalidraw/lasso/utils.ts |  5 ++++-
 packages/math/src/polygon.ts       | 30 ++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/packages/excalidraw/lasso/utils.ts b/packages/excalidraw/lasso/utils.ts
index 7f9d18996..9b4e8dcff 100644
--- a/packages/excalidraw/lasso/utils.ts
+++ b/packages/excalidraw/lasso/utils.ts
@@ -5,6 +5,7 @@ import {
   polygonIncludesPoint,
   lineSegment,
   lineSegmentIntersectionPoints,
+  polygonIncludesPointNonZero,
 } from "@excalidraw/math";
 
 import type { GlobalPoint, LineSegment } from "@excalidraw/math/types";
@@ -73,7 +74,9 @@ const enclosureTest = (
   }
 
   return segments.some((segment) => {
-    return segment.some((point) => polygonIncludesPoint(point, lassoPolygon));
+    return segment.some((point) =>
+      polygonIncludesPointNonZero(point, lassoPolygon),
+    );
   });
 };
 
diff --git a/packages/math/src/polygon.ts b/packages/math/src/polygon.ts
index 762c82dbf..7cc10e526 100644
--- a/packages/math/src/polygon.ts
+++ b/packages/math/src/polygon.ts
@@ -41,6 +41,36 @@ export const polygonIncludesPoint = <Point extends LocalPoint | GlobalPoint>(
   return inside;
 };
 
+export const polygonIncludesPointNonZero = <Point extends [number, number]>(
+  point: Point,
+  polygon: Point[],
+): boolean => {
+  const [x, y] = point;
+  let windingNumber = 0;
+
+  for (let i = 0; i < polygon.length; i++) {
+    const j = (i + 1) % polygon.length;
+    const [xi, yi] = polygon[i];
+    const [xj, yj] = polygon[j];
+
+    if (yi <= y) {
+      if (yj > y) {
+        if ((xj - xi) * (y - yi) - (x - xi) * (yj - yi) > 0) {
+          windingNumber++;
+        }
+      }
+    } else {
+      if (yj <= y) {
+        if ((xj - xi) * (y - yi) - (x - xi) * (yj - yi) < 0) {
+          windingNumber--;
+        }
+      }
+    }
+  }
+
+  return windingNumber !== 0;
+};
+
 export const pointOnPolygon = <Point extends LocalPoint | GlobalPoint>(
   p: Point,
   poly: Polygon<Point>,