basic Socket.io implementation of collaborative editing (#879)

* Enable collaborative syncing for elements

* Don't fall back to local storage if using a room, as that is confusing

* Use remote socket server

* Send updates to new users when they join

* ~

* add mouse tracking

* enable collaboration, rooms, and mouse tracking

* fix syncing bugs and add a button to start syncing mid session

* enable collaboration, rooms, and mouse tracking

* fix syncing bugs and add a button to start syncing mid session

* Add Live button and app state to support tracking collaborator counts

* Enable collaborative syncing for elements

* add mouse tracking

* enable collaboration, rooms, and mouse tracking

* fix syncing bugs and add a button to start syncing mid session

* fix syncing bugs and add a button to start syncing mid session

* Add Live button and app state to support tracking collaborator counts

* prettier

* Fix bug with remote pointers not changing on scroll

* Enable collaborative syncing for elements

* add mouse tracking

* enable collaboration, rooms, and mouse tracking

* fix syncing bugs and add a button to start syncing mid session

* enable collaboration, rooms, and mouse tracking

* fix syncing bugs and add a button to start syncing mid session

* Add Live button and app state to support tracking collaborator counts

* enable collaboration, rooms, and mouse tracking

* fix syncing bugs and add a button to start syncing mid session

* fix syncing bugs and add a button to start syncing mid session

* Fix bug with remote pointers not changing on scroll

* remove UI for collaboration

* remove link

* clean up lingering unused UI

* set random IV passed per encrypted message, reduce room id length, refactored socket broadcasting API, rename room_id to room, removed throttling of pointer movement

* fix package.json conflict
This commit is contained in:
Edwin Lin 2020-03-09 08:48:25 -07:00 committed by GitHub
parent 463854e42a
commit 0e5c29b3f3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 575 additions and 28 deletions

View file

@ -1,5 +1,6 @@
import { AppState, FlooredNumber } from "./types";
import { getDateTime } from "./utils";
import { getCollaborationLinkData } from "./data";
const DEFAULT_PROJECT_NAME = `excalidraw-${getDateTime()}`;
export const DEFAULT_FONT = "20px Virgil";
@ -27,12 +28,15 @@ export function getDefaultAppState(): AppState {
cursorY: 0,
scrolledOutside: false,
name: DEFAULT_PROJECT_NAME,
isCollaborating: !!getCollaborationLinkData(window.location.href),
isResizing: false,
selectionElement: null,
zoom: 1,
openMenu: null,
lastPointerDownWith: "mouse",
selectedElementIds: {},
remotePointers: {},
collaboratorCount: 0,
};
}

View file

@ -1,5 +1,6 @@
import React from "react";
import socketIOClient from "socket.io-client";
import rough from "roughjs/bin/rough";
import { RoughCanvas } from "roughjs/bin/canvas";
import { Point } from "roughjs/bin/geometry";
@ -29,7 +30,16 @@ import {
getSelectedElements,
isSomeElementSelected,
} from "../scene";
import { saveToLocalStorage, loadScene, loadFromBlob } from "../data";
import {
decryptAESGEM,
encryptAESGEM,
saveToLocalStorage,
loadScene,
loadFromBlob,
SOCKET_SERVER,
SocketUpdateData,
} from "../data";
import { restore } from "../data/restore";
import { renderScene } from "../renderer";
import { AppState, GestureEvent, Gesture } from "../types";
@ -77,6 +87,7 @@ import {
import { LayerUI } from "./LayerUI";
import { ScrollBars } from "../scene/types";
import { invalidateShapeForElement } from "../renderer/renderElement";
import { generateCollaborationLink, getCollaborationLinkData } from "../data";
// -----------------------------------------------------------------------------
// TEST HOOKS
@ -88,12 +99,15 @@ declare global {
elements: typeof elements;
appState: AppState;
};
// TEMPORARY until we have a UI to support this
generateCollaborationLink: () => Promise<string>;
}
}
if (process.env.NODE_ENV === "test") {
window.__TEST__ = {} as Window["__TEST__"];
}
window.generateCollaborationLink = generateCollaborationLink;
// -----------------------------------------------------------------------------
@ -136,6 +150,10 @@ function setCursorForShape(shape: string) {
export class App extends React.Component<any, AppState> {
canvas: HTMLCanvasElement | null = null;
rc: RoughCanvas | null = null;
socket: SocketIOClient.Socket | null = null;
socketInitialized: boolean = false; // we don't want the socket to emit any updates until it is fully initalized
roomID: string | null = null;
roomKey: string | null = null;
actionManager: ActionManager;
canvasOnlyActions = ["selectAll"];
@ -207,6 +225,96 @@ export class App extends React.Component<any, AppState> {
event.preventDefault();
};
private initializeSocketClient = () => {
if (this.socket) {
return;
}
const roomMatch = getCollaborationLinkData(window.location.href);
if (roomMatch) {
this.socket = socketIOClient(SOCKET_SERVER);
this.roomID = roomMatch[1];
this.roomKey = roomMatch[2];
this.socket.on("init-room", () => {
this.socket && this.socket.emit("join-room", this.roomID);
});
this.socket.on(
"client-broadcast",
async (encryptedData: ArrayBuffer, iv: Uint8Array) => {
if (!this.roomKey) {
return;
}
const decryptedData = await decryptAESGEM(
encryptedData,
this.roomKey,
iv,
);
switch (decryptedData.type) {
case "INVALID_RESPONSE":
return;
case "SCENE_UPDATE":
const {
elements: sceneElements,
appState: sceneAppState,
} = decryptedData.payload;
const restoredState = restore(
sceneElements || [],
sceneAppState || getDefaultAppState(),
{ scrollToContent: true },
);
elements = restoredState.elements;
this.setState({});
if (this.socketInitialized === false) {
this.socketInitialized = true;
}
break;
case "MOUSE_LOCATION":
const { socketID, pointerCoords } = decryptedData.payload;
this.setState({
remotePointers: {
...this.state.remotePointers,
[socketID]: pointerCoords,
},
});
break;
}
},
);
this.socket.on("first-in-room", () => {
if (this.socket) {
this.socket.off("first-in-room");
}
this.socketInitialized = true;
});
this.socket.on("room-user-count", (collaboratorCount: number) => {
this.setState({ collaboratorCount });
});
this.socket.on("new-user", async (socketID: string) => {
this.broadcastSocketData({
type: "SCENE_UPDATE",
payload: {
elements,
appState: this.state,
},
});
});
}
};
private broadcastSocketData = async (data: SocketUpdateData) => {
if (this.socketInitialized && this.socket && this.roomID && this.roomKey) {
const json = JSON.stringify(data);
const encoded = new TextEncoder().encode(json);
const encrypted = await encryptAESGEM(encoded, this.roomKey);
this.socket.emit(
"server-broadcast",
this.roomID,
encrypted.data,
encrypted.iv,
);
}
};
private unmounted = false;
public async componentDidMount() {
if (process.env.NODE_ENV === "test") {
@ -251,18 +359,24 @@ export class App extends React.Component<any, AppState> {
// Backwards compatibility with legacy url format
const scene = await loadScene(id);
this.syncActionResult(scene);
} else {
const match = window.location.hash.match(
/^#json=([0-9]+),([a-zA-Z0-9_-]+)$/,
);
if (match) {
const scene = await loadScene(match[1], match[2]);
this.syncActionResult(scene);
} else {
const scene = await loadScene(null);
this.syncActionResult(scene);
}
}
const jsonMatch = window.location.hash.match(
/^#json=([0-9]+),([a-zA-Z0-9_-]+)$/,
);
if (jsonMatch) {
const scene = await loadScene(jsonMatch[1], jsonMatch[2]);
this.syncActionResult(scene);
return;
}
const roomMatch = getCollaborationLinkData(window.location.href);
if (roomMatch) {
this.initializeSocketClient();
return;
}
const scene = await loadScene(null);
this.syncActionResult(scene);
}
public componentWillUnmount() {
@ -720,6 +834,12 @@ export class App extends React.Component<any, AppState> {
private handleCanvasPointerMove = (
event: React.PointerEvent<HTMLCanvasElement>,
) => {
const pointerCoords = viewportCoordsToSceneCoords(
event,
this.state,
this.canvas,
);
this.savePointer(pointerCoords);
gesture.pointers.set(event.pointerId, {
x: event.clientX,
y: event.clientY,
@ -1850,11 +1970,43 @@ export class App extends React.Component<any, AppState> {
}
}
private savePointer = (pointerCoords: { x: number; y: number }) => {
if (isNaN(pointerCoords.x) || isNaN(pointerCoords.y)) {
// sometimes the pointer goes off screen
return;
}
this.socket &&
this.broadcastSocketData({
type: "MOUSE_LOCATION",
payload: {
socketID: this.socket.id,
pointerCoords,
},
});
};
private saveDebounced = debounce(() => {
saveToLocalStorage(elements, this.state);
}, 300);
componentDidUpdate() {
if (this.state.isCollaborating && !this.socket) {
this.initializeSocketClient();
}
const pointerViewportCoords: {
[id: string]: { x: number; y: number };
} = {};
for (const clientId in this.state.remotePointers) {
const remotePointerCoord = this.state.remotePointers[clientId];
pointerViewportCoords[clientId] = sceneCoordsToViewportCoords(
{
sceneX: remotePointerCoord.x,
sceneY: remotePointerCoord.y,
},
this.state,
this.canvas,
);
}
const { atLeastOneVisibleElement, scrollBars } = renderScene(
elements,
this.state,
@ -1866,6 +2018,7 @@ export class App extends React.Component<any, AppState> {
scrollY: this.state.scrollY,
viewBackgroundColor: this.state.viewBackgroundColor,
zoom: this.state.zoom,
remotePointerViewportCoords: pointerViewportCoords,
},
{
renderOptimizations: true,
@ -1880,6 +2033,13 @@ export class App extends React.Component<any, AppState> {
}
this.saveDebounced();
if (history.isRecording()) {
this.broadcastSocketData({
type: "SCENE_UPDATE",
payload: {
elements,
appState: this.state,
},
});
history.pushEntry(this.state, elements);
history.skipRecording();
}

View file

@ -50,6 +50,12 @@ export const clipboard = createIcon(
512,
);
export const broadcast = createIcon(
"M150.94 192h33.73c11.01 0 18.61-10.83 14.86-21.18-4.93-13.58-7.55-27.98-7.55-42.82s2.62-29.24 7.55-42.82C203.29 74.83 195.68 64 184.67 64h-33.73c-7.01 0-13.46 4.49-15.41 11.23C130.64 92.21 128 109.88 128 128c0 18.12 2.64 35.79 7.54 52.76 1.94 6.74 8.39 11.24 15.4 11.24zM89.92 23.34C95.56 12.72 87.97 0 75.96 0H40.63c-6.27 0-12.14 3.59-14.74 9.31C9.4 45.54 0 85.65 0 128c0 24.75 3.12 68.33 26.69 118.86 2.62 5.63 8.42 9.14 14.61 9.14h34.84c12.02 0 19.61-12.74 13.95-23.37-49.78-93.32-16.71-178.15-.17-209.29zM614.06 9.29C611.46 3.58 605.6 0 599.33 0h-35.42c-11.98 0-19.66 12.66-14.02 23.25 18.27 34.29 48.42 119.42.28 209.23-5.72 10.68 1.8 23.52 13.91 23.52h35.23c6.27 0 12.13-3.58 14.73-9.29C630.57 210.48 640 170.36 640 128s-9.42-82.48-25.94-118.71zM489.06 64h-33.73c-11.01 0-18.61 10.83-14.86 21.18 4.93 13.58 7.55 27.98 7.55 42.82s-2.62 29.24-7.55 42.82c-3.76 10.35 3.85 21.18 14.86 21.18h33.73c7.02 0 13.46-4.49 15.41-11.24 4.9-16.97 7.53-34.64 7.53-52.76 0-18.12-2.64-35.79-7.54-52.76-1.94-6.75-8.39-11.24-15.4-11.24zm-116.3 100.12c7.05-10.29 11.2-22.71 11.2-36.12 0-35.35-28.63-64-63.96-64-35.32 0-63.96 28.65-63.96 64 0 13.41 4.15 25.83 11.2 36.12l-130.5 313.41c-3.4 8.15.46 17.52 8.61 20.92l29.51 12.31c8.15 3.4 17.52-.46 20.91-8.61L244.96 384h150.07l49.2 118.15c3.4 8.16 12.76 12.01 20.91 8.61l29.51-12.31c8.15-3.4 12-12.77 8.61-20.92l-130.5-313.41zM271.62 320L320 203.81 368.38 320h-96.76z",
640,
512,
);
export const trash = createIcon(
"M32 464a48 48 0 0 0 48 48h288a48 48 0 0 0 48-48V128H32zm272-256a16 16 0 0 1 32 0v224a16 16 0 0 1-32 0zm-96 0a16 16 0 0 1 32 0v224a16 16 0 0 1-32 0zm-96 0a16 16 0 0 1 32 0v224a16 16 0 0 1-32 0zM432 32H312l-9.4-18.7A24 24 0 0 0 281.1 0H166.8a23.72 23.72 0 0 0-21.4 13.3L136 32H16A16 16 0 0 0 0 48v32a16 16 0 0 0 16 16h416a16 16 0 0 0 16-16V48a16 16 0 0 0-16-16z",
448,

View file

@ -23,11 +23,145 @@ const BACKEND_GET = "https://json.excalidraw.com/api/v1/";
const BACKEND_V2_POST = "https://json.excalidraw.com/api/v2/post/";
const BACKEND_V2_GET = "https://json.excalidraw.com/api/v2/";
export const SOCKET_SERVER = "https://excalidraw-socket.herokuapp.com";
export type EncryptedData = {
data: ArrayBuffer;
iv: Uint8Array;
};
export type SocketUpdateData =
| {
type: "SCENE_UPDATE";
payload: {
elements: readonly ExcalidrawElement[];
appState: AppState | null;
};
}
| {
type: "MOUSE_LOCATION";
payload: {
socketID: string;
pointerCoords: { x: number; y: number };
};
}
| {
type: "INVALID_RESPONSE";
};
// TODO: Defined globally, since file handles aren't yet serializable.
// Once `FileSystemFileHandle` can be serialized, make this
// part of `AppState`.
(window as any).handle = null;
function byteToHex(byte: number): string {
return `0${byte.toString(16)}`.slice(-2);
}
async function generateRandomID() {
const arr = new Uint8Array(10);
window.crypto.getRandomValues(arr);
return Array.from(arr, byteToHex).join("");
}
async function generateEncryptionKey() {
const key = await window.crypto.subtle.generateKey(
{
name: "AES-GCM",
length: 128,
},
true, // extractable
["encrypt", "decrypt"],
);
return (await window.crypto.subtle.exportKey("jwk", key)).k;
}
function createIV() {
const arr = new Uint8Array(12);
return window.crypto.getRandomValues(arr);
}
export function getCollaborationLinkData(link: string) {
if (link.length === 0) {
return;
}
const hash = new URL(link).hash;
return hash.match(/^#room=([a-zA-Z0-9_-]+),([a-zA-Z0-9_-]+)$/);
}
export async function generateCollaborationLink() {
const id = await generateRandomID();
const key = await generateEncryptionKey();
return `${window.location.href}#room=${id},${key}`;
}
async function getImportedKey(key: string, usage: string): Promise<CryptoKey> {
return await window.crypto.subtle.importKey(
"jwk",
{
alg: "A128GCM",
ext: true,
k: key,
key_ops: ["encrypt", "decrypt"],
kty: "oct",
},
{
name: "AES-GCM",
length: 128,
},
false, // extractable
[usage],
);
}
export async function encryptAESGEM(
data: Uint8Array,
key: string,
): Promise<EncryptedData> {
const importedKey = await getImportedKey(key, "encrypt");
const iv = createIV();
return {
data: await window.crypto.subtle.encrypt(
{
name: "AES-GCM",
iv,
},
importedKey,
data,
),
iv,
};
}
export async function decryptAESGEM(
data: ArrayBuffer,
key: string,
iv: Uint8Array,
): Promise<SocketUpdateData> {
try {
const importedKey = await getImportedKey(key, "decrypt");
const decrypted = await window.crypto.subtle.decrypt(
{
name: "AES-GCM",
iv: iv,
},
importedKey,
data,
);
const decodedData = new TextDecoder("utf-8").decode(
new Uint8Array(decrypted) as any,
);
return JSON.parse(decodedData);
} catch (error) {
window.alert(t("alerts.decryptFailed"));
console.error(error);
}
return {
type: "INVALID_RESPONSE",
};
}
export async function exportToBackend(
elements: readonly ExcalidrawElement[],
appState: AppState,
@ -101,22 +235,7 @@ export async function importFromBackend(
let data;
if (privateKey) {
const buffer = await response.arrayBuffer();
const key = await window.crypto.subtle.importKey(
"jwk",
{
alg: "A128GCM",
ext: true,
k: privateKey,
key_ops: ["encrypt", "decrypt"],
kty: "oct",
},
{
name: "AES-GCM",
length: 128,
},
false, // extractable
["decrypt"],
);
const key = await getImportedKey(privateKey, "decrypt");
const iv = new Uint8Array(12);
const decrypted = await window.crypto.subtle.decrypt(
{

View file

@ -34,6 +34,9 @@ export function restoreFromLocalStorage() {
if (savedState) {
try {
appState = JSON.parse(savedState) as AppState;
// If we're retrieving from local storage, we should not be collaborating
appState.isCollaborating = false;
appState.collaboratorCount = 0;
} catch {
// Do nothing because appState is already null
}

View file

@ -70,6 +70,7 @@
"importBackendFailed": "Importing from backend failed.",
"cannotExportEmptyCanvas": "Cannot export empty canvas.",
"couldNotCopyToClipboard": "Couldn't copy to clipboard. Try using Chrome browser.",
"decryptFailed": "Couldn't decrypt data.",
"uploadedSecurly": "The upload has been secured with end-to-end encryption, which means that Excalidraw server and third parties can't read the content."
},
"toolBar": {

View file

@ -172,6 +172,15 @@ export function renderScene(
}
}
// Paint remote pointers
for (const clientId in sceneState.remotePointerViewportCoords) {
const { x, y } = sceneState.remotePointerViewportCoords[clientId];
context.beginPath();
context.arc(x, y, 5, 0, 2 * Math.PI);
context.fill();
context.stroke();
}
// Paint scrollbars
if (renderScrollbars) {
const scrollBars = getScrollBars(

View file

@ -49,6 +49,7 @@ export function exportToCanvas(
scrollX: normalizeScroll(-minX + exportPadding),
scrollY: normalizeScroll(-minY + exportPadding),
zoom: 1,
remotePointerViewportCoords: {},
},
{
renderScrollbars: false,

View file

@ -7,6 +7,7 @@ export type SceneState = {
// null indicates transparent bg
viewBackgroundColor: string | null;
zoom: number;
remotePointerViewportCoords: { [id: string]: { x: number; y: number } };
};
export type SceneScroll = {

View file

@ -29,11 +29,14 @@ export type AppState = {
scrolledOutside: boolean;
name: string;
selectedId?: string;
isCollaborating: boolean;
isResizing: boolean;
zoom: number;
openMenu: "canvas" | "shape" | null;
lastPointerDownWith: PointerType;
selectedElementIds: { [id: string]: boolean };
remotePointers: { [id: string]: { x: number; y: number } };
collaboratorCount: number;
};
export type PointerCoords = Readonly<{