smig/excalidraw
1
0
Fork 0
mirror of https://github.com/excalidraw/excalidraw.git synced 2025-05-03 10:00:07 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Offline support with increments peristed and restored to / from indexedb

Browse source
This commit is contained in:
Marcel Mraz 2024-12-12 14:41:20 +01:00
parent 15d2942aaa
commit 040a57f56a
No known key found for this signature in database
GPG key ID: 4EBD6E62DC830CD2
19 changed files with 1827 additions and 1104 deletions
Download patch file Download diff file Expand all files Collapse all files

5
packages/excalidraw/actions/actionExport.tsx
View file

@ -185,8 +185,9 @@ export const actionSaveToActiveFile = register({
return { storeAction: StoreAction.NONE };
}
},
keyTest: (event) =>
event.key === KEYS.S && event[KEYS.CTRL_OR_CMD] && !event.shiftKey,
// CFDO: temporary
// keyTest: (event) =>
// event.key === KEYS.S && event[KEYS.CTRL_OR_CMD] && !event.shiftKey,
});
export const actionSaveFileToDisk = register({

45
packages/excalidraw/change.ts
View file

@ -32,7 +32,6 @@ import type {
} from "./element/types";
import { orderByFractionalIndex, syncMovedIndices } from "./fractionalIndex";
import { getNonDeletedGroupIds } from "./groups";
import { randomId } from "./random";
import { getObservedAppState } from "./store";
import type {
AppState,
@ -40,7 +39,7 @@ import type {
ObservedElementsAppState,
ObservedStandaloneAppState,
} from "./types";
import type { SubtypeOf, ValueOf } from "./utility-types";
import type { DTO, SubtypeOf, ValueOf } from "./utility-types";
import {
arrayToMap,
arrayToObject,
@ -416,7 +415,7 @@ interface Change<T> {
}
export class AppStateChange implements Change<AppState> {
private constructor(private readonly delta: Delta<ObservedAppState>) {}
private constructor(public readonly delta: Delta<ObservedAppState>) {}
public static calculate<T extends ObservedAppState>(
prevAppState: T,
@ -432,6 +431,13 @@ export class AppStateChange implements Change<AppState> {
return new AppStateChange(delta);
}
public static restore(
appStateChangeDTO: DTO<AppStateChange>,
): AppStateChange {
const { delta } = appStateChangeDTO;
return new AppStateChange(delta);
}
public static empty() {
return new AppStateChange(Delta.create({}, {}));
}
@ -797,13 +803,13 @@ export class AppStateChange implements Change<AppState> {
}
}
// CFDO: consider adding here (nonnullable) version & versionNonce & updated & seed (so that we have correct versions when recunstructing from remote)
type ElementPartial<T extends ExcalidrawElement = ExcalidrawElement> = Omit<
ElementUpdate<Ordered<T>>,
"seed"
>;
type ElementsChangeOptions = {
id: string;
shouldRedistribute: boolean;
};
@ -813,10 +819,9 @@ type ElementsChangeOptions = {
*/
export class ElementsChange implements Change<SceneElementsMap> {
private constructor(
public readonly id: string,
private readonly added: Record<string, Delta<ElementPartial>>,
private readonly removed: Record<string, Delta<ElementPartial>>,
private readonly updated: Record<string, Delta<ElementPartial>>,
public readonly added: Record<string, Delta<ElementPartial>>,
public readonly removed: Record<string, Delta<ElementPartial>>,
public readonly updated: Record<string, Delta<ElementPartial>>,
) {}
public static create(
@ -824,11 +829,10 @@ export class ElementsChange implements Change<SceneElementsMap> {
removed: Record<string, Delta<ElementPartial>>,
updated: Record<string, Delta<ElementPartial>>,
options: ElementsChangeOptions = {
id: randomId(),
shouldRedistribute: false,
},
) {
const { id, shouldRedistribute } = options;
const { shouldRedistribute } = options;
let change: ElementsChange;
if (shouldRedistribute) {
@ -852,9 +856,9 @@ export class ElementsChange implements Change<SceneElementsMap> {
}
}
change = new ElementsChange(id, nextAdded, nextRemoved, nextUpdated);
change = new ElementsChange(nextAdded, nextRemoved, nextUpdated);
} else {
change = new ElementsChange(id, added, removed, updated);
change = new ElementsChange(added, removed, updated);
}
if (import.meta.env.DEV || import.meta.env.MODE === ENV.TEST) {
@ -866,6 +870,13 @@ export class ElementsChange implements Change<SceneElementsMap> {
return change;
}
public static restore(
elementsChangeDTO: DTO<ElementsChange>,
): ElementsChange {
const { added, removed, updated } = elementsChangeDTO;
return ElementsChange.create(added, removed, updated);
}
private static satisfiesAddition = ({
deleted,
inserted,
@ -997,15 +1008,6 @@ export class ElementsChange implements Change<SceneElementsMap> {
return ElementsChange.create({}, {}, {});
}
public static load(payload: string) {
const { id, added, removed, updated } = JSON.parse(payload);
return ElementsChange.create(added, removed, updated, {
id,
shouldRedistribute: false,
});
}
public inverse(): ElementsChange {
const inverseInternal = (deltas: Record<string, Delta<ElementPartial>>) => {
const inversedDeltas: Record<string, Delta<ElementPartial>> = {};
@ -1091,7 +1093,6 @@ export class ElementsChange implements Change<SceneElementsMap> {
const updated = applyLatestChangesInternal(this.updated);
return ElementsChange.create(added, removed, updated, {
id: this.id,
shouldRedistribute: true, // redistribute the deltas as `isDeleted` could have been updated
});
}

66
packages/excalidraw/cloudflare/changes.ts
View file

@ -1,66 +0,0 @@
import type {
ChangesRepository,
CLIENT_CHANGE,
SERVER_CHANGE,
} from "../sync/protocol";
// CFDO: add senderId, possibly roomId as well
export class DurableChangesRepository implements ChangesRepository {
constructor(private storage: DurableObjectStorage) {
// #region DEV ONLY
// this.storage.sql.exec(`DROP TABLE IF EXISTS changes;`);
// #endregion
this.storage.sql.exec(`CREATE TABLE IF NOT EXISTS changes(
id TEXT PRIMARY KEY,
payload TEXT NOT NULL,
version INTEGER NOT NULL DEFAULT 1,
createdAt TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);`);
}
public saveAll = (changes: Array<CLIENT_CHANGE>) => {
return this.storage.transactionSync(() => {
const prevVersion = this.getLastVersion();
const nextVersion = prevVersion + changes.length;
// CFDO: in theory payload could contain array of changes, if we would need to optimize writes
for (const [index, change] of changes.entries()) {
const version = prevVersion + index + 1;
// unique id ensures that we don't acknowledge the same change twice
this.storage.sql.exec(
`INSERT INTO changes (id, payload, version) VALUES (?, ?, ?);`,
change.id,
JSON.stringify(change),
version,
);
}
// sanity check
if (nextVersion !== this.getLastVersion()) {
throw new Error(
`Expected last acknowledged version to be "${nextVersion}", but it is "${this.getLastVersion()}!"`,
);
}
return this.getSinceVersion(prevVersion);
});
};
public getSinceVersion = (version: number): Array<SERVER_CHANGE> => {
return this.storage.sql
.exec<SERVER_CHANGE>(
`SELECT id, payload, version FROM changes WHERE version > (?) ORDER BY version ASC;`,
version,
)
.toArray();
};
public getLastVersion = (): number => {
const result = this.storage.sql
.exec(`SELECT MAX(version) FROM changes;`)
.one();
return result ? Number(result["MAX(version)"]) : 0;
};
}

88
packages/excalidraw/cloudflare/repository.ts Normal file
View file

@ -0,0 +1,88 @@
import type {
IncrementsRepository,
CLIENT_INCREMENT,
SERVER_INCREMENT,
} from "../sync/protocol";
// CFDO: add senderId, possibly roomId as well
export class DurableIncrementsRepository implements IncrementsRepository {
constructor(private storage: DurableObjectStorage) {
// #region DEV ONLY
this.storage.sql.exec(`DROP TABLE IF EXISTS increments;`);
// #endregion
this.storage.sql.exec(`CREATE TABLE IF NOT EXISTS increments(
version INTEGER PRIMARY KEY AUTOINCREMENT,
id TEXT NOT NULL UNIQUE,
createdAt TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
payload TEXT
);`);
}
public saveAll(increments: Array<CLIENT_INCREMENT>) {
return this.storage.transactionSync(() => {
const prevVersion = this.getLastVersion();
const acknowledged: Array<SERVER_INCREMENT> = [];
for (const increment of increments) {
try {
// unique id ensures that we don't acknowledge the same increment twice
this.storage.sql.exec(
`INSERT INTO increments (id, payload) VALUES (?, ?);`,
increment.id,
JSON.stringify(increment),
);
} catch (e) {
// check if the increment has been already acknowledged
// in case client for some reason did not receive acknowledgement
// and reconnected while the we still have the increment in the worker
// otherwise the client is doomed to full a restore
if (
e instanceof Error &&
e.message.includes(
"UNIQUE constraint failed: increments.id: SQLITE_CONSTRAINT",
)
) {
acknowledged.push(this.getById(increment.id));
continue;
}
throw e;
}
}
// query the just added increments
acknowledged.push(...this.getSinceVersion(prevVersion));
return acknowledged;
});
}
public getSinceVersion(version: number): Array<SERVER_INCREMENT> {
// CFDO: for versioning we need deletions, but not for the "snapshot" update;
return this.storage.sql
.exec<SERVER_INCREMENT>(
`SELECT id, payload, version FROM increments WHERE version > (?) ORDER BY version, createdAt ASC;`,
version,
)
.toArray();
}
public getLastVersion(): number {
// CFDO: might be in memory to reduce number of rows read (or index on version at least, if btree affect rows read)
const result = this.storage.sql
.exec(`SELECT MAX(version) FROM increments;`)
.one();
return result ? Number(result["MAX(version)"]) : 0;
}
public getById(id: string): SERVER_INCREMENT {
return this.storage.sql
.exec<SERVER_INCREMENT>(
`SELECT id, payload, version FROM increments WHERE id = (?)`,
id,
)
.one();
}
}

4
packages/excalidraw/cloudflare/room.ts
View file

@ -1,5 +1,5 @@
import { DurableObject } from "cloudflare:workers";
import { DurableChangesRepository } from "./changes";
import { DurableIncrementsRepository } from "./repository";
import { ExcalidrawSyncServer } from "../sync/server";
import type { ExcalidrawElement } from "../element/types";
@ -35,7 +35,7 @@ export class DurableRoom extends DurableObject {
});
this.sync = new ExcalidrawSyncServer(
new DurableChangesRepository(ctx.storage),
new DurableIncrementsRepository(ctx.storage),
);
// in case it hibernates, let's get take active connections

2
packages/excalidraw/history.ts
View file

@ -152,7 +152,7 @@ export class History {
entry: HistoryEntry,
prevElements: SceneElementsMap,
) {
const updatedEntry = entry.inverse().applyLatestChanges(prevElements);
const updatedEntry = entry.applyLatestChanges(prevElements);
return stack.push(updatedEntry);
}
}

2
packages/excalidraw/package.json
View file

@ -74,6 +74,7 @@
"image-blob-reduce": "3.0.1",
"jotai": "2.11.0",
"jotai-scope": "0.7.2",
"lodash.debounce": "4.0.8",
"lodash.throttle": "4.1.1",
"nanoid": "3.3.3",
"open-color": "1.9.1",
@ -104,6 +105,7 @@
"@testing-library/jest-dom": "5.16.2",
"@testing-library/react": "16.0.0",
"@types/async-lock": "^1.4.2",
"@types/lodash.debounce": "4.0.9",
"@types/pako": "1.0.3",
"@types/pica": "5.1.3",
"@types/resize-observer-browser": "0.1.7",

124
packages/excalidraw/store.ts
View file

@ -1,16 +1,17 @@
import { ENV } from "./constants";
import { Emitter } from "./emitter";
import { randomId } from "./random";
import { isShallowEqual } from "./utils";
import { getDefaultAppState } from "./appState";
import { AppStateChange, ElementsChange } from "./change";
import { ENV } from "./constants";
import { newElementWith } from "./element/mutateElement";
import { deepCopyElement } from "./element/newElement";
import type { AppState, ObservedAppState } from "./types";
import type { DTO, ValueOf } from "./utility-types";
import type {
OrderedExcalidrawElement,
SceneElementsMap,
} from "./element/types";
import { Emitter } from "./emitter";
import type { AppState, ObservedAppState } from "./types";
import type { ValueOf } from "./utility-types";
import { isShallowEqual } from "./utils";
// hidden non-enumerable property for runtime checks
const hiddenObservedAppStateProp = "__observedAppState";
@ -96,31 +97,31 @@ export class Store {
* Use to schedule calculation of a store increment.
*/
// TODO: Suspicious that this is called so many places. Seems error-prone.
public shouldCaptureIncrement = () => {
public shouldCaptureIncrement() {
this.scheduleAction(StoreAction.CAPTURE);
};
}
/**
* Use to schedule update of the snapshot, useful on updates for which we don't need to calculate increments (i.e. remote updates).
*/
public shouldUpdateSnapshot = () => {
public shouldUpdateSnapshot() {
this.scheduleAction(StoreAction.UPDATE);
};
}
private scheduleAction = (action: StoreActionType) => {
private scheduleAction(action: StoreActionType) {
this.scheduledActions.add(action);
this.satisfiesScheduledActionsInvariant();
};
}
/**
* Based on the scheduled operation, either only updates store snapshot or also calculates increment and emits the result as a `StoreIncrement`.
*
* @emits StoreIncrement when increment is calculated.
*/
public commit = (
public commit(
elements: Map<string, OrderedExcalidrawElement> | undefined,
appState: AppState | ObservedAppState | undefined,
): void => {
): void {
try {
// Capture has precedence since it also performs update
if (this.scheduledActions.has(StoreAction.CAPTURE)) {
@ -133,17 +134,17 @@ export class Store {
// Defensively reset all scheduled actions, potentially cleans up other runtime garbage
this.scheduledActions = new Set();
}
};
}
/**
* Performs diff calculation, calculates and emits the increment.
*
* @emits StoreIncrement when increment is calculated.
*/
public captureIncrement = (
public captureIncrement(
elements: Map<string, OrderedExcalidrawElement> | undefined,
appState: AppState | ObservedAppState | undefined,
) => {
) {
const prevSnapshot = this.snapshot;
const nextSnapshot = this.snapshot.maybeClone(elements, appState);
@ -161,39 +162,39 @@ export class Store {
if (!elementsChange.isEmpty() || !appStateChange.isEmpty()) {
// Notify listeners with the increment
this.onStoreIncrementEmitter.trigger(
new StoreIncrement(elementsChange, appStateChange),
StoreIncrement.create(elementsChange, appStateChange),
);
}
// Update snapshot
this.snapshot = nextSnapshot;
}
};
}
/**
* Updates the snapshot without performing any diff calculation.
*/
public updateSnapshot = (
public updateSnapshot(
elements: Map<string, OrderedExcalidrawElement> | undefined,
appState: AppState | ObservedAppState | undefined,
) => {
) {
const nextSnapshot = this.snapshot.maybeClone(elements, appState);
if (this.snapshot !== nextSnapshot) {
// Update snapshot
this.snapshot = nextSnapshot;
}
};
}
/**
* Filters out yet uncomitted elements from `nextElements`, which are part of in-progress local async actions (ephemerals) and thus were not yet commited to the snapshot.
*
* This is necessary in updates in which we receive reconciled elements, already containing elements which were not yet captured by the local store (i.e. collab).
*/
public filterUncomittedElements = (
public filterUncomittedElements(
prevElements: Map<string, OrderedExcalidrawElement>,
nextElements: Map<string, OrderedExcalidrawElement>,
) => {
) {
for (const [id, prevElement] of prevElements.entries()) {
const nextElement = nextElements.get(id);
@ -215,18 +216,18 @@ export class Store {
}
return nextElements;
};
}
/**
* Apply and emit increment.
*
* @emits StoreIncrement when increment is applied.
*/
public applyIncrementTo = (
public applyIncrementTo(
increment: StoreIncrement,
elements: SceneElementsMap,
appState: AppState,
): [SceneElementsMap, AppState, boolean] => {
): [SceneElementsMap, AppState, boolean] {
const [nextElements, elementsContainVisibleChange] =
increment.elementsChange.applyTo(elements, this.snapshot.elements);
@ -239,17 +240,17 @@ export class Store {
this.onStoreIncrementEmitter.trigger(increment);
return [nextElements, nextAppState, appliedVisibleChanges];
};
}
/**
* Clears the store instance.
*/
public clear = (): void => {
public clear(): void {
this.snapshot = StoreSnapshot.empty();
this.scheduledActions = new Set();
};
}
private satisfiesScheduledActionsInvariant = () => {
private satisfiesScheduledActionsInvariant() {
if (!(this.scheduledActions.size >= 0 && this.scheduledActions.size <= 3)) {
const message = `There can be at most three store actions scheduled at the same time, but there are "${this.scheduledActions.size}".`;
console.error(message, this.scheduledActions.values());
@ -258,20 +259,70 @@ export class Store {
throw new Error(message);
}
}
};
}
}
/**
* Represent an increment to the Store.
*/
export class StoreIncrement {
constructor(
private constructor(
public readonly id: string,
public readonly elementsChange: ElementsChange,
public readonly appStateChange: AppStateChange,
) {}
/**
* Create a new instance of `StoreIncrement`.
*/
public static create(
elementsChange: ElementsChange,
appStateChange: AppStateChange,
opts: {
id: string;
} = {
id: randomId(),
},
) {
return new StoreIncrement(opts.id, elementsChange, appStateChange);
}
/**
* Restore a store increment instance from a DTO.
*/
public static restore(storeIncrementDTO: DTO<StoreIncrement>) {
const { id, elementsChange, appStateChange } = storeIncrementDTO;
return new StoreIncrement(
id,
ElementsChange.restore(elementsChange),
AppStateChange.restore(appStateChange),
);
}
// CFDO: why it would be a string if it can be a DTO?
/**
* Parse and load the increment from the remote payload.
*/
public static load(payload: string) {
// CFDO: ensure typesafety
const {
id,
elementsChange: { added, removed, updated },
} = JSON.parse(payload);
const elementsChange = ElementsChange.create(added, removed, updated, {
shouldRedistribute: false,
});
return new StoreIncrement(id, elementsChange, AppStateChange.empty());
}
/**
* Inverse store increment, creates new instance of `StoreIncrement`.
*/
public inverse(): StoreIncrement {
return new StoreIncrement(
randomId(),
this.elementsChange.inverse(),
this.appStateChange.inverse(),
);
@ -281,10 +332,13 @@ export class StoreIncrement {
* Apply latest (remote) changes to the increment, creates new instance of `StoreIncrement`.
*/
public applyLatestChanges(elements: SceneElementsMap): StoreIncrement {
const updatedElementsChange =
this.elementsChange.applyLatestChanges(elements);
const inversedIncrement = this.inverse();
return new StoreIncrement(updatedElementsChange, this.appStateChange);
return new StoreIncrement(
inversedIncrement.id,
inversedIncrement.elementsChange.applyLatestChanges(elements),
inversedIncrement.appStateChange,
);
}
public isEmpty() {

308
packages/excalidraw/sync/client.ts
View file

@ -1,12 +1,90 @@
/* eslint-disable no-console */
import throttle from "lodash.throttle";
import debounce from "lodash.debounce";
import { Utils } from "./utils";
import { ElementsChange } from "../change";
import { StoreIncrement } from "../store";
import type { ExcalidrawImperativeAPI } from "../types";
import type { SceneElementsMap } from "../element/types";
import type { CLIENT_CHANGE, PUSH_PAYLOAD, SERVER_CHANGE } from "./protocol";
import throttle from "lodash.throttle";
import type {
CLIENT_INCREMENT,
PUSH_PAYLOAD,
SERVER_INCREMENT,
} from "./protocol";
export class ExcalidrawSyncClient {
interface IncrementsRepository {
loadIncrements(): Promise<{ increments: Array<StoreIncrement> } | null>;
saveIncrements(params: { increments: Array<StoreIncrement> }): Promise<void>;
}
interface MetadataRepository {
loadMetadata(): Promise<{ lastAcknowledgedVersion: number } | null>;
saveMetadata(metadata: { lastAcknowledgedVersion: number }): Promise<void>;
}
// CFDO: make sure the increments are always acknowledged (deleted from the repository)
export class SyncQueue {
private readonly queue: Map<string, StoreIncrement>;
private readonly repository: IncrementsRepository;
private constructor(
queue: Map<string, StoreIncrement> = new Map(),
repository: IncrementsRepository,
) {
this.queue = queue;
this.repository = repository;
}
public static async create(repository: IncrementsRepository) {
const data = await repository.loadIncrements();
return new SyncQueue(
new Map(data?.increments?.map((increment) => [increment.id, increment])),
repository,
);
}
public getAll() {
return Array.from(this.queue.values());
}
public get(id: StoreIncrement["id"]) {
return this.queue.get(id);
}
public has(id: StoreIncrement["id"]) {
return this.queue.has(id);
}
public add(...increments: StoreIncrement[]) {
for (const increment of increments) {
this.queue.set(increment.id, increment);
}
this.persist();
}
public remove(...ids: StoreIncrement["id"][]) {
for (const id of ids) {
this.queue.delete(id);
}
this.persist();
}
public persist = throttle(
async () => {
try {
await this.repository.saveIncrements({ increments: this.getAll() });
} catch (e) {
console.error("Failed to persist the sync queue:", e);
}
},
1000,
{ leading: false, trailing: true },
);
}
export class SyncClient {
private static readonly HOST_URL = import.meta.env.DEV
? "ws://localhost:8787"
: "https://excalidraw-sync.marcel-529.workers.dev";
@ -17,18 +95,29 @@ export class ExcalidrawSyncClient {
private static readonly RECONNECT_INTERVAL = 10_000;
private lastAcknowledgedVersion = 0;
private readonly api: ExcalidrawImperativeAPI;
private readonly roomId: string;
private readonly queuedChanges: Map<
string,
{ queuedAt: number; change: CLIENT_CHANGE }
> = new Map();
public readonly acknowledgedChanges: Array<ElementsChange> = [];
private readonly queue: SyncQueue;
private readonly repository: MetadataRepository;
private get localChanges() {
return Array.from(this.queuedChanges.values()).map(({ change }) => change);
// CFDO: shouldn't be stateful, only request / response
private readonly acknowledgedIncrementsMap: Map<string, StoreIncrement> =
new Map();
public get acknowledgedIncrements() {
return Array.from(this.acknowledgedIncrementsMap.values());
}
private readonly roomId: string;
private _lastAcknowledgedVersion = 0;
private get lastAcknowledgedVersion() {
return this._lastAcknowledgedVersion;
}
private set lastAcknowledgedVersion(version: number) {
this._lastAcknowledgedVersion = version;
this.repository.saveMetadata({ lastAcknowledgedVersion: version });
}
private server: WebSocket | null = null;
@ -38,15 +127,33 @@ export class ExcalidrawSyncClient {
private isConnecting: { done: (error?: Error) => void } | null = null;
constructor(
private constructor(
api: ExcalidrawImperativeAPI,
roomId: string = ExcalidrawSyncClient.ROOM_ID,
repository: MetadataRepository,
queue: SyncQueue,
options: { roomId: string; lastAcknowledgedVersion: number },
) {
this.api = api;
this.roomId = roomId;
this.repository = repository;
this.queue = queue;
this.roomId = options.roomId;
this.lastAcknowledgedVersion = options.lastAcknowledgedVersion;
}
// CFDO: persist in idb
this.lastAcknowledgedVersion = 0;
public static async create(
api: ExcalidrawImperativeAPI,
repository: IncrementsRepository & MetadataRepository,
roomId: string = SyncClient.ROOM_ID,
) {
const [queue, metadata] = await Promise.all([
SyncQueue.create(repository),
repository.loadMetadata(),
]);
return new SyncClient(api, repository, queue, {
roomId,
lastAcknowledgedVersion: metadata?.lastAcknowledgedVersion ?? 0,
});
}
// CFDO: throttle does not work that well here (after some period it tries to reconnect too often)
@ -74,7 +181,7 @@ export class ExcalidrawSyncClient {
return await new Promise<void>((resolve, reject) => {
this.server = new WebSocket(
`${ExcalidrawSyncClient.HOST_URL}/connect?roomId=${this.roomId}`,
`${SyncClient.HOST_URL}/connect?roomId=${this.roomId}`,
);
// wait for 10 seconds before timing out
@ -103,7 +210,7 @@ export class ExcalidrawSyncClient {
this.disconnect(e as Error);
}
},
ExcalidrawSyncClient.RECONNECT_INTERVAL,
SyncClient.RECONNECT_INTERVAL,
{ leading: true },
);
@ -125,7 +232,7 @@ export class ExcalidrawSyncClient {
this.reconnect();
}
},
ExcalidrawSyncClient.RECONNECT_INTERVAL,
SyncClient.RECONNECT_INTERVAL,
{ leading: true },
);
@ -145,8 +252,8 @@ export class ExcalidrawSyncClient {
// resolve the current connection
this.isConnecting.done();
// initiate pull
this.pull();
// CFDO: hack to pull everything for on init
this.pull(0);
};
private onClose = (event: CloseEvent) => {
@ -185,43 +292,36 @@ export class ExcalidrawSyncClient {
}
};
private pull = (): void => {
private pull(sinceVersion?: number): void {
this.send({
type: "pull",
payload: {
lastAcknowledgedVersion: this.lastAcknowledgedVersion,
lastAcknowledgedVersion: sinceVersion ?? this.lastAcknowledgedVersion,
},
});
};
}
public push = (
public push(
type: "durable" | "ephemeral" = "durable",
changes: Array<CLIENT_CHANGE> = [],
): void => {
const payload: PUSH_PAYLOAD = { type, changes: [] };
...increments: Array<CLIENT_INCREMENT>
): void {
const payload: PUSH_PAYLOAD = { type, increments: [] };
if (type === "durable") {
// CFDO: persist in idb (with insertion order)
for (const change of changes) {
this.queuedChanges.set(change.id, {
queuedAt: Date.now(),
change,
});
}
// batch all queued changes
payload.changes = this.localChanges;
this.queue.add(...increments);
// batch all (already) queued increments
payload.increments = this.queue.getAll();
} else {
payload.changes = changes;
payload.increments = increments;
}
if (payload.changes.length > 0) {
if (payload.increments.length > 0) {
this.send({
type: "push",
payload,
});
}
};
}
public relay(buffer: ArrayBuffer): void {
this.send({
@ -230,60 +330,67 @@ export class ExcalidrawSyncClient {
});
}
// CFDO: refactor by applying all operations to store, not to the elements
private handleAcknowledged(payload: { changes: Array<SERVER_CHANGE> }) {
const { changes: remoteChanges } = payload;
// CFDO: should be flushed once regular push / pull goes through
private debouncedPush = (ms: number = 1000) =>
debounce(this.push, ms, { leading: true, trailing: false });
const oldAcknowledgedVersion = this.lastAcknowledgedVersion;
private debouncedPull = (ms: number = 1000) =>
debounce(this.pull, ms, { leading: true, trailing: false });
// CFDO: refactor by applying all operations to store, not to the elements
private handleAcknowledged(payload: { increments: Array<SERVER_INCREMENT> }) {
let nextAcknowledgedVersion = this.lastAcknowledgedVersion;
let elements = new Map(
// CFDO: retrieve the map already
this.api.getSceneElementsIncludingDeleted().map((el) => [el.id, el]),
) as SceneElementsMap;
try {
// apply remote changes
for (const remoteChange of remoteChanges) {
if (this.queuedChanges.has(remoteChange.id)) {
const { change, queuedAt } = this.queuedChanges.get(remoteChange.id)!;
this.acknowledgedChanges.push(change);
console.info(
`Acknowledged change "${remoteChange.id}" after ${
Date.now() - queuedAt
}ms`,
);
// local change acknowledge by the server, safe to remove
this.queuedChanges.delete(remoteChange.id);
} else {
// CFDO: we might not need to be that strict here
if (this.lastAcknowledgedVersion + 1 !== remoteChange.version) {
throw new Error(
`Received out of order change, expected "${
this.lastAcknowledgedVersion + 1
}", but received "${remoteChange.version}"`,
);
}
const { increments: remoteIncrements } = payload;
const change = ElementsChange.load(remoteChange.payload);
[elements] = change.applyTo(
elements,
this.api.store.snapshot.elements,
);
this.acknowledgedChanges.push(change);
// apply remote increments
for (const { id, version, payload } of remoteIncrements.sort((a, b) =>
a.version <= b.version ? -1 : 1,
)) {
// CFDO: temporary to load all increments on init
this.acknowledgedIncrementsMap.set(id, StoreIncrement.load(payload));
// local increment shall not have to be applied again
if (this.queue.has(id)) {
this.queue.remove(id);
continue;
}
this.lastAcknowledgedVersion = remoteChange.version;
// we've already applied this increment
if (version <= nextAcknowledgedVersion) {
continue;
}
if (version === nextAcknowledgedVersion + 1) {
nextAcknowledgedVersion = version;
} else {
// it's fine to apply increments our of order,
// as they are idempontent, so that we can re-apply them again,
// as long as we don't mark them as acknowledged
console.debug(
`Received out of order increment, expected "${
nextAcknowledgedVersion + 1
}", but received "${version}"`,
);
}
// apply remote increment with higher version than the last acknowledged one
const remoteIncrement = StoreIncrement.load(payload);
[elements] = remoteIncrement.elementsChange.applyTo(
elements,
this.api.store.snapshot.elements,
);
}
console.debug(`${now()} remote changes`, remoteChanges);
console.debug(`${now()} local changes`, this.localChanges);
console.debug(
`${now()} acknowledged changes`,
this.acknowledgedChanges.slice(-remoteChanges.length),
);
// apply local changes
// CFDO: only necessary when remote changes modified same element properties!
for (const localChange of this.localChanges) {
[elements] = localChange.applyTo(
// apply local increments
for (const localIncrement of this.queue.getAll()) {
// CFDO: in theory only necessary when remote increments modified same element properties!
[elements] = localIncrement.elementsChange.applyTo(
elements,
this.api.store.snapshot.elements,
);
@ -294,38 +401,31 @@ export class ExcalidrawSyncClient {
storeAction: "update",
});
// push all queued changes
this.push();
this.lastAcknowledgedVersion = nextAcknowledgedVersion;
} catch (e) {
console.error("Failed to apply acknowledged changes:", e);
// rollback the last acknowledged version
this.lastAcknowledgedVersion = oldAcknowledgedVersion;
// pull again to get the latest changes
this.pull();
console.error("Failed to apply acknowledged increments:", e);
this.debouncedPull().call(this);
return;
}
this.debouncedPush().call(this);
}
private handleRejected(payload: { ids: Array<string>; message: string }) {
// handle rejected changes
// handle rejected increments
console.error("Rejected message received:", payload);
}
private handleRelayed(payload: { changes: Array<CLIENT_CHANGE> }) {
// apply relayed changes / buffer
private handleRelayed(payload: { increments: Array<CLIENT_INCREMENT> }) {
// apply relayed increments / buffer
console.log("Relayed message received:", payload);
}
private send(message: { type: string; payload: any }): void {
if (!this.isConnected) {
console.error("Can't send a message without an active connection!");
return;
throw new Error("Can't send a message without an active connection!");
}
this.server?.send(JSON.stringify(message));
}
}
const now = () => {
const date = new Date();
return `[${date.getHours()}:${date.getMinutes()}:${date.getSeconds()}.${date.getMilliseconds()}]`;
};

20
packages/excalidraw/sync/protocol.ts
View file

@ -1,34 +1,34 @@
import type { ElementsChange } from "../change";
import type { StoreIncrement } from "../store";
export type RELAY_PAYLOAD = { buffer: ArrayBuffer };
export type PULL_PAYLOAD = { lastAcknowledgedVersion: number };
export type PUSH_PAYLOAD = {
type: "durable" | "ephemeral";
changes: Array<CLIENT_CHANGE>;
increments: Array<CLIENT_INCREMENT>;
};
export type CLIENT_CHANGE = ElementsChange;
export type CLIENT_INCREMENT = StoreIncrement;
export type CLIENT_MESSAGE =
| { type: "relay"; payload: RELAY_PAYLOAD }
| { type: "pull"; payload: PULL_PAYLOAD }
| { type: "push"; payload: PUSH_PAYLOAD };
export type SERVER_CHANGE = { id: string; version: number; payload: string };
export type SERVER_INCREMENT = { id: string; version: number; payload: string };
export type SERVER_MESSAGE =
| {
type: "relayed";
payload: { changes: Array<CLIENT_CHANGE> } | RELAY_PAYLOAD;
payload: { increments: Array<CLIENT_INCREMENT> } | RELAY_PAYLOAD;
}
| { type: "acknowledged"; payload: { changes: Array<SERVER_CHANGE> } }
| { type: "acknowledged"; payload: { increments: Array<SERVER_INCREMENT> } }
| {
type: "rejected";
payload: { changes: Array<CLIENT_CHANGE>; message: string };
payload: { increments: Array<CLIENT_INCREMENT>; message: string };
};
export interface ChangesRepository {
saveAll(changes: Array<CLIENT_CHANGE>): Array<SERVER_CHANGE>;
getSinceVersion(version: number): Array<SERVER_CHANGE>;
export interface IncrementsRepository {
saveAll(increments: Array<CLIENT_INCREMENT>): Array<SERVER_INCREMENT>;
getSinceVersion(version: number): Array<SERVER_INCREMENT>;
getLastVersion(): number;
}

55
packages/excalidraw/sync/server.ts
View file

@ -2,13 +2,14 @@ import AsyncLock from "async-lock";
import { Utils } from "./utils";
import type {
ChangesRepository,
CLIENT_CHANGE,
IncrementsRepository,
CLIENT_INCREMENT,
CLIENT_MESSAGE,
PULL_PAYLOAD,
PUSH_PAYLOAD,
RELAY_PAYLOAD,
SERVER_MESSAGE,
SERVER_INCREMENT,
} from "./protocol";
// CFDO: message could be binary (cbor, protobuf, etc.)
@ -20,7 +21,7 @@ export class ExcalidrawSyncServer {
private readonly lock: AsyncLock = new AsyncLock();
private readonly sessions: Set<WebSocket> = new Set();
constructor(private readonly changesRepository: ChangesRepository) {}
constructor(private readonly incrementsRepository: IncrementsRepository) {}
public onConnect(client: WebSocket) {
this.sessions.add(client);
@ -59,16 +60,11 @@ export class ExcalidrawSyncServer {
// CFDO: test for invalid payload
const lastAcknowledgedClientVersion = payload.lastAcknowledgedVersion;
const lastAcknowledgedServerVersion =
this.changesRepository.getLastVersion();
this.incrementsRepository.getLastVersion();
const versionΔ =
lastAcknowledgedServerVersion - lastAcknowledgedClientVersion;
if (versionΔ === 0) {
console.info(`Client is up to date!`);
return;
}
if (versionΔ < 0) {
// CFDO: restore the client from the snapshot / deltas?
console.error(
@ -77,38 +73,43 @@ export class ExcalidrawSyncServer {
return;
}
const increments: SERVER_INCREMENT[] = [];
if (versionΔ > 0) {
// CFDO: for versioning we need deletions, but not for the "snapshot" update
const changes = this.changesRepository.getSinceVersion(
lastAcknowledgedClientVersion,
increments.push(
...this.incrementsRepository.getSinceVersion(
lastAcknowledgedClientVersion,
),
);
this.send(client, {
type: "acknowledged",
payload: {
changes,
},
});
}
this.send(client, {
type: "acknowledged",
payload: {
increments,
},
});
}
private push(client: WebSocket, payload: PUSH_PAYLOAD) {
const { type, changes } = payload;
const { type, increments } = payload;
switch (type) {
case "ephemeral":
return this.relay(client, { changes });
return this.relay(client, { increments });
case "durable":
const [acknowledged, error] = Utils.try(() => {
// CFDO: try to apply the changes to the snapshot
return this.changesRepository.saveAll(changes);
});
// CFDO: try to apply the increments to the snapshot
const [acknowledged, error] = Utils.try(() =>
this.incrementsRepository.saveAll(increments),
);
if (error) {
// everything should be automatically rolled-back -> double-check
return this.send(client, {
type: "rejected",
payload: {
message: error.message,
changes,
increments,
},
});
}
@ -116,7 +117,7 @@ export class ExcalidrawSyncServer {
return this.broadcast({
type: "acknowledged",
payload: {
changes: acknowledged,
increments: acknowledged,
},
});
default:
@ -126,7 +127,7 @@ export class ExcalidrawSyncServer {
private relay(
client: WebSocket,
payload: { changes: Array<CLIENT_CHANGE> } | RELAY_PAYLOAD,
payload: { increments: Array<CLIENT_INCREMENT> } | RELAY_PAYLOAD,
) {
return this.broadcast(
{

1957
packages/excalidraw/tests/__snapshots__/history.test.tsx.snap
View file

File diff suppressed because it is too large Load diff

2
packages/excalidraw/tests/history.test.tsx
View file

@ -99,7 +99,7 @@ describe("history", () => {
API.setElements([rect]);
const corrupedEntry = new StoreIncrement(
const corrupedEntry = StoreIncrement.create(
ElementsChange.empty(),
AppStateChange.empty(),
);

5
packages/excalidraw/utility-types.ts
View file

@ -65,3 +65,8 @@ export type MakeBrand<T extends string> = {
/** Maybe just promise or already fulfilled one! */
export type MaybePromise<T> = T | Promise<T>;
/** Strip all the methods or functions from a type */
export type DTO<T> = {
[K in keyof T as T[K] extends Function ? never : K]: T[K];
};