smig/excalidraw
1
0
Fork 0
mirror of https://github.com/excalidraw/excalidraw.git synced 2025-05-03 10:00:07 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Offline support with increments peristed and restored to / from indexedb

Browse source
This commit is contained in:
Marcel Mraz 2024-12-12 14:41:20 +01:00
parent 15d2942aaa
commit 040a57f56a
No known key found for this signature in database
GPG key ID: 4EBD6E62DC830CD2
19 changed files with 1827 additions and 1104 deletions
Download patch file Download diff file Expand all files Collapse all files

89
excalidraw-app/App.tsx
View file

@ -75,7 +75,6 @@ import {
exportToExcalidrawPlus, exportToExcalidrawPlus,
} from "./components/ExportToExcalidrawPlus"; } from "./components/ExportToExcalidrawPlus";
import { updateStaleImageStatuses } from "./data/FileManager"; import { updateStaleImageStatuses } from "./data/FileManager";
import { newElementWith } from "../packages/excalidraw/element/mutateElement";
import { isInitializedImageElement } from "../packages/excalidraw/element/typeChecks"; import { isInitializedImageElement } from "../packages/excalidraw/element/typeChecks";
import { loadFilesFromFirebase } from "./data/firebase"; import { loadFilesFromFirebase } from "./data/firebase";
import { import {
@ -372,8 +371,8 @@ const ExcalidrawWrapper = () => {
const [syncAPI] = useAtom(syncAPIAtom); const [syncAPI] = useAtom(syncAPIAtom);
const [nextVersion, setNextVersion] = useState(-1); const [nextVersion, setNextVersion] = useState(-1);
const currentVersion = useRef(-1); const currentVersion = useRef(-1);
const [acknowledgedChanges, setAcknowledgedChanges] = useState< const [acknowledgedIncrements, setAcknowledgedIncrements] = useState<
ElementsChange[] StoreIncrement[]
>([]); >([]);
const [isCollaborating] = useAtomWithInitialValue(isCollaboratingAtom, () => { const [isCollaborating] = useAtomWithInitialValue(isCollaboratingAtom, () => {
return isCollaborationLink(window.location.href); return isCollaborationLink(window.location.href);
@ -382,7 +381,7 @@ const ExcalidrawWrapper = () => {
useEffect(() => { useEffect(() => {
const interval = setInterval(() => { const interval = setInterval(() => {
setAcknowledgedChanges([...(syncAPI?.acknowledgedChanges ?? [])]); setAcknowledgedIncrements([...(syncAPI?.acknowledgedIncrements ?? [])]);
}, 250); }, 250);
syncAPI?.reconnect(); syncAPI?.reconnect();
@ -648,35 +647,36 @@ const ExcalidrawWrapper = () => {
// this check is redundant, but since this is a hot path, it's best // this check is redundant, but since this is a hot path, it's best
// not to evaludate the nested expression every time // not to evaludate the nested expression every time
if (!LocalData.isSavePaused()) { // CFDO: temporary
LocalData.save(elements, appState, files, () => { // if (!LocalData.isSavePaused()) {
if (excalidrawAPI) { // LocalData.save(elements, appState, files, () => {
let didChange = false; // if (excalidrawAPI) {
// let didChange = false;
const elements = excalidrawAPI // const elements = excalidrawAPI
.getSceneElementsIncludingDeleted() // .getSceneElementsIncludingDeleted()
.map((element) => { // .map((element) => {
if ( // if (
LocalData.fileStorage.shouldUpdateImageElementStatus(element) // LocalData.fileStorage.shouldUpdateImageElementStatus(element)
) { // ) {
const newElement = newElementWith(element, { status: "saved" }); // const newElement = newElementWith(element, { status: "saved" });
if (newElement !== element) { // if (newElement !== element) {
didChange = true; // didChange = true;
} // }
return newElement; // return newElement;
} // }
return element; // return element;
}); // });
if (didChange) { // if (didChange) {
excalidrawAPI.updateScene({ // excalidrawAPI.updateScene({
elements, // elements,
storeAction: StoreAction.UPDATE, // storeAction: StoreAction.UPDATE,
}); // });
} // }
} // }
}); // });
} // }
// Render the debug scene if the debug canvas is available // Render the debug scene if the debug canvas is available
if (debugCanvasRef.current && excalidrawAPI) { if (debugCanvasRef.current && excalidrawAPI) {
@ -694,10 +694,13 @@ const ExcalidrawWrapper = () => {
// - wysiwyg, dragging elements / points, mouse movements, etc. // - wysiwyg, dragging elements / points, mouse movements, etc.
const { elementsChange } = increment; const { elementsChange } = increment;
// some appState like selections should also be transfered (we could even persist it) // CFDO: some appState like selections should also be transfered (we could even persist it)
if (!elementsChange.isEmpty()) { if (!elementsChange.isEmpty()) {
console.log(elementsChange) try {
syncAPI?.push("durable", [elementsChange]); syncAPI?.push("durable", increment);
} catch (e) {
console.error(e);
}
} }
}; };
@ -828,22 +831,23 @@ const ExcalidrawWrapper = () => {
excalidrawAPI?.getSceneElements().map((x) => [x.id, x]), excalidrawAPI?.getSceneElements().map((x) => [x.id, x]),
); );
let changes: ElementsChange[] = []; let increments: StoreIncrement[] = [];
const goingLeft = const goingLeft =
currentVersion.current === -1 || value - currentVersion.current <= 0; currentVersion.current === -1 || value - currentVersion.current <= 0;
if (goingLeft) { if (goingLeft) {
changes = acknowledgedChanges increments = acknowledgedIncrements
.slice(value) .slice(value)
.reverse() .reverse()
.map((x) => x.inverse()); .map((x) => x.inverse());
} else { } else {
changes = acknowledgedChanges.slice(currentVersion.current, value) ?? []; increments =
acknowledgedIncrements.slice(currentVersion.current, value) ?? [];
} }
for (const change of changes) { for (const increment of increments) {
[elements] = change.applyTo( [elements] = increment.elementsChange.applyTo(
elements as SceneElementsMap, elements as SceneElementsMap,
excalidrawAPI?.store.snapshot.elements!, excalidrawAPI?.store.snapshot.elements!,
); );
@ -852,7 +856,7 @@ const ExcalidrawWrapper = () => {
excalidrawAPI?.updateScene({ excalidrawAPI?.updateScene({
appState: { appState: {
...excalidrawAPI?.getAppState(), ...excalidrawAPI?.getAppState(),
viewModeEnabled: value !== acknowledgedChanges.length, viewModeEnabled: value !== acknowledgedIncrements.length,
}, },
elements: Array.from(elements.values()), elements: Array.from(elements.values()),
storeAction: StoreAction.UPDATE, storeAction: StoreAction.UPDATE,
@ -878,8 +882,8 @@ const ExcalidrawWrapper = () => {
}} }}
step={1} step={1}
min={0} min={0}
max={acknowledgedChanges.length} max={acknowledgedIncrements.length}
value={nextVersion === -1 ? acknowledgedChanges.length : nextVersion} value={nextVersion === -1 ? acknowledgedIncrements.length : nextVersion}
onChange={(value) => { onChange={(value) => {
setNextVersion(value as number); setNextVersion(value as number);
debouncedTimeTravel(value as number); debouncedTimeTravel(value as number);
@ -967,7 +971,6 @@ const ExcalidrawWrapper = () => {
/> />
<OverwriteConfirmDialog> <OverwriteConfirmDialog>
<OverwriteConfirmDialog.Actions.ExportToImage /> <OverwriteConfirmDialog.Actions.ExportToImage />
<OverwriteConfirmDialog.Actions.SaveToDisk />
{excalidrawAPI && ( {excalidrawAPI && (
<OverwriteConfirmDialog.Action <OverwriteConfirmDialog.Action
title={t("overwriteConfirm.action.excalidrawPlus.title")} title={t("overwriteConfirm.action.excalidrawPlus.title")}

1
excalidraw-app/app_constants.ts
View file

@ -45,6 +45,7 @@ export const STORAGE_KEYS = {
VERSION_FILES: "version-files", VERSION_FILES: "version-files",
IDB_LIBRARY: "excalidraw-library", IDB_LIBRARY: "excalidraw-library",
IDB_SYNC: "excalidraw-sync",
// do not use apart from migrations // do not use apart from migrations
__LEGACY_LOCAL_STORAGE_LIBRARY: "excalidraw-library", __LEGACY_LOCAL_STORAGE_LIBRARY: "excalidraw-library",

14
excalidraw-app/collab/Collab.tsx
View file

@ -78,7 +78,7 @@ import {
import { newElementWith } from "../../packages/excalidraw/element/mutateElement"; import { newElementWith } from "../../packages/excalidraw/element/mutateElement";
import { decryptData } from "../../packages/excalidraw/data/encryption"; import { decryptData } from "../../packages/excalidraw/data/encryption";
import { resetBrowserStateVersions } from "../data/tabSync"; import { resetBrowserStateVersions } from "../data/tabSync";
import { LocalData } from "../data/LocalData"; import { LocalData, SyncIndexedDBAdapter } from "../data/LocalData";
import { appJotaiStore, atom } from "../app-jotai"; import { appJotaiStore, atom } from "../app-jotai";
import type { Mutable, ValueOf } from "../../packages/excalidraw/utility-types"; import type { Mutable, ValueOf } from "../../packages/excalidraw/utility-types";
import { getVisibleSceneBounds } from "../../packages/excalidraw/element/bounds"; import { getVisibleSceneBounds } from "../../packages/excalidraw/element/bounds";
@ -88,9 +88,9 @@ import type {
ReconciledExcalidrawElement, ReconciledExcalidrawElement,
RemoteExcalidrawElement, RemoteExcalidrawElement,
} from "../../packages/excalidraw/data/reconcile"; } from "../../packages/excalidraw/data/reconcile";
import { ExcalidrawSyncClient } from "../../packages/excalidraw/sync/client"; import { SyncClient } from "../../packages/excalidraw/sync/client";
export const syncAPIAtom = atom<ExcalidrawSyncClient | null>(null); export const syncAPIAtom = atom<SyncClient | null>(null);
export const collabAPIAtom = atom<CollabAPI | null>(null); export const collabAPIAtom = atom<CollabAPI | null>(null);
export const isCollaboratingAtom = atom(false); export const isCollaboratingAtom = atom(false);
export const isOfflineAtom = atom(false); export const isOfflineAtom = atom(false);
@ -236,9 +236,11 @@ class Collab extends PureComponent<CollabProps, CollabState> {
}; };
appJotaiStore.set(collabAPIAtom, collabAPI); appJotaiStore.set(collabAPIAtom, collabAPI);
appJotaiStore.set(
syncAPIAtom, SyncClient.create(this.excalidrawAPI, SyncIndexedDBAdapter).then(
new ExcalidrawSyncClient(this.excalidrawAPI), (syncAPI) => {
appJotaiStore.set(syncAPIAtom, syncAPI);
},
); );
if (import.meta.env.MODE === ENV.TEST || import.meta.env.DEV) { if (import.meta.env.MODE === ENV.TEST || import.meta.env.DEV) {

135
excalidraw-app/data/LocalData.ts
View file

@ -36,12 +36,16 @@ import type {
BinaryFileData, BinaryFileData,
BinaryFiles, BinaryFiles,
} from "../../packages/excalidraw/types"; } from "../../packages/excalidraw/types";
import type { MaybePromise } from "../../packages/excalidraw/utility-types"; import type {
DTO,
MaybePromise,
} from "../../packages/excalidraw/utility-types";
import { debounce } from "../../packages/excalidraw/utils"; import { debounce } from "../../packages/excalidraw/utils";
import { SAVE_TO_LOCAL_STORAGE_TIMEOUT, STORAGE_KEYS } from "../app_constants"; import { SAVE_TO_LOCAL_STORAGE_TIMEOUT, STORAGE_KEYS } from "../app_constants";
import { FileManager } from "./FileManager"; import { FileManager } from "./FileManager";
import { Locker } from "./Locker"; import { Locker } from "./Locker";
import { updateBrowserStateVersion } from "./tabSync"; import { updateBrowserStateVersion } from "./tabSync";
import { StoreIncrement } from "../../packages/excalidraw/store";
const filesStore = createStore("files-db", "files-store"); const filesStore = createStore("files-db", "files-store");
@ -65,34 +69,35 @@ class LocalFileManager extends FileManager {
}; };
} }
const saveDataStateToLocalStorage = ( // CFDO: temporary
elements: readonly ExcalidrawElement[], // const saveDataStateToLocalStorage = (
appState: AppState, // elements: readonly ExcalidrawElement[],
) => { // appState: AppState,
try { // ) => {
const _appState = clearAppStateForLocalStorage(appState); // try {
// const _appState = clearAppStateForLocalStorage(appState);
if ( // if (
_appState.openSidebar?.name === DEFAULT_SIDEBAR.name && // _appState.openSidebar?.name === DEFAULT_SIDEBAR.name &&
_appState.openSidebar.tab === CANVAS_SEARCH_TAB // _appState.openSidebar.tab === CANVAS_SEARCH_TAB
) { // ) {
_appState.openSidebar = null; // _appState.openSidebar = null;
} // }
localStorage.setItem( // localStorage.setItem(
STORAGE_KEYS.LOCAL_STORAGE_ELEMENTS, // STORAGE_KEYS.LOCAL_STORAGE_ELEMENTS,
JSON.stringify(clearElementsForLocalStorage(elements)), // JSON.stringify(clearElementsForLocalStorage(elements)),
); // );
localStorage.setItem( // localStorage.setItem(
STORAGE_KEYS.LOCAL_STORAGE_APP_STATE, // STORAGE_KEYS.LOCAL_STORAGE_APP_STATE,
JSON.stringify(_appState), // JSON.stringify(_appState),
); // );
updateBrowserStateVersion(STORAGE_KEYS.VERSION_DATA_STATE); // updateBrowserStateVersion(STORAGE_KEYS.VERSION_DATA_STATE);
} catch (error: any) { // } catch (error: any) {
// Unable to access window.localStorage // // Unable to access window.localStorage
console.error(error); // console.error(error);
} // }
}; // };
type SavingLockTypes = "collaboration"; type SavingLockTypes = "collaboration";
@ -104,13 +109,12 @@ export class LocalData {
files: BinaryFiles, files: BinaryFiles,
onFilesSaved: () => void, onFilesSaved: () => void,
) => { ) => {
saveDataStateToLocalStorage(elements, appState); // saveDataStateToLocalStorage(elements, appState);
// await this.fileStorage.saveFiles({
await this.fileStorage.saveFiles({ // elements,
elements, // files,
files, // });
}); // onFilesSaved();
onFilesSaved();
}, },
SAVE_TO_LOCAL_STORAGE_TIMEOUT, SAVE_TO_LOCAL_STORAGE_TIMEOUT,
); );
@ -256,3 +260,66 @@ export class LibraryLocalStorageMigrationAdapter {
localStorage.removeItem(STORAGE_KEYS.__LEGACY_LOCAL_STORAGE_LIBRARY); localStorage.removeItem(STORAGE_KEYS.__LEGACY_LOCAL_STORAGE_LIBRARY);
} }
} }
interface SyncIncrementPersistedData {
increments: DTO<StoreIncrement>[];
}
interface SyncMetaPersistedData {
lastAcknowledgedVersion: number;
}
export class SyncIndexedDBAdapter {
/** IndexedDB database and store name */
private static idb_name = STORAGE_KEYS.IDB_SYNC;
/** library data store keys */
private static incrementsKey = "increments";
private static metadataKey = "metadata";
private static store = createStore(
`${SyncIndexedDBAdapter.idb_name}-db`,
`${SyncIndexedDBAdapter.idb_name}-store`,
);
static async loadIncrements() {
const IDBData = await get<SyncIncrementPersistedData>(
SyncIndexedDBAdapter.incrementsKey,
SyncIndexedDBAdapter.store,
);
if (IDBData?.increments?.length) {
return {
increments: IDBData.increments.map((storeIncrementDTO) =>
StoreIncrement.restore(storeIncrementDTO),
),
};
}
return null;
}
static async saveIncrements(data: SyncIncrementPersistedData): Promise<void> {
return set(
SyncIndexedDBAdapter.incrementsKey,
data,
SyncIndexedDBAdapter.store,
);
}
static async loadMetadata() {
const IDBData = await get<SyncMetaPersistedData>(
SyncIndexedDBAdapter.metadataKey,
SyncIndexedDBAdapter.store,
);
return IDBData || null;
}
static async saveMetadata(data: SyncMetaPersistedData): Promise<void> {
return set(
SyncIndexedDBAdapter.metadataKey,
data,
SyncIndexedDBAdapter.store,
);
}
}

5
packages/excalidraw/actions/actionExport.tsx
View file

@ -185,8 +185,9 @@ export const actionSaveToActiveFile = register({
return { storeAction: StoreAction.NONE }; return { storeAction: StoreAction.NONE };
} }
}, },
keyTest: (event) => // CFDO: temporary
event.key === KEYS.S && event[KEYS.CTRL_OR_CMD] && !event.shiftKey, // keyTest: (event) =>
// event.key === KEYS.S && event[KEYS.CTRL_OR_CMD] && !event.shiftKey,
}); });
export const actionSaveFileToDisk = register({ export const actionSaveFileToDisk = register({

45
packages/excalidraw/change.ts
View file

@ -32,7 +32,6 @@ import type {
} from "./element/types"; } from "./element/types";
import { orderByFractionalIndex, syncMovedIndices } from "./fractionalIndex"; import { orderByFractionalIndex, syncMovedIndices } from "./fractionalIndex";
import { getNonDeletedGroupIds } from "./groups"; import { getNonDeletedGroupIds } from "./groups";
import { randomId } from "./random";
import { getObservedAppState } from "./store"; import { getObservedAppState } from "./store";
import type { import type {
AppState, AppState,
@ -40,7 +39,7 @@ import type {
ObservedElementsAppState, ObservedElementsAppState,
ObservedStandaloneAppState, ObservedStandaloneAppState,
} from "./types"; } from "./types";
import type { SubtypeOf, ValueOf } from "./utility-types"; import type { DTO, SubtypeOf, ValueOf } from "./utility-types";
import { import {
arrayToMap, arrayToMap,
arrayToObject, arrayToObject,
@ -416,7 +415,7 @@ interface Change<T> {
} }
export class AppStateChange implements Change<AppState> { export class AppStateChange implements Change<AppState> {
private constructor(private readonly delta: Delta<ObservedAppState>) {} private constructor(public readonly delta: Delta<ObservedAppState>) {}
public static calculate<T extends ObservedAppState>( public static calculate<T extends ObservedAppState>(
prevAppState: T, prevAppState: T,
@ -432,6 +431,13 @@ export class AppStateChange implements Change<AppState> {
return new AppStateChange(delta); return new AppStateChange(delta);
} }
public static restore(
appStateChangeDTO: DTO<AppStateChange>,
): AppStateChange {
const { delta } = appStateChangeDTO;
return new AppStateChange(delta);
}
public static empty() { public static empty() {
return new AppStateChange(Delta.create({}, {})); return new AppStateChange(Delta.create({}, {}));
} }
@ -797,13 +803,13 @@ export class AppStateChange implements Change<AppState> {
} }
} }
// CFDO: consider adding here (nonnullable) version & versionNonce & updated & seed (so that we have correct versions when recunstructing from remote)
type ElementPartial<T extends ExcalidrawElement = ExcalidrawElement> = Omit< type ElementPartial<T extends ExcalidrawElement = ExcalidrawElement> = Omit<
ElementUpdate<Ordered<T>>, ElementUpdate<Ordered<T>>,
"seed" "seed"
>; >;
type ElementsChangeOptions = { type ElementsChangeOptions = {
id: string;
shouldRedistribute: boolean; shouldRedistribute: boolean;
}; };
@ -813,10 +819,9 @@ type ElementsChangeOptions = {
*/ */
export class ElementsChange implements Change<SceneElementsMap> { export class ElementsChange implements Change<SceneElementsMap> {
private constructor( private constructor(
public readonly id: string, public readonly added: Record<string, Delta<ElementPartial>>,
private readonly added: Record<string, Delta<ElementPartial>>, public readonly removed: Record<string, Delta<ElementPartial>>,
private readonly removed: Record<string, Delta<ElementPartial>>, public readonly updated: Record<string, Delta<ElementPartial>>,
private readonly updated: Record<string, Delta<ElementPartial>>,
) {} ) {}
public static create( public static create(
@ -824,11 +829,10 @@ export class ElementsChange implements Change<SceneElementsMap> {
removed: Record<string, Delta<ElementPartial>>, removed: Record<string, Delta<ElementPartial>>,
updated: Record<string, Delta<ElementPartial>>, updated: Record<string, Delta<ElementPartial>>,
options: ElementsChangeOptions = { options: ElementsChangeOptions = {
id: randomId(),
shouldRedistribute: false, shouldRedistribute: false,
}, },
) { ) {
const { id, shouldRedistribute } = options; const { shouldRedistribute } = options;
let change: ElementsChange; let change: ElementsChange;
if (shouldRedistribute) { if (shouldRedistribute) {
@ -852,9 +856,9 @@ export class ElementsChange implements Change<SceneElementsMap> {
} }
} }
change = new ElementsChange(id, nextAdded, nextRemoved, nextUpdated); change = new ElementsChange(nextAdded, nextRemoved, nextUpdated);
} else { } else {
change = new ElementsChange(id, added, removed, updated); change = new ElementsChange(added, removed, updated);
} }
if (import.meta.env.DEV || import.meta.env.MODE === ENV.TEST) { if (import.meta.env.DEV || import.meta.env.MODE === ENV.TEST) {
@ -866,6 +870,13 @@ export class ElementsChange implements Change<SceneElementsMap> {
return change; return change;
} }
public static restore(
elementsChangeDTO: DTO<ElementsChange>,
): ElementsChange {
const { added, removed, updated } = elementsChangeDTO;
return ElementsChange.create(added, removed, updated);
}
private static satisfiesAddition = ({ private static satisfiesAddition = ({
deleted, deleted,
inserted, inserted,
@ -997,15 +1008,6 @@ export class ElementsChange implements Change<SceneElementsMap> {
return ElementsChange.create({}, {}, {}); return ElementsChange.create({}, {}, {});
} }
public static load(payload: string) {
const { id, added, removed, updated } = JSON.parse(payload);
return ElementsChange.create(added, removed, updated, {
id,
shouldRedistribute: false,
});
}
public inverse(): ElementsChange { public inverse(): ElementsChange {
const inverseInternal = (deltas: Record<string, Delta<ElementPartial>>) => { const inverseInternal = (deltas: Record<string, Delta<ElementPartial>>) => {
const inversedDeltas: Record<string, Delta<ElementPartial>> = {}; const inversedDeltas: Record<string, Delta<ElementPartial>> = {};
@ -1091,7 +1093,6 @@ export class ElementsChange implements Change<SceneElementsMap> {
const updated = applyLatestChangesInternal(this.updated); const updated = applyLatestChangesInternal(this.updated);
return ElementsChange.create(added, removed, updated, { return ElementsChange.create(added, removed, updated, {
id: this.id,
shouldRedistribute: true, // redistribute the deltas as `isDeleted` could have been updated shouldRedistribute: true, // redistribute the deltas as `isDeleted` could have been updated
}); });
} }

66
packages/excalidraw/cloudflare/changes.ts
View file

@ -1,66 +0,0 @@
import type {
ChangesRepository,
CLIENT_CHANGE,
SERVER_CHANGE,
} from "../sync/protocol";
// CFDO: add senderId, possibly roomId as well
export class DurableChangesRepository implements ChangesRepository {
constructor(private storage: DurableObjectStorage) {
// #region DEV ONLY
// this.storage.sql.exec(`DROP TABLE IF EXISTS changes;`);
// #endregion
this.storage.sql.exec(`CREATE TABLE IF NOT EXISTS changes(
id TEXT PRIMARY KEY,
payload TEXT NOT NULL,
version INTEGER NOT NULL DEFAULT 1,
createdAt TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);`);
}
public saveAll = (changes: Array<CLIENT_CHANGE>) => {
return this.storage.transactionSync(() => {
const prevVersion = this.getLastVersion();
const nextVersion = prevVersion + changes.length;
// CFDO: in theory payload could contain array of changes, if we would need to optimize writes
for (const [index, change] of changes.entries()) {
const version = prevVersion + index + 1;
// unique id ensures that we don't acknowledge the same change twice
this.storage.sql.exec(
`INSERT INTO changes (id, payload, version) VALUES (?, ?, ?);`,
change.id,
JSON.stringify(change),
version,
);
}
// sanity check
if (nextVersion !== this.getLastVersion()) {
throw new Error(
`Expected last acknowledged version to be "${nextVersion}", but it is "${this.getLastVersion()}!"`,
);
}
return this.getSinceVersion(prevVersion);
});
};
public getSinceVersion = (version: number): Array<SERVER_CHANGE> => {
return this.storage.sql
.exec<SERVER_CHANGE>(
`SELECT id, payload, version FROM changes WHERE version > (?) ORDER BY version ASC;`,
version,
)
.toArray();
};
public getLastVersion = (): number => {
const result = this.storage.sql
.exec(`SELECT MAX(version) FROM changes;`)
.one();
return result ? Number(result["MAX(version)"]) : 0;
};
}

88
packages/excalidraw/cloudflare/repository.ts Normal file
View file

@ -0,0 +1,88 @@
import type {
IncrementsRepository,
CLIENT_INCREMENT,
SERVER_INCREMENT,
} from "../sync/protocol";
// CFDO: add senderId, possibly roomId as well
export class DurableIncrementsRepository implements IncrementsRepository {
constructor(private storage: DurableObjectStorage) {
// #region DEV ONLY
this.storage.sql.exec(`DROP TABLE IF EXISTS increments;`);
// #endregion
this.storage.sql.exec(`CREATE TABLE IF NOT EXISTS increments(
version INTEGER PRIMARY KEY AUTOINCREMENT,
id TEXT NOT NULL UNIQUE,
createdAt TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
payload TEXT
);`);
}
public saveAll(increments: Array<CLIENT_INCREMENT>) {
return this.storage.transactionSync(() => {
const prevVersion = this.getLastVersion();
const acknowledged: Array<SERVER_INCREMENT> = [];
for (const increment of increments) {
try {
// unique id ensures that we don't acknowledge the same increment twice
this.storage.sql.exec(
`INSERT INTO increments (id, payload) VALUES (?, ?);`,
increment.id,
JSON.stringify(increment),
);
} catch (e) {
// check if the increment has been already acknowledged
// in case client for some reason did not receive acknowledgement
// and reconnected while the we still have the increment in the worker
// otherwise the client is doomed to full a restore
if (
e instanceof Error &&
e.message.includes(
"UNIQUE constraint failed: increments.id: SQLITE_CONSTRAINT",
)
) {
acknowledged.push(this.getById(increment.id));
continue;
}
throw e;
}
}
// query the just added increments
acknowledged.push(...this.getSinceVersion(prevVersion));
return acknowledged;
});
}
public getSinceVersion(version: number): Array<SERVER_INCREMENT> {
// CFDO: for versioning we need deletions, but not for the "snapshot" update;
return this.storage.sql
.exec<SERVER_INCREMENT>(
`SELECT id, payload, version FROM increments WHERE version > (?) ORDER BY version, createdAt ASC;`,
version,
)
.toArray();
}
public getLastVersion(): number {
// CFDO: might be in memory to reduce number of rows read (or index on version at least, if btree affect rows read)
const result = this.storage.sql
.exec(`SELECT MAX(version) FROM increments;`)
.one();
return result ? Number(result["MAX(version)"]) : 0;
}
public getById(id: string): SERVER_INCREMENT {
return this.storage.sql
.exec<SERVER_INCREMENT>(
`SELECT id, payload, version FROM increments WHERE id = (?)`,
id,
)
.one();
}
}

4
packages/excalidraw/cloudflare/room.ts
View file

@ -1,5 +1,5 @@
import { DurableObject } from "cloudflare:workers"; import { DurableObject } from "cloudflare:workers";
import { DurableChangesRepository } from "./changes"; import { DurableIncrementsRepository } from "./repository";
import { ExcalidrawSyncServer } from "../sync/server"; import { ExcalidrawSyncServer } from "../sync/server";
import type { ExcalidrawElement } from "../element/types"; import type { ExcalidrawElement } from "../element/types";
@ -35,7 +35,7 @@ export class DurableRoom extends DurableObject {
}); });
this.sync = new ExcalidrawSyncServer( this.sync = new ExcalidrawSyncServer(
new DurableChangesRepository(ctx.storage), new DurableIncrementsRepository(ctx.storage),
); );
// in case it hibernates, let's get take active connections // in case it hibernates, let's get take active connections

2
packages/excalidraw/history.ts
View file

@ -152,7 +152,7 @@ export class History {
entry: HistoryEntry, entry: HistoryEntry,
prevElements: SceneElementsMap, prevElements: SceneElementsMap,
) { ) {
const updatedEntry = entry.inverse().applyLatestChanges(prevElements); const updatedEntry = entry.applyLatestChanges(prevElements);
return stack.push(updatedEntry); return stack.push(updatedEntry);
} }
} }

2
packages/excalidraw/package.json
View file

@ -74,6 +74,7 @@
"image-blob-reduce": "3.0.1", "image-blob-reduce": "3.0.1",
"jotai": "2.11.0", "jotai": "2.11.0",
"jotai-scope": "0.7.2", "jotai-scope": "0.7.2",
"lodash.debounce": "4.0.8",
"lodash.throttle": "4.1.1", "lodash.throttle": "4.1.1",
"nanoid": "3.3.3", "nanoid": "3.3.3",
"open-color": "1.9.1", "open-color": "1.9.1",
@ -104,6 +105,7 @@
"@testing-library/jest-dom": "5.16.2", "@testing-library/jest-dom": "5.16.2",
"@testing-library/react": "16.0.0", "@testing-library/react": "16.0.0",
"@types/async-lock": "^1.4.2", "@types/async-lock": "^1.4.2",
"@types/lodash.debounce": "4.0.9",
"@types/pako": "1.0.3", "@types/pako": "1.0.3",
"@types/pica": "5.1.3", "@types/pica": "5.1.3",
"@types/resize-observer-browser": "0.1.7", "@types/resize-observer-browser": "0.1.7",

124
packages/excalidraw/store.ts
View file

@ -1,16 +1,17 @@
import { ENV } from "./constants";
import { Emitter } from "./emitter";
import { randomId } from "./random";
import { isShallowEqual } from "./utils";
import { getDefaultAppState } from "./appState"; import { getDefaultAppState } from "./appState";
import { AppStateChange, ElementsChange } from "./change"; import { AppStateChange, ElementsChange } from "./change";
import { ENV } from "./constants";
import { newElementWith } from "./element/mutateElement"; import { newElementWith } from "./element/mutateElement";
import { deepCopyElement } from "./element/newElement"; import { deepCopyElement } from "./element/newElement";
import type { AppState, ObservedAppState } from "./types";
import type { DTO, ValueOf } from "./utility-types";
import type { import type {
OrderedExcalidrawElement, OrderedExcalidrawElement,
SceneElementsMap, SceneElementsMap,
} from "./element/types"; } from "./element/types";
import { Emitter } from "./emitter";
import type { AppState, ObservedAppState } from "./types";
import type { ValueOf } from "./utility-types";
import { isShallowEqual } from "./utils";
// hidden non-enumerable property for runtime checks // hidden non-enumerable property for runtime checks
const hiddenObservedAppStateProp = "__observedAppState"; const hiddenObservedAppStateProp = "__observedAppState";
@ -96,31 +97,31 @@ export class Store {
* Use to schedule calculation of a store increment. * Use to schedule calculation of a store increment.
*/ */
// TODO: Suspicious that this is called so many places. Seems error-prone. // TODO: Suspicious that this is called so many places. Seems error-prone.
public shouldCaptureIncrement = () => { public shouldCaptureIncrement() {
this.scheduleAction(StoreAction.CAPTURE); this.scheduleAction(StoreAction.CAPTURE);
}; }
/** /**
* Use to schedule update of the snapshot, useful on updates for which we don't need to calculate increments (i.e. remote updates). * Use to schedule update of the snapshot, useful on updates for which we don't need to calculate increments (i.e. remote updates).
*/ */
public shouldUpdateSnapshot = () => { public shouldUpdateSnapshot() {
this.scheduleAction(StoreAction.UPDATE); this.scheduleAction(StoreAction.UPDATE);
}; }
private scheduleAction = (action: StoreActionType) => { private scheduleAction(action: StoreActionType) {
this.scheduledActions.add(action); this.scheduledActions.add(action);
this.satisfiesScheduledActionsInvariant(); this.satisfiesScheduledActionsInvariant();
}; }
/** /**
* Based on the scheduled operation, either only updates store snapshot or also calculates increment and emits the result as a `StoreIncrement`. * Based on the scheduled operation, either only updates store snapshot or also calculates increment and emits the result as a `StoreIncrement`.
* *
* @emits StoreIncrement when increment is calculated. * @emits StoreIncrement when increment is calculated.
*/ */
public commit = ( public commit(
elements: Map<string, OrderedExcalidrawElement> | undefined, elements: Map<string, OrderedExcalidrawElement> | undefined,
appState: AppState | ObservedAppState | undefined, appState: AppState | ObservedAppState | undefined,
): void => { ): void {
try { try {
// Capture has precedence since it also performs update // Capture has precedence since it also performs update
if (this.scheduledActions.has(StoreAction.CAPTURE)) { if (this.scheduledActions.has(StoreAction.CAPTURE)) {
@ -133,17 +134,17 @@ export class Store {
// Defensively reset all scheduled actions, potentially cleans up other runtime garbage // Defensively reset all scheduled actions, potentially cleans up other runtime garbage
this.scheduledActions = new Set(); this.scheduledActions = new Set();
} }
}; }
/** /**
* Performs diff calculation, calculates and emits the increment. * Performs diff calculation, calculates and emits the increment.
* *
* @emits StoreIncrement when increment is calculated. * @emits StoreIncrement when increment is calculated.
*/ */
public captureIncrement = ( public captureIncrement(
elements: Map<string, OrderedExcalidrawElement> | undefined, elements: Map<string, OrderedExcalidrawElement> | undefined,
appState: AppState | ObservedAppState | undefined, appState: AppState | ObservedAppState | undefined,
) => { ) {
const prevSnapshot = this.snapshot; const prevSnapshot = this.snapshot;
const nextSnapshot = this.snapshot.maybeClone(elements, appState); const nextSnapshot = this.snapshot.maybeClone(elements, appState);
@ -161,39 +162,39 @@ export class Store {
if (!elementsChange.isEmpty() || !appStateChange.isEmpty()) { if (!elementsChange.isEmpty() || !appStateChange.isEmpty()) {
// Notify listeners with the increment // Notify listeners with the increment
this.onStoreIncrementEmitter.trigger( this.onStoreIncrementEmitter.trigger(
new StoreIncrement(elementsChange, appStateChange), StoreIncrement.create(elementsChange, appStateChange),
); );
} }
// Update snapshot // Update snapshot
this.snapshot = nextSnapshot; this.snapshot = nextSnapshot;
} }
}; }
/** /**
* Updates the snapshot without performing any diff calculation. * Updates the snapshot without performing any diff calculation.
*/ */
public updateSnapshot = ( public updateSnapshot(
elements: Map<string, OrderedExcalidrawElement> | undefined, elements: Map<string, OrderedExcalidrawElement> | undefined,
appState: AppState | ObservedAppState | undefined, appState: AppState | ObservedAppState | undefined,
) => { ) {
const nextSnapshot = this.snapshot.maybeClone(elements, appState); const nextSnapshot = this.snapshot.maybeClone(elements, appState);
if (this.snapshot !== nextSnapshot) { if (this.snapshot !== nextSnapshot) {
// Update snapshot // Update snapshot
this.snapshot = nextSnapshot; this.snapshot = nextSnapshot;
} }
}; }
/** /**
* Filters out yet uncomitted elements from `nextElements`, which are part of in-progress local async actions (ephemerals) and thus were not yet commited to the snapshot. * Filters out yet uncomitted elements from `nextElements`, which are part of in-progress local async actions (ephemerals) and thus were not yet commited to the snapshot.
* *
* This is necessary in updates in which we receive reconciled elements, already containing elements which were not yet captured by the local store (i.e. collab). * This is necessary in updates in which we receive reconciled elements, already containing elements which were not yet captured by the local store (i.e. collab).
*/ */
public filterUncomittedElements = ( public filterUncomittedElements(
prevElements: Map<string, OrderedExcalidrawElement>, prevElements: Map<string, OrderedExcalidrawElement>,
nextElements: Map<string, OrderedExcalidrawElement>, nextElements: Map<string, OrderedExcalidrawElement>,
) => { ) {
for (const [id, prevElement] of prevElements.entries()) { for (const [id, prevElement] of prevElements.entries()) {
const nextElement = nextElements.get(id); const nextElement = nextElements.get(id);
@ -215,18 +216,18 @@ export class Store {
} }
return nextElements; return nextElements;
}; }
/** /**
* Apply and emit increment. * Apply and emit increment.
* *
* @emits StoreIncrement when increment is applied. * @emits StoreIncrement when increment is applied.
*/ */
public applyIncrementTo = ( public applyIncrementTo(
increment: StoreIncrement, increment: StoreIncrement,
elements: SceneElementsMap, elements: SceneElementsMap,
appState: AppState, appState: AppState,
): [SceneElementsMap, AppState, boolean] => { ): [SceneElementsMap, AppState, boolean] {
const [nextElements, elementsContainVisibleChange] = const [nextElements, elementsContainVisibleChange] =
increment.elementsChange.applyTo(elements, this.snapshot.elements); increment.elementsChange.applyTo(elements, this.snapshot.elements);
@ -239,17 +240,17 @@ export class Store {
this.onStoreIncrementEmitter.trigger(increment); this.onStoreIncrementEmitter.trigger(increment);
return [nextElements, nextAppState, appliedVisibleChanges]; return [nextElements, nextAppState, appliedVisibleChanges];
}; }
/** /**
* Clears the store instance. * Clears the store instance.
*/ */
public clear = (): void => { public clear(): void {
this.snapshot = StoreSnapshot.empty(); this.snapshot = StoreSnapshot.empty();
this.scheduledActions = new Set(); this.scheduledActions = new Set();
}; }
private satisfiesScheduledActionsInvariant = () => { private satisfiesScheduledActionsInvariant() {
if (!(this.scheduledActions.size >= 0 && this.scheduledActions.size <= 3)) { if (!(this.scheduledActions.size >= 0 && this.scheduledActions.size <= 3)) {
const message = `There can be at most three store actions scheduled at the same time, but there are "${this.scheduledActions.size}".`; const message = `There can be at most three store actions scheduled at the same time, but there are "${this.scheduledActions.size}".`;
console.error(message, this.scheduledActions.values()); console.error(message, this.scheduledActions.values());
@ -258,20 +259,70 @@ export class Store {
throw new Error(message); throw new Error(message);
} }
} }
}; }
} }
/** /**
* Represent an increment to the Store. * Represent an increment to the Store.
*/ */
export class StoreIncrement { export class StoreIncrement {
constructor( private constructor(
public readonly id: string,
public readonly elementsChange: ElementsChange, public readonly elementsChange: ElementsChange,
public readonly appStateChange: AppStateChange, public readonly appStateChange: AppStateChange,
) {} ) {}
/**
* Create a new instance of `StoreIncrement`.
*/
public static create(
elementsChange: ElementsChange,
appStateChange: AppStateChange,
opts: {
id: string;
} = {
id: randomId(),
},
) {
return new StoreIncrement(opts.id, elementsChange, appStateChange);
}
/**
* Restore a store increment instance from a DTO.
*/
public static restore(storeIncrementDTO: DTO<StoreIncrement>) {
const { id, elementsChange, appStateChange } = storeIncrementDTO;
return new StoreIncrement(
id,
ElementsChange.restore(elementsChange),
AppStateChange.restore(appStateChange),
);
}
// CFDO: why it would be a string if it can be a DTO?
/**
* Parse and load the increment from the remote payload.
*/
public static load(payload: string) {
// CFDO: ensure typesafety
const {
id,
elementsChange: { added, removed, updated },
} = JSON.parse(payload);
const elementsChange = ElementsChange.create(added, removed, updated, {
shouldRedistribute: false,
});
return new StoreIncrement(id, elementsChange, AppStateChange.empty());
}
/**
* Inverse store increment, creates new instance of `StoreIncrement`.
*/
public inverse(): StoreIncrement { public inverse(): StoreIncrement {
return new StoreIncrement( return new StoreIncrement(
randomId(),
this.elementsChange.inverse(), this.elementsChange.inverse(),
this.appStateChange.inverse(), this.appStateChange.inverse(),
); );
@ -281,10 +332,13 @@ export class StoreIncrement {
* Apply latest (remote) changes to the increment, creates new instance of `StoreIncrement`. * Apply latest (remote) changes to the increment, creates new instance of `StoreIncrement`.
*/ */
public applyLatestChanges(elements: SceneElementsMap): StoreIncrement { public applyLatestChanges(elements: SceneElementsMap): StoreIncrement {
const updatedElementsChange = const inversedIncrement = this.inverse();
this.elementsChange.applyLatestChanges(elements);
return new StoreIncrement(updatedElementsChange, this.appStateChange); return new StoreIncrement(
inversedIncrement.id,
inversedIncrement.elementsChange.applyLatestChanges(elements),
inversedIncrement.appStateChange,
);
} }
public isEmpty() { public isEmpty() {

308
packages/excalidraw/sync/client.ts
View file

@ -1,12 +1,90 @@
/* eslint-disable no-console */ /* eslint-disable no-console */
import throttle from "lodash.throttle";
import debounce from "lodash.debounce";
import { Utils } from "./utils"; import { Utils } from "./utils";
import { ElementsChange } from "../change"; import { StoreIncrement } from "../store";
import type { ExcalidrawImperativeAPI } from "../types"; import type { ExcalidrawImperativeAPI } from "../types";
import type { SceneElementsMap } from "../element/types"; import type { SceneElementsMap } from "../element/types";
import type { CLIENT_CHANGE, PUSH_PAYLOAD, SERVER_CHANGE } from "./protocol"; import type {
import throttle from "lodash.throttle"; CLIENT_INCREMENT,
PUSH_PAYLOAD,
SERVER_INCREMENT,
} from "./protocol";
export class ExcalidrawSyncClient { interface IncrementsRepository {
loadIncrements(): Promise<{ increments: Array<StoreIncrement> } | null>;
saveIncrements(params: { increments: Array<StoreIncrement> }): Promise<void>;
}
interface MetadataRepository {
loadMetadata(): Promise<{ lastAcknowledgedVersion: number } | null>;
saveMetadata(metadata: { lastAcknowledgedVersion: number }): Promise<void>;
}
// CFDO: make sure the increments are always acknowledged (deleted from the repository)
export class SyncQueue {
private readonly queue: Map<string, StoreIncrement>;
private readonly repository: IncrementsRepository;
private constructor(
queue: Map<string, StoreIncrement> = new Map(),
repository: IncrementsRepository,
) {
this.queue = queue;
this.repository = repository;
}
public static async create(repository: IncrementsRepository) {
const data = await repository.loadIncrements();
return new SyncQueue(
new Map(data?.increments?.map((increment) => [increment.id, increment])),
repository,
);
}
public getAll() {
return Array.from(this.queue.values());
}
public get(id: StoreIncrement["id"]) {
return this.queue.get(id);
}
public has(id: StoreIncrement["id"]) {
return this.queue.has(id);
}
public add(...increments: StoreIncrement[]) {
for (const increment of increments) {
this.queue.set(increment.id, increment);
}
this.persist();
}
public remove(...ids: StoreIncrement["id"][]) {
for (const id of ids) {
this.queue.delete(id);
}
this.persist();
}
public persist = throttle(
async () => {
try {
await this.repository.saveIncrements({ increments: this.getAll() });
} catch (e) {
console.error("Failed to persist the sync queue:", e);
}
},
1000,
{ leading: false, trailing: true },
);
}
export class SyncClient {
private static readonly HOST_URL = import.meta.env.DEV private static readonly HOST_URL = import.meta.env.DEV
? "ws://localhost:8787" ? "ws://localhost:8787"
: "https://excalidraw-sync.marcel-529.workers.dev"; : "https://excalidraw-sync.marcel-529.workers.dev";
@ -17,18 +95,29 @@ export class ExcalidrawSyncClient {
private static readonly RECONNECT_INTERVAL = 10_000; private static readonly RECONNECT_INTERVAL = 10_000;
private lastAcknowledgedVersion = 0;
private readonly api: ExcalidrawImperativeAPI; private readonly api: ExcalidrawImperativeAPI;
private readonly roomId: string; private readonly queue: SyncQueue;
private readonly queuedChanges: Map< private readonly repository: MetadataRepository;
string,
{ queuedAt: number; change: CLIENT_CHANGE }
> = new Map();
public readonly acknowledgedChanges: Array<ElementsChange> = [];
private get localChanges() { // CFDO: shouldn't be stateful, only request / response
return Array.from(this.queuedChanges.values()).map(({ change }) => change); private readonly acknowledgedIncrementsMap: Map<string, StoreIncrement> =
new Map();
public get acknowledgedIncrements() {
return Array.from(this.acknowledgedIncrementsMap.values());
}
private readonly roomId: string;
private _lastAcknowledgedVersion = 0;
private get lastAcknowledgedVersion() {
return this._lastAcknowledgedVersion;
}
private set lastAcknowledgedVersion(version: number) {
this._lastAcknowledgedVersion = version;
this.repository.saveMetadata({ lastAcknowledgedVersion: version });
} }
private server: WebSocket | null = null; private server: WebSocket | null = null;
@ -38,15 +127,33 @@ export class ExcalidrawSyncClient {
private isConnecting: { done: (error?: Error) => void } | null = null; private isConnecting: { done: (error?: Error) => void } | null = null;
constructor( private constructor(
api: ExcalidrawImperativeAPI, api: ExcalidrawImperativeAPI,
roomId: string = ExcalidrawSyncClient.ROOM_ID, repository: MetadataRepository,
queue: SyncQueue,
options: { roomId: string; lastAcknowledgedVersion: number },
) { ) {
this.api = api; this.api = api;
this.roomId = roomId; this.repository = repository;
this.queue = queue;
this.roomId = options.roomId;
this.lastAcknowledgedVersion = options.lastAcknowledgedVersion;
}
// CFDO: persist in idb public static async create(
this.lastAcknowledgedVersion = 0; api: ExcalidrawImperativeAPI,
repository: IncrementsRepository & MetadataRepository,
roomId: string = SyncClient.ROOM_ID,
) {
const [queue, metadata] = await Promise.all([
SyncQueue.create(repository),
repository.loadMetadata(),
]);
return new SyncClient(api, repository, queue, {
roomId,
lastAcknowledgedVersion: metadata?.lastAcknowledgedVersion ?? 0,
});
} }
// CFDO: throttle does not work that well here (after some period it tries to reconnect too often) // CFDO: throttle does not work that well here (after some period it tries to reconnect too often)
@ -74,7 +181,7 @@ export class ExcalidrawSyncClient {
return await new Promise<void>((resolve, reject) => { return await new Promise<void>((resolve, reject) => {
this.server = new WebSocket( this.server = new WebSocket(
`${ExcalidrawSyncClient.HOST_URL}/connect?roomId=${this.roomId}`, `${SyncClient.HOST_URL}/connect?roomId=${this.roomId}`,
); );
// wait for 10 seconds before timing out // wait for 10 seconds before timing out
@ -103,7 +210,7 @@ export class ExcalidrawSyncClient {
this.disconnect(e as Error); this.disconnect(e as Error);
} }
}, },
ExcalidrawSyncClient.RECONNECT_INTERVAL, SyncClient.RECONNECT_INTERVAL,
{ leading: true }, { leading: true },
); );
@ -125,7 +232,7 @@ export class ExcalidrawSyncClient {
this.reconnect(); this.reconnect();
} }
}, },
ExcalidrawSyncClient.RECONNECT_INTERVAL, SyncClient.RECONNECT_INTERVAL,
{ leading: true }, { leading: true },
); );
@ -145,8 +252,8 @@ export class ExcalidrawSyncClient {
// resolve the current connection // resolve the current connection
this.isConnecting.done(); this.isConnecting.done();
// initiate pull // CFDO: hack to pull everything for on init
this.pull(); this.pull(0);
}; };
private onClose = (event: CloseEvent) => { private onClose = (event: CloseEvent) => {
@ -185,43 +292,36 @@ export class ExcalidrawSyncClient {
} }
}; };
private pull = (): void => { private pull(sinceVersion?: number): void {
this.send({ this.send({
type: "pull", type: "pull",
payload: { payload: {
lastAcknowledgedVersion: this.lastAcknowledgedVersion, lastAcknowledgedVersion: sinceVersion ?? this.lastAcknowledgedVersion,
}, },
}); });
}; }
public push = ( public push(
type: "durable" | "ephemeral" = "durable", type: "durable" | "ephemeral" = "durable",
changes: Array<CLIENT_CHANGE> = [], ...increments: Array<CLIENT_INCREMENT>
): void => { ): void {
const payload: PUSH_PAYLOAD = { type, changes: [] }; const payload: PUSH_PAYLOAD = { type, increments: [] };
if (type === "durable") { if (type === "durable") {
// CFDO: persist in idb (with insertion order) this.queue.add(...increments);
for (const change of changes) { // batch all (already) queued increments
this.queuedChanges.set(change.id, { payload.increments = this.queue.getAll();
queuedAt: Date.now(),
change,
});
}
// batch all queued changes
payload.changes = this.localChanges;
} else { } else {
payload.changes = changes; payload.increments = increments;
} }
if (payload.changes.length > 0) { if (payload.increments.length > 0) {
this.send({ this.send({
type: "push", type: "push",
payload, payload,
}); });
} }
}; }
public relay(buffer: ArrayBuffer): void { public relay(buffer: ArrayBuffer): void {
this.send({ this.send({
@ -230,60 +330,67 @@ export class ExcalidrawSyncClient {
}); });
} }
// CFDO: refactor by applying all operations to store, not to the elements // CFDO: should be flushed once regular push / pull goes through
private handleAcknowledged(payload: { changes: Array<SERVER_CHANGE> }) { private debouncedPush = (ms: number = 1000) =>
const { changes: remoteChanges } = payload; debounce(this.push, ms, { leading: true, trailing: false });
const oldAcknowledgedVersion = this.lastAcknowledgedVersion; private debouncedPull = (ms: number = 1000) =>
debounce(this.pull, ms, { leading: true, trailing: false });
// CFDO: refactor by applying all operations to store, not to the elements
private handleAcknowledged(payload: { increments: Array<SERVER_INCREMENT> }) {
let nextAcknowledgedVersion = this.lastAcknowledgedVersion;
let elements = new Map( let elements = new Map(
// CFDO: retrieve the map already
this.api.getSceneElementsIncludingDeleted().map((el) => [el.id, el]), this.api.getSceneElementsIncludingDeleted().map((el) => [el.id, el]),
) as SceneElementsMap; ) as SceneElementsMap;
try { try {
// apply remote changes const { increments: remoteIncrements } = payload;
for (const remoteChange of remoteChanges) {
if (this.queuedChanges.has(remoteChange.id)) {
const { change, queuedAt } = this.queuedChanges.get(remoteChange.id)!;
this.acknowledgedChanges.push(change);
console.info(
`Acknowledged change "${remoteChange.id}" after ${
Date.now() - queuedAt
}ms`,
);
// local change acknowledge by the server, safe to remove
this.queuedChanges.delete(remoteChange.id);
} else {
// CFDO: we might not need to be that strict here
if (this.lastAcknowledgedVersion + 1 !== remoteChange.version) {
throw new Error(
`Received out of order change, expected "${
this.lastAcknowledgedVersion + 1
}", but received "${remoteChange.version}"`,
);
}
const change = ElementsChange.load(remoteChange.payload); // apply remote increments
[elements] = change.applyTo( for (const { id, version, payload } of remoteIncrements.sort((a, b) =>
elements, a.version <= b.version ? -1 : 1,
this.api.store.snapshot.elements, )) {
); // CFDO: temporary to load all increments on init
this.acknowledgedChanges.push(change); this.acknowledgedIncrementsMap.set(id, StoreIncrement.load(payload));
// local increment shall not have to be applied again
if (this.queue.has(id)) {
this.queue.remove(id);
continue;
} }
this.lastAcknowledgedVersion = remoteChange.version; // we've already applied this increment
if (version <= nextAcknowledgedVersion) {
continue;
}
if (version === nextAcknowledgedVersion + 1) {
nextAcknowledgedVersion = version;
} else {
// it's fine to apply increments our of order,
// as they are idempontent, so that we can re-apply them again,
// as long as we don't mark them as acknowledged
console.debug(
`Received out of order increment, expected "${
nextAcknowledgedVersion + 1
}", but received "${version}"`,
);
}
// apply remote increment with higher version than the last acknowledged one
const remoteIncrement = StoreIncrement.load(payload);
[elements] = remoteIncrement.elementsChange.applyTo(
elements,
this.api.store.snapshot.elements,
);
} }
console.debug(`${now()} remote changes`, remoteChanges); // apply local increments
console.debug(`${now()} local changes`, this.localChanges); for (const localIncrement of this.queue.getAll()) {
console.debug( // CFDO: in theory only necessary when remote increments modified same element properties!
`${now()} acknowledged changes`, [elements] = localIncrement.elementsChange.applyTo(
this.acknowledgedChanges.slice(-remoteChanges.length),
);
// apply local changes
// CFDO: only necessary when remote changes modified same element properties!
for (const localChange of this.localChanges) {
[elements] = localChange.applyTo(
elements, elements,
this.api.store.snapshot.elements, this.api.store.snapshot.elements,
); );
@ -294,38 +401,31 @@ export class ExcalidrawSyncClient {
storeAction: "update", storeAction: "update",
}); });
// push all queued changes this.lastAcknowledgedVersion = nextAcknowledgedVersion;
this.push();
} catch (e) { } catch (e) {
console.error("Failed to apply acknowledged changes:", e); console.error("Failed to apply acknowledged increments:", e);
// rollback the last acknowledged version this.debouncedPull().call(this);
this.lastAcknowledgedVersion = oldAcknowledgedVersion; return;
// pull again to get the latest changes
this.pull();
} }
this.debouncedPush().call(this);
} }
private handleRejected(payload: { ids: Array<string>; message: string }) { private handleRejected(payload: { ids: Array<string>; message: string }) {
// handle rejected changes // handle rejected increments
console.error("Rejected message received:", payload); console.error("Rejected message received:", payload);
} }
private handleRelayed(payload: { changes: Array<CLIENT_CHANGE> }) { private handleRelayed(payload: { increments: Array<CLIENT_INCREMENT> }) {
// apply relayed changes / buffer // apply relayed increments / buffer
console.log("Relayed message received:", payload); console.log("Relayed message received:", payload);
} }
private send(message: { type: string; payload: any }): void { private send(message: { type: string; payload: any }): void {
if (!this.isConnected) { if (!this.isConnected) {
console.error("Can't send a message without an active connection!"); throw new Error("Can't send a message without an active connection!");
return;
} }
this.server?.send(JSON.stringify(message)); this.server?.send(JSON.stringify(message));
} }
} }
const now = () => {
const date = new Date();
return `[${date.getHours()}:${date.getMinutes()}:${date.getSeconds()}.${date.getMilliseconds()}]`;
};

20
packages/excalidraw/sync/protocol.ts
View file

@ -1,34 +1,34 @@
import type { ElementsChange } from "../change"; import type { StoreIncrement } from "../store";
export type RELAY_PAYLOAD = { buffer: ArrayBuffer }; export type RELAY_PAYLOAD = { buffer: ArrayBuffer };
export type PULL_PAYLOAD = { lastAcknowledgedVersion: number }; export type PULL_PAYLOAD = { lastAcknowledgedVersion: number };
export type PUSH_PAYLOAD = { export type PUSH_PAYLOAD = {
type: "durable" | "ephemeral"; type: "durable" | "ephemeral";
changes: Array<CLIENT_CHANGE>; increments: Array<CLIENT_INCREMENT>;
}; };
export type CLIENT_CHANGE = ElementsChange; export type CLIENT_INCREMENT = StoreIncrement;
export type CLIENT_MESSAGE = export type CLIENT_MESSAGE =
| { type: "relay"; payload: RELAY_PAYLOAD } | { type: "relay"; payload: RELAY_PAYLOAD }
| { type: "pull"; payload: PULL_PAYLOAD } | { type: "pull"; payload: PULL_PAYLOAD }
| { type: "push"; payload: PUSH_PAYLOAD }; | { type: "push"; payload: PUSH_PAYLOAD };
export type SERVER_CHANGE = { id: string; version: number; payload: string }; export type SERVER_INCREMENT = { id: string; version: number; payload: string };
export type SERVER_MESSAGE = export type SERVER_MESSAGE =
| { | {
type: "relayed"; type: "relayed";
payload: { changes: Array<CLIENT_CHANGE> } | RELAY_PAYLOAD; payload: { increments: Array<CLIENT_INCREMENT> } | RELAY_PAYLOAD;
} }
| { type: "acknowledged"; payload: { changes: Array<SERVER_CHANGE> } } | { type: "acknowledged"; payload: { increments: Array<SERVER_INCREMENT> } }
| { | {
type: "rejected"; type: "rejected";
payload: { changes: Array<CLIENT_CHANGE>; message: string }; payload: { increments: Array<CLIENT_INCREMENT>; message: string };
}; };
export interface ChangesRepository { export interface IncrementsRepository {
saveAll(changes: Array<CLIENT_CHANGE>): Array<SERVER_CHANGE>; saveAll(increments: Array<CLIENT_INCREMENT>): Array<SERVER_INCREMENT>;
getSinceVersion(version: number): Array<SERVER_CHANGE>; getSinceVersion(version: number): Array<SERVER_INCREMENT>;
getLastVersion(): number; getLastVersion(): number;
} }

55
packages/excalidraw/sync/server.ts
View file

@ -2,13 +2,14 @@ import AsyncLock from "async-lock";
import { Utils } from "./utils"; import { Utils } from "./utils";
import type { import type {
ChangesRepository, IncrementsRepository,
CLIENT_CHANGE, CLIENT_INCREMENT,
CLIENT_MESSAGE, CLIENT_MESSAGE,
PULL_PAYLOAD, PULL_PAYLOAD,
PUSH_PAYLOAD, PUSH_PAYLOAD,
RELAY_PAYLOAD, RELAY_PAYLOAD,
SERVER_MESSAGE, SERVER_MESSAGE,
SERVER_INCREMENT,
} from "./protocol"; } from "./protocol";
// CFDO: message could be binary (cbor, protobuf, etc.) // CFDO: message could be binary (cbor, protobuf, etc.)
@ -20,7 +21,7 @@ export class ExcalidrawSyncServer {
private readonly lock: AsyncLock = new AsyncLock(); private readonly lock: AsyncLock = new AsyncLock();
private readonly sessions: Set<WebSocket> = new Set(); private readonly sessions: Set<WebSocket> = new Set();
constructor(private readonly changesRepository: ChangesRepository) {} constructor(private readonly incrementsRepository: IncrementsRepository) {}
public onConnect(client: WebSocket) { public onConnect(client: WebSocket) {
this.sessions.add(client); this.sessions.add(client);
@ -59,16 +60,11 @@ export class ExcalidrawSyncServer {
// CFDO: test for invalid payload // CFDO: test for invalid payload
const lastAcknowledgedClientVersion = payload.lastAcknowledgedVersion; const lastAcknowledgedClientVersion = payload.lastAcknowledgedVersion;
const lastAcknowledgedServerVersion = const lastAcknowledgedServerVersion =
this.changesRepository.getLastVersion(); this.incrementsRepository.getLastVersion();
const versionΔ = const versionΔ =
lastAcknowledgedServerVersion - lastAcknowledgedClientVersion; lastAcknowledgedServerVersion - lastAcknowledgedClientVersion;
if (versionΔ === 0) {
console.info(`Client is up to date!`);
return;
}
if (versionΔ < 0) { if (versionΔ < 0) {
// CFDO: restore the client from the snapshot / deltas? // CFDO: restore the client from the snapshot / deltas?
console.error( console.error(
@ -77,38 +73,43 @@ export class ExcalidrawSyncServer {
return; return;
} }
const increments: SERVER_INCREMENT[] = [];
if (versionΔ > 0) { if (versionΔ > 0) {
// CFDO: for versioning we need deletions, but not for the "snapshot" update increments.push(
const changes = this.changesRepository.getSinceVersion( ...this.incrementsRepository.getSinceVersion(
lastAcknowledgedClientVersion, lastAcknowledgedClientVersion,
),
); );
this.send(client, {
type: "acknowledged",
payload: {
changes,
},
});
} }
this.send(client, {
type: "acknowledged",
payload: {
increments,
},
});
} }
private push(client: WebSocket, payload: PUSH_PAYLOAD) { private push(client: WebSocket, payload: PUSH_PAYLOAD) {
const { type, changes } = payload; const { type, increments } = payload;
switch (type) { switch (type) {
case "ephemeral": case "ephemeral":
return this.relay(client, { changes }); return this.relay(client, { increments });
case "durable": case "durable":
const [acknowledged, error] = Utils.try(() => { // CFDO: try to apply the increments to the snapshot
// CFDO: try to apply the changes to the snapshot const [acknowledged, error] = Utils.try(() =>
return this.changesRepository.saveAll(changes); this.incrementsRepository.saveAll(increments),
}); );
if (error) { if (error) {
// everything should be automatically rolled-back -> double-check
return this.send(client, { return this.send(client, {
type: "rejected", type: "rejected",
payload: { payload: {
message: error.message, message: error.message,
changes, increments,
}, },
}); });
} }
@ -116,7 +117,7 @@ export class ExcalidrawSyncServer {
return this.broadcast({ return this.broadcast({
type: "acknowledged", type: "acknowledged",
payload: { payload: {
changes: acknowledged, increments: acknowledged,
}, },
}); });
default: default:
@ -126,7 +127,7 @@ export class ExcalidrawSyncServer {
private relay( private relay(
client: WebSocket, client: WebSocket,
payload: { changes: Array<CLIENT_CHANGE> } | RELAY_PAYLOAD, payload: { increments: Array<CLIENT_INCREMENT> } | RELAY_PAYLOAD,
) { ) {
return this.broadcast( return this.broadcast(
{ {

1957
packages/excalidraw/tests/__snapshots__/history.test.tsx.snap
View file

File diff suppressed because it is too large Load diff

2
packages/excalidraw/tests/history.test.tsx
View file

@ -99,7 +99,7 @@ describe("history", () => {
API.setElements([rect]); API.setElements([rect]);
const corrupedEntry = new StoreIncrement( const corrupedEntry = StoreIncrement.create(
ElementsChange.empty(), ElementsChange.empty(),
AppStateChange.empty(), AppStateChange.empty(),
); );

5
packages/excalidraw/utility-types.ts
View file

@ -65,3 +65,8 @@ export type MakeBrand<T extends string> = {
/** Maybe just promise or already fulfilled one! */ /** Maybe just promise or already fulfilled one! */
export type MaybePromise<T> = T | Promise<T>; export type MaybePromise<T> = T | Promise<T>;
/** Strip all the methods or functions from a type */
export type DTO<T> = {
[K in keyof T as T[K] extends Function ? never : K]: T[K];
};

9
yarn.lock
View file

@ -3383,6 +3383,13 @@
resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee" resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee"
integrity sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ== integrity sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==
"@types/lodash.debounce@4.0.9":
version "4.0.9"
resolved "https://registry.yarnpkg.com/@types/lodash.debounce/-/lodash.debounce-4.0.9.tgz#0f5f21c507bce7521b5e30e7a24440975ac860a5"
integrity sha512-Ma5JcgTREwpLRwMM+XwBR7DaWe96nC38uCBDFKZWbNKD+osjVzdpnUSwBcqCptrp16sSOLBAUb50Car5I0TCsQ==
dependencies:
"@types/lodash" "*"
"@types/lodash.throttle@4.1.7": "@types/lodash.throttle@4.1.7":
version "4.1.7" version "4.1.7"
resolved "https://registry.yarnpkg.com/@types/lodash.throttle/-/lodash.throttle-4.1.7.tgz#4ef379eb4f778068022310ef166625f420b6ba58" resolved "https://registry.yarnpkg.com/@types/lodash.throttle/-/lodash.throttle-4.1.7.tgz#4ef379eb4f778068022310ef166625f420b6ba58"
@ -7964,7 +7971,7 @@ lodash.camelcase@^4.3.0:
resolved "https://registry.yarnpkg.com/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz#b28aa6288a2b9fc651035c7711f65ab6190331a6" resolved "https://registry.yarnpkg.com/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz#b28aa6288a2b9fc651035c7711f65ab6190331a6"
integrity sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA== integrity sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==
lodash.debounce@^4.0.8: lodash.debounce@4.0.8, lodash.debounce@^4.0.8:
version "4.0.8" version "4.0.8"
resolved "https://registry.yarnpkg.com/lodash.debounce/-/lodash.debounce-4.0.8.tgz#82d79bff30a67c4005ffd5e2515300ad9ca4d7af" resolved "https://registry.yarnpkg.com/lodash.debounce/-/lodash.debounce-4.0.8.tgz#82d79bff30a67c4005ffd5e2515300ad9ca4d7af"
integrity sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow== integrity sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==