mirror of
https://github.com/excalidraw/excalidraw.git
synced 2025-05-03 10:00:07 -04:00
feat: rewrite d2c to not require token (#8269)
This commit is contained in:
David Luzar
GitHub
parent
fb4bb29aa5
commit
b5d7f5b4ba
19 changed files with 282 additions and 564 deletions
|
|
@ -22,7 +22,6 @@ import { t } from "../packages/excalidraw/i18n";
|
|||
import {
|
||||
Excalidraw,
|
||||
LiveCollaborationTrigger,
|
||||
TTDDialog,
|
||||
TTDDialogTrigger,
|
||||
StoreAction,
|
||||
reconcileElements,
|
||||
|
|
@ -121,6 +120,7 @@ import {
|
|||
import { appThemeAtom, useHandleAppTheme } from "./useHandleAppTheme";
|
||||
import { getPreferredLanguage } from "./app-language/language-detector";
|
||||
import { useAppLangCode } from "./app-language/language-state";
|
||||
import { AIComponents } from "./components/AI";
|
||||
|
||||
polyfill();
|
||||
|
||||
|
|
@ -846,63 +846,8 @@ const ExcalidrawWrapper = () => {
|
|||
)}
|
||||
</OverwriteConfirmDialog>
|
||||
<AppFooter />
|
||||
<TTDDialog
|
||||
onTextSubmit={async (input) => {
|
||||
try {
|
||||
const response = await fetch(
|
||||
`${
|
||||
import.meta.env.VITE_APP_AI_BACKEND
|
||||
}/v1/ai/text-to-diagram/generate`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
Accept: "application/json",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ prompt: input }),
|
||||
},
|
||||
);
|
||||
{excalidrawAPI && <AIComponents excalidrawAPI={excalidrawAPI} />}
|
||||
|
||||
const rateLimit = response.headers.has("X-Ratelimit-Limit")
|
||||
? parseInt(response.headers.get("X-Ratelimit-Limit") || "0", 10)
|
||||
: undefined;
|
||||
|
||||
const rateLimitRemaining = response.headers.has(
|
||||
"X-Ratelimit-Remaining",
|
||||
)
|
||||
? parseInt(
|
||||
response.headers.get("X-Ratelimit-Remaining") || "0",
|
||||
10,
|
||||
)
|
||||
: undefined;
|
||||
|
||||
const json = await response.json();
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 429) {
|
||||
return {
|
||||
rateLimit,
|
||||
rateLimitRemaining,
|
||||
error: new Error(
|
||||
"Too many requests today, please try again tomorrow!",
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error(json.message || "Generation failed...");
|
||||
}
|
||||
|
||||
const generatedResponse = json.generatedResponse;
|
||||
if (!generatedResponse) {
|
||||
throw new Error("Generation failed...");
|
||||
}
|
||||
|
||||
return { generatedResponse, rateLimit, rateLimitRemaining };
|
||||
} catch (err: any) {
|
||||
throw new Error("Request failed");
|
||||
}
|
||||
}}
|
||||
/>
|
||||
<TTDDialogTrigger />
|
||||
{isCollaborating && isOffline && (
|
||||
<div className="collab-offline-warning">
|
||||
|
|
|
|||
159
excalidraw-app/components/AI.tsx
Normal file
159
excalidraw-app/components/AI.tsx
Normal file
|
|
@ -0,0 +1,159 @@
|
|||
import type { ExcalidrawImperativeAPI } from "../../packages/excalidraw/types";
|
||||
import {
|
||||
DiagramToCodePlugin,
|
||||
exportToBlob,
|
||||
getTextFromElements,
|
||||
MIME_TYPES,
|
||||
TTDDialog,
|
||||
} from "../../packages/excalidraw";
|
||||
import { getDataURL } from "../../packages/excalidraw/data/blob";
|
||||
import { safelyParseJSON } from "../../packages/excalidraw/utils";
|
||||
|
||||
export const AIComponents = ({
|
||||
excalidrawAPI,
|
||||
}: {
|
||||
excalidrawAPI: ExcalidrawImperativeAPI;
|
||||
}) => {
|
||||
return (
|
||||
<>
|
||||
<DiagramToCodePlugin
|
||||
generate={async ({ frame, children }) => {
|
||||
const appState = excalidrawAPI.getAppState();
|
||||
|
||||
const blob = await exportToBlob({
|
||||
elements: children,
|
||||
appState: {
|
||||
...appState,
|
||||
exportBackground: true,
|
||||
viewBackgroundColor: appState.viewBackgroundColor,
|
||||
},
|
||||
exportingFrame: frame,
|
||||
files: excalidrawAPI.getFiles(),
|
||||
mimeType: MIME_TYPES.jpg,
|
||||
});
|
||||
|
||||
const dataURL = await getDataURL(blob);
|
||||
|
||||
const textFromFrameChildren = getTextFromElements(children);
|
||||
|
||||
const response = await fetch(
|
||||
`${
|
||||
import.meta.env.VITE_APP_AI_BACKEND
|
||||
}/v1/ai/diagram-to-code/generate`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
Accept: "application/json",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
texts: textFromFrameChildren,
|
||||
image: dataURL,
|
||||
theme: appState.theme,
|
||||
}),
|
||||
},
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
const text = await response.text();
|
||||
const errorJSON = safelyParseJSON(text);
|
||||
|
||||
if (!errorJSON) {
|
||||
throw new Error(text);
|
||||
}
|
||||
|
||||
if (errorJSON.statusCode === 429) {
|
||||
return {
|
||||
html: `<html>
|
||||
<body style="margin: 0; text-align: center">
|
||||
<div style="display: flex; align-items: center; justify-content: center; flex-direction: column; height: 100vh; padding: 0 60px">
|
||||
<div style="color:red">Too many requests today,</br>please try again tomorrow!</div>
|
||||
</br>
|
||||
</br>
|
||||
<div>You can also try <a href="${
|
||||
import.meta.env.VITE_APP_PLUS_LP
|
||||
}/plus?utm_source=excalidraw&utm_medium=app&utm_content=d2c" target="_blank" rel="noreferrer noopener">Excalidraw+</a> to get more requests.</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>`,
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error(errorJSON.message || text);
|
||||
}
|
||||
|
||||
try {
|
||||
const { html } = await response.json();
|
||||
|
||||
if (!html) {
|
||||
throw new Error("Generation failed (invalid response)");
|
||||
}
|
||||
return {
|
||||
html,
|
||||
};
|
||||
} catch (error: any) {
|
||||
throw new Error("Generation failed (invalid response)");
|
||||
}
|
||||
}}
|
||||
/>
|
||||
|
||||
<TTDDialog
|
||||
onTextSubmit={async (input) => {
|
||||
try {
|
||||
const response = await fetch(
|
||||
`${
|
||||
import.meta.env.VITE_APP_AI_BACKEND
|
||||
}/v1/ai/text-to-diagram/generate`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
Accept: "application/json",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ prompt: input }),
|
||||
},
|
||||
);
|
||||
|
||||
const rateLimit = response.headers.has("X-Ratelimit-Limit")
|
||||
? parseInt(response.headers.get("X-Ratelimit-Limit") || "0", 10)
|
||||
: undefined;
|
||||
|
||||
const rateLimitRemaining = response.headers.has(
|
||||
"X-Ratelimit-Remaining",
|
||||
)
|
||||
? parseInt(
|
||||
response.headers.get("X-Ratelimit-Remaining") || "0",
|
||||
10,
|
||||
)
|
||||
: undefined;
|
||||
|
||||
const json = await response.json();
|
||||
|
||||
if (!response.ok) {
|
||||
if (response.status === 429) {
|
||||
return {
|
||||
rateLimit,
|
||||
rateLimitRemaining,
|
||||
error: new Error(
|
||||
"Too many requests today, please try again tomorrow!",
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error(json.message || "Generation failed...");
|
||||
}
|
||||
|
||||
const generatedResponse = json.generatedResponse;
|
||||
if (!generatedResponse) {
|
||||
throw new Error("Generation failed...");
|
||||
}
|
||||
|
||||
return { generatedResponse, rateLimit, rateLimitRemaining };
|
||||
} catch (err: any) {
|
||||
throw new Error("Request failed");
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
|
@ -10,7 +10,7 @@ import {
|
|||
} from "../clipboard";
|
||||
import { actionDeleteSelected } from "./actionDeleteSelected";
|
||||
import { exportCanvas, prepareElementsForExport } from "../data/index";
|
||||
import { isTextElement } from "../element";
|
||||
import { getTextFromElements, isTextElement } from "../element";
|
||||
import { t } from "../i18n";
|
||||
import { isFirefox } from "../constants";
|
||||
import { DuplicateIcon, cutIcon, pngIcon, svgIcon } from "../components/icons";
|
||||
|
|
@ -239,16 +239,8 @@ export const copyText = register({
|
|||
includeBoundTextElement: true,
|
||||
});
|
||||
|
||||
const text = selectedElements
|
||||
.reduce((acc: string[], element) => {
|
||||
if (isTextElement(element)) {
|
||||
acc.push(element.text);
|
||||
}
|
||||
return acc;
|
||||
}, [])
|
||||
.join("\n\n");
|
||||
try {
|
||||
copyTextToSystemClipboard(text);
|
||||
copyTextToSystemClipboard(getTextFromElements(selectedElements));
|
||||
} catch (e) {
|
||||
throw new Error(t("errors.copyToSystemClipboardFailed"));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -45,7 +45,6 @@ import {
|
|||
frameToolIcon,
|
||||
mermaidLogoIcon,
|
||||
laserPointerToolIcon,
|
||||
OpenAIIcon,
|
||||
MagicIcon,
|
||||
} from "./icons";
|
||||
import { KEYS } from "../keys";
|
||||
|
|
@ -400,7 +399,7 @@ export const ShapesSwitcher = ({
|
|||
>
|
||||
{t("toolBar.mermaidToExcalidraw")}
|
||||
</DropdownMenu.Item>
|
||||
{app.props.aiEnabled !== false && (
|
||||
{app.props.aiEnabled !== false && app.plugins.diagramToCode && (
|
||||
<>
|
||||
<DropdownMenu.Item
|
||||
onSelect={() => app.onMagicframeToolSelect()}
|
||||
|
|
@ -410,20 +409,6 @@ export const ShapesSwitcher = ({
|
|||
{t("toolBar.magicframe")}
|
||||
<DropdownMenu.Item.Badge>AI</DropdownMenu.Item.Badge>
|
||||
</DropdownMenu.Item>
|
||||
<DropdownMenu.Item
|
||||
onSelect={() => {
|
||||
trackEvent("ai", "open-settings", "d2c");
|
||||
app.setOpenDialog({
|
||||
name: "settings",
|
||||
source: "settings",
|
||||
tab: "diagram-to-code",
|
||||
});
|
||||
}}
|
||||
icon={OpenAIIcon}
|
||||
data-testid="toolbar-magicSettings"
|
||||
>
|
||||
{t("toolBar.magicSettings")}
|
||||
</DropdownMenu.Item>
|
||||
</>
|
||||
)}
|
||||
</DropdownMenu.Content>
|
||||
|
|
|
|||
|
|
@ -83,7 +83,6 @@ import {
|
|||
ZOOM_STEP,
|
||||
POINTER_EVENTS,
|
||||
TOOL_TYPE,
|
||||
EDITOR_LS_KEYS,
|
||||
isIOS,
|
||||
supportsResizeObserver,
|
||||
DEFAULT_COLLISION_THRESHOLD,
|
||||
|
|
@ -183,6 +182,7 @@ import type {
|
|||
ExcalidrawIframeElement,
|
||||
ExcalidrawEmbeddableElement,
|
||||
Ordered,
|
||||
MagicGenerationData,
|
||||
ExcalidrawNonSelectionElement,
|
||||
ExcalidrawArrowElement,
|
||||
} from "../element/types";
|
||||
|
|
@ -257,6 +257,7 @@ import type {
|
|||
UnsubscribeCallback,
|
||||
EmbedsValidationStatus,
|
||||
ElementsPendingErasure,
|
||||
GenerateDiagramToCode,
|
||||
NullableGridSize,
|
||||
} from "../types";
|
||||
import {
|
||||
|
|
@ -405,13 +406,9 @@ import {
|
|||
} from "../cursor";
|
||||
import { Emitter } from "../emitter";
|
||||
import { ElementCanvasButtons } from "../element/ElementCanvasButtons";
|
||||
import type { MagicCacheData } from "../data/magic";
|
||||
import { diagramToHTML } from "../data/magic";
|
||||
import { exportToBlob } from "../../utils/export";
|
||||
import { COLOR_PALETTE } from "../colors";
|
||||
import { ElementCanvasButton } from "./MagicButton";
|
||||
import { MagicIcon, copyIcon, fullscreenIcon } from "./icons";
|
||||
import { EditorLocalStorage } from "../data/EditorLocalStorage";
|
||||
import FollowMode from "./FollowMode/FollowMode";
|
||||
import { Store, StoreAction } from "../store";
|
||||
import { AnimationFrameHandler } from "../animation-frame-handler";
|
||||
|
|
@ -1018,7 +1015,7 @@ class App extends React.Component<AppProps, AppState> {
|
|||
if (isIframeElement(el)) {
|
||||
src = null;
|
||||
|
||||
const data: MagicCacheData = (el.customData?.generationData ??
|
||||
const data: MagicGenerationData = (el.customData?.generationData ??
|
||||
this.magicGenerations.get(el.id)) || {
|
||||
status: "error",
|
||||
message: "No generation data",
|
||||
|
|
@ -1559,10 +1556,6 @@ class App extends React.Component<AppProps, AppState> {
|
|||
}
|
||||
app={this}
|
||||
isCollaborating={this.props.isCollaborating}
|
||||
openAIKey={this.OPENAI_KEY}
|
||||
isOpenAIKeyPersisted={this.OPENAI_KEY_IS_PERSISTED}
|
||||
onOpenAIAPIKeyChange={this.onOpenAIKeyChange}
|
||||
onMagicSettingsConfirm={this.onMagicSettingsConfirm}
|
||||
>
|
||||
{this.props.children}
|
||||
</LayerUI>
|
||||
|
|
@ -1807,7 +1800,7 @@ class App extends React.Component<AppProps, AppState> {
|
|||
|
||||
private magicGenerations = new Map<
|
||||
ExcalidrawIframeElement["id"],
|
||||
MagicCacheData
|
||||
MagicGenerationData
|
||||
>();
|
||||
|
||||
private updateMagicGeneration = ({
|
||||
|
|
@ -1815,7 +1808,7 @@ class App extends React.Component<AppProps, AppState> {
|
|||
data,
|
||||
}: {
|
||||
frameElement: ExcalidrawIframeElement;
|
||||
data: MagicCacheData;
|
||||
data: MagicGenerationData;
|
||||
}) => {
|
||||
if (data.status === "pending") {
|
||||
// We don't wanna persist pending state to storage. It should be in-app
|
||||
|
|
@ -1838,31 +1831,26 @@ class App extends React.Component<AppProps, AppState> {
|
|||
this.triggerRender();
|
||||
};
|
||||
|
||||
private getTextFromElements(elements: readonly ExcalidrawElement[]) {
|
||||
const text = elements
|
||||
.reduce((acc: string[], element) => {
|
||||
if (isTextElement(element)) {
|
||||
acc.push(element.text);
|
||||
}
|
||||
return acc;
|
||||
}, [])
|
||||
.join("\n\n");
|
||||
return text;
|
||||
public plugins: {
|
||||
diagramToCode?: {
|
||||
generate: GenerateDiagramToCode;
|
||||
};
|
||||
} = {};
|
||||
|
||||
public setPlugins(plugins: Partial<App["plugins"]>) {
|
||||
Object.assign(this.plugins, plugins);
|
||||
}
|
||||
|
||||
private async onMagicFrameGenerate(
|
||||
magicFrame: ExcalidrawMagicFrameElement,
|
||||
source: "button" | "upstream",
|
||||
) {
|
||||
if (!this.OPENAI_KEY) {
|
||||
const generateDiagramToCode = this.plugins.diagramToCode?.generate;
|
||||
|
||||
if (!generateDiagramToCode) {
|
||||
this.setState({
|
||||
openDialog: {
|
||||
name: "settings",
|
||||
tab: "diagram-to-code",
|
||||
source: "generation",
|
||||
},
|
||||
errorMessage: "No diagram to code plugin found",
|
||||
});
|
||||
trackEvent("ai", "generate (missing key)", "d2c");
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -1901,46 +1889,16 @@ class App extends React.Component<AppProps, AppState> {
|
|||
selectedElementIds: { [frameElement.id]: true },
|
||||
});
|
||||
|
||||
const blob = await exportToBlob({
|
||||
elements: this.scene.getNonDeletedElements(),
|
||||
appState: {
|
||||
...this.state,
|
||||
exportBackground: true,
|
||||
viewBackgroundColor: this.state.viewBackgroundColor,
|
||||
},
|
||||
exportingFrame: magicFrame,
|
||||
files: this.files,
|
||||
});
|
||||
|
||||
const dataURL = await getDataURL(blob);
|
||||
|
||||
const textFromFrameChildren = this.getTextFromElements(magicFrameChildren);
|
||||
|
||||
trackEvent("ai", "generate (start)", "d2c");
|
||||
|
||||
const result = await diagramToHTML({
|
||||
image: dataURL,
|
||||
apiKey: this.OPENAI_KEY,
|
||||
text: textFromFrameChildren,
|
||||
theme: this.state.theme,
|
||||
try {
|
||||
const { html } = await generateDiagramToCode({
|
||||
frame: magicFrame,
|
||||
children: magicFrameChildren,
|
||||
});
|
||||
|
||||
if (!result.ok) {
|
||||
trackEvent("ai", "generate (failed)", "d2c");
|
||||
console.error(result.error);
|
||||
this.updateMagicGeneration({
|
||||
frameElement,
|
||||
data: {
|
||||
status: "error",
|
||||
code: "ERR_OAI",
|
||||
message: result.error?.message || "Unknown error during generation",
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
trackEvent("ai", "generate (success)", "d2c");
|
||||
|
||||
if (result.choices[0].message.content == null) {
|
||||
if (!html.trim()) {
|
||||
this.updateMagicGeneration({
|
||||
frameElement,
|
||||
data: {
|
||||
|
|
@ -1952,17 +1910,29 @@ class App extends React.Component<AppProps, AppState> {
|
|||
return;
|
||||
}
|
||||
|
||||
const message = result.choices[0].message.content;
|
||||
|
||||
const html = message.slice(
|
||||
message.indexOf("<!DOCTYPE html>"),
|
||||
message.indexOf("</html>") + "</html>".length,
|
||||
);
|
||||
const parsedHtml =
|
||||
html.includes("<!DOCTYPE html>") && html.includes("</html>")
|
||||
? html.slice(
|
||||
html.indexOf("<!DOCTYPE html>"),
|
||||
html.indexOf("</html>") + "</html>".length,
|
||||
)
|
||||
: html;
|
||||
|
||||
this.updateMagicGeneration({
|
||||
frameElement,
|
||||
data: { status: "done", html },
|
||||
data: { status: "done", html: parsedHtml },
|
||||
});
|
||||
} catch (error: any) {
|
||||
trackEvent("ai", "generate (failed)", "d2c");
|
||||
this.updateMagicGeneration({
|
||||
frameElement,
|
||||
data: {
|
||||
status: "error",
|
||||
code: "ERR_OAI",
|
||||
message: error.message || "Unknown error during generation",
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private onIframeSrcCopy(element: ExcalidrawIframeElement) {
|
||||
|
|
@ -1976,70 +1946,7 @@ class App extends React.Component<AppProps, AppState> {
|
|||
}
|
||||
}
|
||||
|
||||
private OPENAI_KEY: string | null = EditorLocalStorage.get(
|
||||
EDITOR_LS_KEYS.OAI_API_KEY,
|
||||
);
|
||||
private OPENAI_KEY_IS_PERSISTED: boolean =
|
||||
EditorLocalStorage.has(EDITOR_LS_KEYS.OAI_API_KEY) || false;
|
||||
|
||||
private onOpenAIKeyChange = (
|
||||
openAIKey: string | null,
|
||||
shouldPersist: boolean,
|
||||
) => {
|
||||
this.OPENAI_KEY = openAIKey || null;
|
||||
if (shouldPersist) {
|
||||
const didPersist = EditorLocalStorage.set(
|
||||
EDITOR_LS_KEYS.OAI_API_KEY,
|
||||
openAIKey,
|
||||
);
|
||||
this.OPENAI_KEY_IS_PERSISTED = didPersist;
|
||||
} else {
|
||||
this.OPENAI_KEY_IS_PERSISTED = false;
|
||||
}
|
||||
};
|
||||
|
||||
private onMagicSettingsConfirm = (
|
||||
apiKey: string,
|
||||
shouldPersist: boolean,
|
||||
source: "tool" | "generation" | "settings",
|
||||
) => {
|
||||
this.OPENAI_KEY = apiKey || null;
|
||||
this.onOpenAIKeyChange(this.OPENAI_KEY, shouldPersist);
|
||||
|
||||
if (source === "settings") {
|
||||
return;
|
||||
}
|
||||
|
||||
const selectedElements = this.scene.getSelectedElements({
|
||||
selectedElementIds: this.state.selectedElementIds,
|
||||
});
|
||||
|
||||
if (apiKey) {
|
||||
if (selectedElements.length) {
|
||||
this.onMagicframeToolSelect();
|
||||
} else {
|
||||
this.setActiveTool({ type: "magicframe" });
|
||||
}
|
||||
} else if (!isMagicFrameElement(selectedElements[0])) {
|
||||
// even if user didn't end up setting api key, let's pick the tool
|
||||
// so they can draw up a frame and move forward
|
||||
this.setActiveTool({ type: "magicframe" });
|
||||
}
|
||||
};
|
||||
|
||||
public onMagicframeToolSelect = () => {
|
||||
if (!this.OPENAI_KEY) {
|
||||
this.setState({
|
||||
openDialog: {
|
||||
name: "settings",
|
||||
tab: "diagram-to-code",
|
||||
source: "tool",
|
||||
},
|
||||
});
|
||||
trackEvent("ai", "tool-select (missing key)", "d2c");
|
||||
return;
|
||||
}
|
||||
|
||||
const selectedElements = this.scene.getSelectedElements({
|
||||
selectedElementIds: this.state.selectedElementIds,
|
||||
});
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
import { useLayoutEffect } from "react";
|
||||
import { useApp } from "../App";
|
||||
import type { GenerateDiagramToCode } from "../../types";
|
||||
|
||||
export const DiagramToCodePlugin = (props: {
|
||||
generate: GenerateDiagramToCode;
|
||||
}) => {
|
||||
const app = useApp();
|
||||
|
||||
useLayoutEffect(() => {
|
||||
app.setPlugins({
|
||||
diagramToCode: { generate: props.generate },
|
||||
});
|
||||
}, [app, props.generate]);
|
||||
|
||||
return null;
|
||||
};
|
||||
|
|
@ -60,7 +60,6 @@ import { mutateElement } from "../element/mutateElement";
|
|||
import { ShapeCache } from "../scene/ShapeCache";
|
||||
import Scene from "../scene/Scene";
|
||||
import { LaserPointerButton } from "./LaserPointerButton";
|
||||
import { MagicSettings } from "./MagicSettings";
|
||||
import { TTDDialog } from "./TTDDialog/TTDDialog";
|
||||
import { Stats } from "./Stats";
|
||||
import { actionToggleStats } from "../actions";
|
||||
|
|
@ -85,14 +84,6 @@ interface LayerUIProps {
|
|||
children?: React.ReactNode;
|
||||
app: AppClassProperties;
|
||||
isCollaborating: boolean;
|
||||
openAIKey: string | null;
|
||||
isOpenAIKeyPersisted: boolean;
|
||||
onOpenAIAPIKeyChange: (apiKey: string, shouldPersist: boolean) => void;
|
||||
onMagicSettingsConfirm: (
|
||||
apiKey: string,
|
||||
shouldPersist: boolean,
|
||||
source: "tool" | "generation" | "settings",
|
||||
) => void;
|
||||
}
|
||||
|
||||
const DefaultMainMenu: React.FC<{
|
||||
|
|
@ -149,10 +140,6 @@ const LayerUI = ({
|
|||
children,
|
||||
app,
|
||||
isCollaborating,
|
||||
openAIKey,
|
||||
isOpenAIKeyPersisted,
|
||||
onOpenAIAPIKeyChange,
|
||||
onMagicSettingsConfirm,
|
||||
}: LayerUIProps) => {
|
||||
const device = useDevice();
|
||||
const tunnels = useInitializeTunnels();
|
||||
|
|
@ -482,25 +469,6 @@ const LayerUI = ({
|
|||
}}
|
||||
/>
|
||||
)}
|
||||
{appState.openDialog?.name === "settings" && (
|
||||
<MagicSettings
|
||||
openAIKey={openAIKey}
|
||||
isPersisted={isOpenAIKeyPersisted}
|
||||
onChange={onOpenAIAPIKeyChange}
|
||||
onConfirm={(apiKey, shouldPersist) => {
|
||||
const source =
|
||||
appState.openDialog?.name === "settings"
|
||||
? appState.openDialog?.source
|
||||
: "settings";
|
||||
setAppState({ openDialog: null }, () => {
|
||||
onMagicSettingsConfirm(apiKey, shouldPersist, source);
|
||||
});
|
||||
}}
|
||||
onClose={() => {
|
||||
setAppState({ openDialog: null });
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
<ActiveConfirmDialog />
|
||||
<tunnels.OverwriteConfirmDialogTunnel.Out />
|
||||
{renderImageExportDialog()}
|
||||
|
|
|
|||
|
|
@ -1,18 +0,0 @@
|
|||
.excalidraw {
|
||||
.MagicSettings {
|
||||
.Island {
|
||||
height: 100%;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
}
|
||||
|
||||
.MagicSettings-confirm {
|
||||
padding: 0.5rem 1rem;
|
||||
}
|
||||
|
||||
.MagicSettings__confirm {
|
||||
margin-top: 2rem;
|
||||
margin-right: auto;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,160 +0,0 @@
|
|||
import { useState } from "react";
|
||||
import { Dialog } from "./Dialog";
|
||||
import { TextField } from "./TextField";
|
||||
import { MagicIcon, OpenAIIcon } from "./icons";
|
||||
import { FilledButton } from "./FilledButton";
|
||||
import { CheckboxItem } from "./CheckboxItem";
|
||||
import { KEYS } from "../keys";
|
||||
import { useUIAppState } from "../context/ui-appState";
|
||||
import { InlineIcon } from "./InlineIcon";
|
||||
import { Paragraph } from "./Paragraph";
|
||||
|
||||
import "./MagicSettings.scss";
|
||||
import TTDDialogTabs from "./TTDDialog/TTDDialogTabs";
|
||||
import { TTDDialogTab } from "./TTDDialog/TTDDialogTab";
|
||||
|
||||
export const MagicSettings = (props: {
|
||||
openAIKey: string | null;
|
||||
isPersisted: boolean;
|
||||
onChange: (key: string, shouldPersist: boolean) => void;
|
||||
onConfirm: (key: string, shouldPersist: boolean) => void;
|
||||
onClose: () => void;
|
||||
}) => {
|
||||
const [keyInputValue, setKeyInputValue] = useState(props.openAIKey || "");
|
||||
const [shouldPersist, setShouldPersist] = useState<boolean>(
|
||||
props.isPersisted,
|
||||
);
|
||||
|
||||
const appState = useUIAppState();
|
||||
|
||||
const onConfirm = () => {
|
||||
props.onConfirm(keyInputValue.trim(), shouldPersist);
|
||||
};
|
||||
|
||||
if (appState.openDialog?.name !== "settings") {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<Dialog
|
||||
onCloseRequest={() => {
|
||||
props.onClose();
|
||||
props.onConfirm(keyInputValue.trim(), shouldPersist);
|
||||
}}
|
||||
title={
|
||||
<div style={{ display: "flex" }}>
|
||||
Wireframe to Code (AI){" "}
|
||||
<div
|
||||
style={{
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
padding: "0.1rem 0.5rem",
|
||||
marginLeft: "1rem",
|
||||
fontSize: 14,
|
||||
borderRadius: "12px",
|
||||
background: "var(--color-promo)",
|
||||
color: "var(--color-surface-lowest)",
|
||||
}}
|
||||
>
|
||||
Experimental
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
className="MagicSettings"
|
||||
autofocus={false}
|
||||
>
|
||||
{/* <h2
|
||||
style={{
|
||||
margin: 0,
|
||||
fontSize: "1.25rem",
|
||||
paddingLeft: "2.5rem",
|
||||
}}
|
||||
>
|
||||
AI Settings
|
||||
</h2> */}
|
||||
<TTDDialogTabs dialog="settings" tab={appState.openDialog.tab}>
|
||||
{/* <TTDDialogTabTriggers>
|
||||
<TTDDialogTabTrigger tab="text-to-diagram">
|
||||
<InlineIcon icon={brainIcon} /> Text to diagram
|
||||
</TTDDialogTabTrigger>
|
||||
<TTDDialogTabTrigger tab="diagram-to-code">
|
||||
<InlineIcon icon={MagicIcon} /> Wireframe to code
|
||||
</TTDDialogTabTrigger>
|
||||
</TTDDialogTabTriggers> */}
|
||||
{/* <TTDDialogTab className="ttd-dialog-content" tab="text-to-diagram">
|
||||
TODO
|
||||
</TTDDialogTab> */}
|
||||
<TTDDialogTab
|
||||
// className="ttd-dialog-content"
|
||||
tab="diagram-to-code"
|
||||
>
|
||||
<Paragraph>
|
||||
For the diagram-to-code feature we use{" "}
|
||||
<InlineIcon icon={OpenAIIcon} />
|
||||
OpenAI.
|
||||
</Paragraph>
|
||||
<Paragraph>
|
||||
While the OpenAI API is in beta, its use is strictly limited — as
|
||||
such we require you use your own API key. You can create an{" "}
|
||||
<a
|
||||
href="https://platform.openai.com/login?launch"
|
||||
rel="noopener noreferrer"
|
||||
target="_blank"
|
||||
>
|
||||
OpenAI account
|
||||
</a>
|
||||
, add a small credit (5 USD minimum), and{" "}
|
||||
<a
|
||||
href="https://platform.openai.com/api-keys"
|
||||
rel="noopener noreferrer"
|
||||
target="_blank"
|
||||
>
|
||||
generate your own API key
|
||||
</a>
|
||||
.
|
||||
</Paragraph>
|
||||
<Paragraph>
|
||||
Your OpenAI key does not leave the browser, and you can also set
|
||||
your own limit in your OpenAI account dashboard if needed.
|
||||
</Paragraph>
|
||||
<TextField
|
||||
isRedacted
|
||||
value={keyInputValue}
|
||||
placeholder="Paste your API key here"
|
||||
label="OpenAI API key"
|
||||
onChange={(value) => {
|
||||
setKeyInputValue(value);
|
||||
props.onChange(value.trim(), shouldPersist);
|
||||
}}
|
||||
selectOnRender
|
||||
onKeyDown={(event) => event.key === KEYS.ENTER && onConfirm()}
|
||||
/>
|
||||
<Paragraph>
|
||||
By default, your API token is not persisted anywhere so you'll need
|
||||
to insert it again after reload. But, you can persist locally in
|
||||
your browser below.
|
||||
</Paragraph>
|
||||
|
||||
<CheckboxItem checked={shouldPersist} onChange={setShouldPersist}>
|
||||
Persist API key in browser storage
|
||||
</CheckboxItem>
|
||||
|
||||
<Paragraph>
|
||||
Once API key is set, you can use the <InlineIcon icon={MagicIcon} />{" "}
|
||||
tool to wrap your elements in a frame that will then allow you to
|
||||
turn it into code. This dialog can be accessed using the{" "}
|
||||
<b>AI Settings</b> <InlineIcon icon={OpenAIIcon} />.
|
||||
</Paragraph>
|
||||
|
||||
<FilledButton
|
||||
className="MagicSettings__confirm"
|
||||
size="large"
|
||||
label="Confirm"
|
||||
onClick={onConfirm}
|
||||
/>
|
||||
</TTDDialogTab>
|
||||
</TTDDialogTabs>
|
||||
</Dialog>
|
||||
);
|
||||
};
|
||||
|
|
@ -7,10 +7,7 @@ import { isMemberOf } from "../../utils";
|
|||
const TTDDialogTabs = (
|
||||
props: {
|
||||
children: ReactNode;
|
||||
} & (
|
||||
| { dialog: "ttd"; tab: "text-to-diagram" | "mermaid" }
|
||||
| { dialog: "settings"; tab: "text-to-diagram" | "diagram-to-code" }
|
||||
),
|
||||
} & { dialog: "ttd"; tab: "text-to-diagram" | "mermaid" },
|
||||
) => {
|
||||
const setAppState = useExcalidrawSetAppState();
|
||||
|
||||
|
|
@ -39,13 +36,6 @@ const TTDDialogTabs = (
|
|||
}
|
||||
}
|
||||
if (
|
||||
props.dialog === "settings" &&
|
||||
isMemberOf(["text-to-diagram", "diagram-to-code"], tab)
|
||||
) {
|
||||
setAppState({
|
||||
openDialog: { name: props.dialog, tab, source: "settings" },
|
||||
});
|
||||
} else if (
|
||||
props.dialog === "ttd" &&
|
||||
isMemberOf(["text-to-diagram", "mermaid"], tab)
|
||||
) {
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ const DropdownMenuItemLink = ({
|
|||
onSelect,
|
||||
className = "",
|
||||
selected,
|
||||
rel = "noreferrer",
|
||||
...rest
|
||||
}: {
|
||||
href: string;
|
||||
|
|
@ -22,6 +23,7 @@ const DropdownMenuItemLink = ({
|
|||
className?: string;
|
||||
selected?: boolean;
|
||||
onSelect?: (event: Event) => void;
|
||||
rel?: string;
|
||||
} & React.AnchorHTMLAttributes<HTMLAnchorElement>) => {
|
||||
const handleClick = useHandleDropdownMenuItemClick(rest.onClick, onSelect);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,105 +0,0 @@
|
|||
import { THEME } from "../constants";
|
||||
import type { Theme } from "../element/types";
|
||||
import type { DataURL } from "../types";
|
||||
import type { OpenAIInput, OpenAIOutput } from "./ai/types";
|
||||
|
||||
export type MagicCacheData =
|
||||
| {
|
||||
status: "pending";
|
||||
}
|
||||
| { status: "done"; html: string }
|
||||
| {
|
||||
status: "error";
|
||||
message?: string;
|
||||
code: "ERR_GENERATION_INTERRUPTED" | string;
|
||||
};
|
||||
|
||||
const SYSTEM_PROMPT = `You are a skilled front-end developer who builds interactive prototypes from wireframes, and is an expert at CSS Grid and Flex design.
|
||||
Your role is to transform low-fidelity wireframes into working front-end HTML code.
|
||||
|
||||
YOU MUST FOLLOW FOLLOWING RULES:
|
||||
|
||||
- Use HTML, CSS, JavaScript to build a responsive, accessible, polished prototype
|
||||
- Leverage Tailwind for styling and layout (import as script <script src="https://cdn.tailwindcss.com"></script>)
|
||||
- Inline JavaScript when needed
|
||||
- Fetch dependencies from CDNs when needed (using unpkg or skypack)
|
||||
- Source images from Unsplash or create applicable placeholders
|
||||
- Interpret annotations as intended vs literal UI
|
||||
- Fill gaps using your expertise in UX and business logic
|
||||
- generate primarily for desktop UI, but make it responsive.
|
||||
- Use grid and flexbox wherever applicable.
|
||||
- Convert the wireframe in its entirety, don't omit elements if possible.
|
||||
|
||||
If the wireframes, diagrams, or text is unclear or unreadable, refer to provided text for clarification.
|
||||
|
||||
Your goal is a production-ready prototype that brings the wireframes to life.
|
||||
|
||||
Please output JUST THE HTML file containing your best attempt at implementing the provided wireframes.`;
|
||||
|
||||
export async function diagramToHTML({
|
||||
image,
|
||||
apiKey,
|
||||
text,
|
||||
theme = THEME.LIGHT,
|
||||
}: {
|
||||
image: DataURL;
|
||||
apiKey: string;
|
||||
text: string;
|
||||
theme?: Theme;
|
||||
}) {
|
||||
const body: OpenAIInput.ChatCompletionCreateParamsBase = {
|
||||
model: "gpt-4-vision-preview",
|
||||
// 4096 are max output tokens allowed for `gpt-4-vision-preview` currently
|
||||
max_tokens: 4096,
|
||||
temperature: 0.1,
|
||||
messages: [
|
||||
{
|
||||
role: "system",
|
||||
content: SYSTEM_PROMPT,
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "image_url",
|
||||
image_url: {
|
||||
url: image,
|
||||
detail: "high",
|
||||
},
|
||||
},
|
||||
{
|
||||
type: "text",
|
||||
text: `Above is the reference wireframe. Please make a new website based on these and return just the HTML file. Also, please make it for the ${theme} theme. What follows are the wireframe's text annotations (if any)...`,
|
||||
},
|
||||
{
|
||||
type: "text",
|
||||
text,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
let result:
|
||||
| ({ ok: true } & OpenAIOutput.ChatCompletion)
|
||||
| ({ ok: false } & OpenAIOutput.APIError);
|
||||
|
||||
const resp = await fetch("https://api.openai.com/v1/chat/completions", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
if (resp.ok) {
|
||||
const json: OpenAIOutput.ChatCompletion = await resp.json();
|
||||
result = { ...json, ok: true };
|
||||
} else {
|
||||
const json: OpenAIOutput.APIError = await resp.json();
|
||||
result = { ...json, ok: false };
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
|
@ -46,7 +46,7 @@ export {
|
|||
dragNewElement,
|
||||
} from "./dragElements";
|
||||
export { isTextElement, isExcalidrawElement } from "./typeChecks";
|
||||
export { redrawTextBoundingBox } from "./textElement";
|
||||
export { redrawTextBoundingBox, getTextFromElements } from "./textElement";
|
||||
export {
|
||||
getPerfectElementSize,
|
||||
getLockedLinearCursorAlignSize,
|
||||
|
|
|
|||
|
|
@ -886,3 +886,19 @@ export const getMinTextElementWidth = (
|
|||
) => {
|
||||
return measureText("", font, lineHeight).width + BOUND_TEXT_PADDING * 2;
|
||||
};
|
||||
|
||||
/** retrieves text from text elements and concatenates to a single string */
|
||||
export const getTextFromElements = (
|
||||
elements: readonly ExcalidrawElement[],
|
||||
separator = "\n\n",
|
||||
) => {
|
||||
const text = elements
|
||||
.reduce((acc: string[], element) => {
|
||||
if (isTextElement(element)) {
|
||||
acc.push(element.text);
|
||||
}
|
||||
return acc;
|
||||
}, [])
|
||||
.join(separator);
|
||||
return text;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -12,7 +12,6 @@ import type {
|
|||
Merge,
|
||||
ValueOf,
|
||||
} from "../utility-types";
|
||||
import type { MagicCacheData } from "../data/magic";
|
||||
|
||||
export type ChartType = "bar" | "line";
|
||||
export type FillStyle = "hachure" | "cross-hatch" | "solid" | "zigzag";
|
||||
|
|
@ -101,11 +100,22 @@ export type ExcalidrawEmbeddableElement = _ExcalidrawElementBase &
|
|||
type: "embeddable";
|
||||
}>;
|
||||
|
||||
export type MagicGenerationData =
|
||||
| {
|
||||
status: "pending";
|
||||
}
|
||||
| { status: "done"; html: string }
|
||||
| {
|
||||
status: "error";
|
||||
message?: string;
|
||||
code: "ERR_GENERATION_INTERRUPTED" | string;
|
||||
};
|
||||
|
||||
export type ExcalidrawIframeElement = _ExcalidrawElementBase &
|
||||
Readonly<{
|
||||
type: "iframe";
|
||||
// TODO move later to AI-specific frame
|
||||
customData?: { generationData?: MagicCacheData };
|
||||
customData?: { generationData?: MagicGenerationData };
|
||||
}>;
|
||||
|
||||
export type ExcalidrawIframeLikeElement =
|
||||
|
|
|
|||
|
|
@ -213,6 +213,7 @@ export {
|
|||
hashString,
|
||||
isInvisiblySmallElement,
|
||||
getNonDeletedElements,
|
||||
getTextFromElements,
|
||||
} from "./element";
|
||||
export { defaultLang, useI18n, languages } from "./i18n";
|
||||
export {
|
||||
|
|
@ -287,3 +288,6 @@ export {
|
|||
isElementInsideBBox,
|
||||
elementPartiallyOverlapsWithOrContainsBBox,
|
||||
} from "../utils/withinBounds";
|
||||
|
||||
export { DiagramToCodePlugin } from "./components/DiagramToCodePlugin/DiagramToCodePlugin";
|
||||
export { getDataURL } from "./data/blob";
|
||||
|
|
|
|||
|
|
@ -272,8 +272,7 @@
|
|||
"laser": "Laser pointer",
|
||||
"hand": "Hand (panning tool)",
|
||||
"extraTools": "More tools",
|
||||
"mermaidToExcalidraw": "Mermaid to Excalidraw",
|
||||
"magicSettings": "AI settings"
|
||||
"mermaidToExcalidraw": "Mermaid to Excalidraw"
|
||||
},
|
||||
"element": {
|
||||
"rectangle": "Rectangle",
|
||||
|
|
|
|||
|
|
@ -325,14 +325,6 @@ export interface AppState {
|
|||
openDialog:
|
||||
| null
|
||||
| { name: "imageExport" | "help" | "jsonExport" }
|
||||
| {
|
||||
name: "settings";
|
||||
source:
|
||||
| "tool" // when magicframe tool is selected
|
||||
| "generation" // when magicframe generate button is clicked
|
||||
| "settings"; // when AI settings dialog is explicitly invoked
|
||||
tab: "text-to-diagram" | "diagram-to-code";
|
||||
}
|
||||
| { name: "ttd"; tab: "text-to-diagram" | "mermaid" }
|
||||
| { name: "commandPalette" };
|
||||
/**
|
||||
|
|
@ -655,6 +647,8 @@ export type AppClassProperties = {
|
|||
dismissLinearEditor: App["dismissLinearEditor"];
|
||||
flowChartCreator: App["flowChartCreator"];
|
||||
getEffectiveGridSize: App["getEffectiveGridSize"];
|
||||
setPlugins: App["setPlugins"];
|
||||
plugins: App["plugins"];
|
||||
};
|
||||
|
||||
export type PointerDownState = Readonly<{
|
||||
|
|
@ -842,3 +836,8 @@ export type PendingExcalidrawElements = ExcalidrawElement[];
|
|||
export type NullableGridSize =
|
||||
| (AppState["gridSize"] & MakeBrand<"NullableGridSize">)
|
||||
| null;
|
||||
|
||||
export type GenerateDiagramToCode = (props: {
|
||||
frame: ExcalidrawMagicFrameElement;
|
||||
children: readonly ExcalidrawElement[];
|
||||
}) => MaybePromise<{ html: string }>;
|
||||
|
|
|
|||
|
|
@ -1166,3 +1166,11 @@ export const promiseTry = async <TValue, TArgs extends unknown[]>(
|
|||
|
||||
export const isAnyTrue = (...args: boolean[]): boolean =>
|
||||
Math.max(...args.map((arg) => (arg ? 1 : 0))) > 0;
|
||||
|
||||
export const safelyParseJSON = (json: string): Record<string, any> | null => {
|
||||
try {
|
||||
return JSON.parse(json);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue